From 0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:47:29 +0200 Subject: Adding upstream version 115.8.0esr. Signed-off-by: Daniel Baumann --- intl/icu/source/i18n/BUILD.bazel | 130 + intl/icu/source/i18n/Makefile.in | 180 + intl/icu/source/i18n/alphaindex.cpp | 1235 +++ intl/icu/source/i18n/anytrans.cpp | 411 + intl/icu/source/i18n/anytrans.h | 131 + intl/icu/source/i18n/astro.cpp | 1603 ++++ intl/icu/source/i18n/astro.h | 757 ++ intl/icu/source/i18n/basictz.cpp | 539 ++ intl/icu/source/i18n/bocsu.cpp | 144 + intl/icu/source/i18n/bocsu.h | 161 + intl/icu/source/i18n/brktrans.cpp | 195 + intl/icu/source/i18n/brktrans.h | 104 + intl/icu/source/i18n/buddhcal.cpp | 174 + intl/icu/source/i18n/buddhcal.h | 187 + intl/icu/source/i18n/calendar.cpp | 4077 ++++++++++ intl/icu/source/i18n/casetrn.cpp | 193 + intl/icu/source/i18n/casetrn.h | 105 + intl/icu/source/i18n/cecal.cpp | 158 + intl/icu/source/i18n/cecal.h | 155 + intl/icu/source/i18n/chnsecal.cpp | 992 +++ intl/icu/source/i18n/chnsecal.h | 337 + intl/icu/source/i18n/choicfmt.cpp | 577 ++ intl/icu/source/i18n/coleitr.cpp | 473 ++ intl/icu/source/i18n/coll.cpp | 1021 +++ intl/icu/source/i18n/collation.cpp | 141 + intl/icu/source/i18n/collation.h | 503 ++ intl/icu/source/i18n/collationbuilder.cpp | 1723 ++++ intl/icu/source/i18n/collationbuilder.h | 411 + intl/icu/source/i18n/collationcompare.cpp | 356 + intl/icu/source/i18n/collationcompare.h | 38 + intl/icu/source/i18n/collationdata.cpp | 390 + intl/icu/source/i18n/collationdata.h | 258 + intl/icu/source/i18n/collationdatabuilder.cpp | 1683 ++++ intl/icu/source/i18n/collationdatabuilder.h | 269 + intl/icu/source/i18n/collationdatareader.cpp | 482 ++ intl/icu/source/i18n/collationdatareader.h | 253 + intl/icu/source/i18n/collationdatawriter.cpp | 352 + intl/icu/source/i18n/collationdatawriter.h | 57 + intl/icu/source/i18n/collationfastlatin.cpp | 1099 +++ intl/icu/source/i18n/collationfastlatin.h | 319 + intl/icu/source/i18n/collationfastlatinbuilder.cpp | 717 ++ intl/icu/source/i18n/collationfastlatinbuilder.h | 100 + intl/icu/source/i18n/collationfcd.cpp | 301 + intl/icu/source/i18n/collationfcd.h | 137 + intl/icu/source/i18n/collationiterator.cpp | 955 +++ intl/icu/source/i18n/collationiterator.h | 336 + intl/icu/source/i18n/collationkeys.cpp | 673 ++ intl/icu/source/i18n/collationkeys.h | 169 + intl/icu/source/i18n/collationroot.cpp | 140 + intl/icu/source/i18n/collationroot.h | 48 + intl/icu/source/i18n/collationrootelements.cpp | 341 + intl/icu/source/i18n/collationrootelements.h | 276 + intl/icu/source/i18n/collationruleparser.cpp | 881 +++ intl/icu/source/i18n/collationruleparser.h | 197 + intl/icu/source/i18n/collationsets.cpp | 612 ++ intl/icu/source/i18n/collationsets.h | 144 + intl/icu/source/i18n/collationsettings.cpp | 377 + intl/icu/source/i18n/collationsettings.h | 274 + intl/icu/source/i18n/collationtailoring.cpp | 113 + intl/icu/source/i18n/collationtailoring.h | 117 + intl/icu/source/i18n/collationweights.cpp | 570 ++ intl/icu/source/i18n/collationweights.h | 113 + intl/icu/source/i18n/collunsafe.h | 127 + intl/icu/source/i18n/compactdecimalformat.cpp | 75 + intl/icu/source/i18n/coptccal.cpp | 182 + intl/icu/source/i18n/coptccal.h | 258 + intl/icu/source/i18n/cpdtrans.cpp | 617 ++ intl/icu/source/i18n/cpdtrans.h | 232 + intl/icu/source/i18n/csdetect.cpp | 492 ++ intl/icu/source/i18n/csdetect.h | 69 + intl/icu/source/i18n/csmatch.cpp | 73 + intl/icu/source/i18n/csmatch.h | 71 + intl/icu/source/i18n/csr2022.cpp | 195 + intl/icu/source/i18n/csr2022.h | 95 + intl/icu/source/i18n/csrecog.cpp | 30 + intl/icu/source/i18n/csrecog.h | 57 + intl/icu/source/i18n/csrmbcs.cpp | 527 ++ intl/icu/source/i18n/csrmbcs.h | 207 + intl/icu/source/i18n/csrsbcs.cpp | 1271 +++ intl/icu/source/i18n/csrsbcs.h | 295 + intl/icu/source/i18n/csrucode.cpp | 200 + intl/icu/source/i18n/csrucode.h | 108 + intl/icu/source/i18n/csrutf8.cpp | 111 + intl/icu/source/i18n/csrutf8.h | 44 + intl/icu/source/i18n/curramt.cpp | 56 + intl/icu/source/i18n/currfmt.cpp | 62 + intl/icu/source/i18n/currfmt.h | 94 + intl/icu/source/i18n/currpinf.cpp | 441 ++ intl/icu/source/i18n/currunit.cpp | 126 + intl/icu/source/i18n/dangical.cpp | 167 + intl/icu/source/i18n/dangical.h | 135 + intl/icu/source/i18n/datefmt.cpp | 751 ++ intl/icu/source/i18n/dayperiodrules.cpp | 515 ++ intl/icu/source/i18n/dayperiodrules.h | 89 + intl/icu/source/i18n/dcfmtsym.cpp | 603 ++ intl/icu/source/i18n/decContext.cpp | 432 ++ intl/icu/source/i18n/decContext.h | 271 + intl/icu/source/i18n/decNumber.cpp | 8190 ++++++++++++++++++++ intl/icu/source/i18n/decNumber.h | 198 + intl/icu/source/i18n/decNumberLocal.h | 728 ++ intl/icu/source/i18n/decimfmt.cpp | 1920 +++++ intl/icu/source/i18n/displayoptions.cpp | 167 + .../source/i18n/double-conversion-bignum-dtoa.cpp | 659 ++ .../source/i18n/double-conversion-bignum-dtoa.h | 102 + intl/icu/source/i18n/double-conversion-bignum.cpp | 815 ++ intl/icu/source/i18n/double-conversion-bignum.h | 170 + .../i18n/double-conversion-cached-powers.cpp | 193 + .../source/i18n/double-conversion-cached-powers.h | 82 + intl/icu/source/i18n/double-conversion-diy-fp.h | 155 + .../i18n/double-conversion-double-to-string.cpp | 462 ++ .../i18n/double-conversion-double-to-string.h | 468 ++ .../source/i18n/double-conversion-fast-dtoa.cpp | 683 ++ intl/icu/source/i18n/double-conversion-fast-dtoa.h | 106 + intl/icu/source/i18n/double-conversion-ieee.h | 465 ++ .../i18n/double-conversion-string-to-double.cpp | 843 ++ .../i18n/double-conversion-string-to-double.h | 256 + intl/icu/source/i18n/double-conversion-strtod.cpp | 628 ++ intl/icu/source/i18n/double-conversion-strtod.h | 82 + intl/icu/source/i18n/double-conversion-utils.h | 435 ++ intl/icu/source/i18n/double-conversion.h | 46 + intl/icu/source/i18n/dt_impl.h | 92 + intl/icu/source/i18n/dtfmtsym.cpp | 2556 ++++++ intl/icu/source/i18n/dtitv_impl.h | 99 + intl/icu/source/i18n/dtitvfmt.cpp | 1909 +++++ intl/icu/source/i18n/dtitvinf.cpp | 814 ++ intl/icu/source/i18n/dtptngen.cpp | 3020 ++++++++ intl/icu/source/i18n/dtptngen_impl.h | 310 + intl/icu/source/i18n/dtrule.cpp | 141 + intl/icu/source/i18n/erarules.cpp | 326 + intl/icu/source/i18n/erarules.h | 99 + intl/icu/source/i18n/esctrn.cpp | 181 + intl/icu/source/i18n/esctrn.h | 144 + intl/icu/source/i18n/ethpccal.cpp | 259 + intl/icu/source/i18n/ethpccal.h | 367 + intl/icu/source/i18n/fmtable.cpp | 1042 +++ intl/icu/source/i18n/fmtable_cnv.cpp | 44 + intl/icu/source/i18n/fmtableimp.h | 31 + intl/icu/source/i18n/format.cpp | 219 + intl/icu/source/i18n/formatted_string_builder.cpp | 470 ++ intl/icu/source/i18n/formatted_string_builder.h | 272 + intl/icu/source/i18n/formattedval_impl.h | 318 + intl/icu/source/i18n/formattedval_iterimpl.cpp | 176 + intl/icu/source/i18n/formattedval_sbimpl.cpp | 350 + intl/icu/source/i18n/formattedvalue.cpp | 234 + intl/icu/source/i18n/fphdlimp.cpp | 125 + intl/icu/source/i18n/fphdlimp.h | 109 + intl/icu/source/i18n/fpositer.cpp | 114 + intl/icu/source/i18n/funcrepl.cpp | 130 + intl/icu/source/i18n/funcrepl.h | 121 + intl/icu/source/i18n/gender.cpp | 254 + intl/icu/source/i18n/gregocal.cpp | 1307 ++++ intl/icu/source/i18n/gregoimp.cpp | 176 + intl/icu/source/i18n/gregoimp.h | 312 + intl/icu/source/i18n/hebrwcal.cpp | 790 ++ intl/icu/source/i18n/hebrwcal.h | 492 ++ intl/icu/source/i18n/i18n.rc | 110 + intl/icu/source/i18n/i18n.vcxproj | 514 ++ intl/icu/source/i18n/i18n.vcxproj.filters | 1292 +++ intl/icu/source/i18n/i18n_uwp.vcxproj | 746 ++ intl/icu/source/i18n/indiancal.cpp | 379 + intl/icu/source/i18n/indiancal.h | 333 + intl/icu/source/i18n/inputext.cpp | 164 + intl/icu/source/i18n/inputext.h | 63 + intl/icu/source/i18n/islamcal.cpp | 979 +++ intl/icu/source/i18n/islamcal.h | 763 ++ intl/icu/source/i18n/iso8601cal.cpp | 37 + intl/icu/source/i18n/iso8601cal.h | 102 + intl/icu/source/i18n/japancal.cpp | 309 + intl/icu/source/i18n/japancal.h | 234 + intl/icu/source/i18n/listformatter.cpp | 732 ++ intl/icu/source/i18n/measfmt.cpp | 893 +++ intl/icu/source/i18n/measunit.cpp | 2391 ++++++ intl/icu/source/i18n/measunit_extra.cpp | 1260 +++ intl/icu/source/i18n/measunit_impl.h | 376 + intl/icu/source/i18n/measure.cpp | 78 + intl/icu/source/i18n/msgfmt.cpp | 2009 +++++ intl/icu/source/i18n/msgfmt_impl.h | 45 + intl/icu/source/i18n/name2uni.cpp | 259 + intl/icu/source/i18n/name2uni.h | 93 + intl/icu/source/i18n/nfrlist.h | 112 + intl/icu/source/i18n/nfrs.cpp | 1035 +++ intl/icu/source/i18n/nfrs.h | 111 + intl/icu/source/i18n/nfrule.cpp | 1632 ++++ intl/icu/source/i18n/nfrule.h | 129 + intl/icu/source/i18n/nfsubs.cpp | 1343 ++++ intl/icu/source/i18n/nfsubs.h | 262 + intl/icu/source/i18n/nortrans.cpp | 178 + intl/icu/source/i18n/nortrans.h | 102 + intl/icu/source/i18n/nultrans.cpp | 38 + intl/icu/source/i18n/nultrans.h | 73 + intl/icu/source/i18n/number_affixutils.cpp | 444 ++ intl/icu/source/i18n/number_affixutils.h | 244 + intl/icu/source/i18n/number_asformat.cpp | 117 + intl/icu/source/i18n/number_asformat.h | 106 + intl/icu/source/i18n/number_capi.cpp | 430 + intl/icu/source/i18n/number_compact.cpp | 353 + intl/icu/source/i18n/number_compact.h | 100 + intl/icu/source/i18n/number_currencysymbols.cpp | 135 + intl/icu/source/i18n/number_currencysymbols.h | 71 + intl/icu/source/i18n/number_decimalquantity.cpp | 1474 ++++ intl/icu/source/i18n/number_decimalquantity.h | 559 ++ intl/icu/source/i18n/number_decimfmtprops.cpp | 154 + intl/icu/source/i18n/number_decimfmtprops.h | 176 + intl/icu/source/i18n/number_decnum.h | 94 + intl/icu/source/i18n/number_fluent.cpp | 738 ++ intl/icu/source/i18n/number_formatimpl.cpp | 647 ++ intl/icu/source/i18n/number_formatimpl.h | 180 + intl/icu/source/i18n/number_grouping.cpp | 109 + intl/icu/source/i18n/number_integerwidth.cpp | 71 + intl/icu/source/i18n/number_longnames.cpp | 1766 +++++ intl/icu/source/i18n/number_longnames.h | 272 + intl/icu/source/i18n/number_mapper.cpp | 525 ++ intl/icu/source/i18n/number_mapper.h | 277 + intl/icu/source/i18n/number_microprops.h | 197 + intl/icu/source/i18n/number_modifiers.cpp | 494 ++ intl/icu/source/i18n/number_modifiers.h | 360 + intl/icu/source/i18n/number_multiplier.cpp | 160 + intl/icu/source/i18n/number_multiplier.h | 57 + intl/icu/source/i18n/number_notation.cpp | 88 + intl/icu/source/i18n/number_output.cpp | 86 + intl/icu/source/i18n/number_padding.cpp | 96 + intl/icu/source/i18n/number_patternmodifier.cpp | 348 + intl/icu/source/i18n/number_patternmodifier.h | 265 + intl/icu/source/i18n/number_patternstring.cpp | 1212 +++ intl/icu/source/i18n/number_patternstring.h | 340 + intl/icu/source/i18n/number_rounding.cpp | 552 ++ intl/icu/source/i18n/number_roundingutils.h | 247 + intl/icu/source/i18n/number_scientific.cpp | 177 + intl/icu/source/i18n/number_scientific.h | 68 + intl/icu/source/i18n/number_simple.cpp | 255 + intl/icu/source/i18n/number_skeletons.cpp | 1813 +++++ intl/icu/source/i18n/number_skeletons.h | 393 + intl/icu/source/i18n/number_symbolswrapper.cpp | 131 + intl/icu/source/i18n/number_types.h | 374 + intl/icu/source/i18n/number_usageprefs.cpp | 216 + intl/icu/source/i18n/number_usageprefs.h | 126 + intl/icu/source/i18n/number_utils.cpp | 275 + intl/icu/source/i18n/number_utils.h | 112 + intl/icu/source/i18n/number_utypes.h | 59 + intl/icu/source/i18n/numfmt.cpp | 1536 ++++ intl/icu/source/i18n/numparse_affixes.cpp | 463 ++ intl/icu/source/i18n/numparse_affixes.h | 230 + intl/icu/source/i18n/numparse_compositions.cpp | 108 + intl/icu/source/i18n/numparse_compositions.h | 124 + intl/icu/source/i18n/numparse_currency.cpp | 189 + intl/icu/source/i18n/numparse_currency.h | 74 + intl/icu/source/i18n/numparse_decimal.cpp | 459 ++ intl/icu/source/i18n/numparse_decimal.h | 76 + intl/icu/source/i18n/numparse_impl.cpp | 365 + intl/icu/source/i18n/numparse_impl.h | 111 + intl/icu/source/i18n/numparse_parsednumber.cpp | 126 + intl/icu/source/i18n/numparse_scientific.cpp | 163 + intl/icu/source/i18n/numparse_scientific.h | 47 + intl/icu/source/i18n/numparse_symbols.cpp | 198 + intl/icu/source/i18n/numparse_symbols.h | 173 + intl/icu/source/i18n/numparse_types.h | 272 + intl/icu/source/i18n/numparse_utils.h | 43 + intl/icu/source/i18n/numparse_validators.cpp | 85 + intl/icu/source/i18n/numparse_validators.h | 95 + intl/icu/source/i18n/numrange_capi.cpp | 198 + intl/icu/source/i18n/numrange_fluent.cpp | 410 + intl/icu/source/i18n/numrange_impl.cpp | 459 ++ intl/icu/source/i18n/numrange_impl.h | 89 + intl/icu/source/i18n/numsys.cpp | 362 + intl/icu/source/i18n/numsys_impl.h | 45 + intl/icu/source/i18n/olsontz.cpp | 1082 +++ intl/icu/source/i18n/olsontz.h | 455 ++ intl/icu/source/i18n/persncal.cpp | 301 + intl/icu/source/i18n/persncal.h | 325 + intl/icu/source/i18n/pluralranges.cpp | 144 + intl/icu/source/i18n/pluralranges.h | 67 + intl/icu/source/i18n/plurfmt.cpp | 607 ++ intl/icu/source/i18n/plurrule.cpp | 2006 +++++ intl/icu/source/i18n/plurrule_impl.h | 442 ++ intl/icu/source/i18n/quant.cpp | 151 + intl/icu/source/i18n/quant.h | 126 + intl/icu/source/i18n/quantityformatter.cpp | 243 + intl/icu/source/i18n/quantityformatter.h | 165 + intl/icu/source/i18n/rbnf.cpp | 1991 +++++ intl/icu/source/i18n/rbt.cpp | 307 + intl/icu/source/i18n/rbt.h | 223 + intl/icu/source/i18n/rbt_data.cpp | 119 + intl/icu/source/i18n/rbt_data.h | 154 + intl/icu/source/i18n/rbt_pars.cpp | 1773 +++++ intl/icu/source/i18n/rbt_pars.h | 357 + intl/icu/source/i18n/rbt_rule.cpp | 559 ++ intl/icu/source/i18n/rbt_rule.h | 310 + intl/icu/source/i18n/rbt_set.cpp | 466 ++ intl/icu/source/i18n/rbt_set.h | 167 + intl/icu/source/i18n/rbtz.cpp | 936 +++ intl/icu/source/i18n/regexcmp.cpp | 4675 +++++++++++ intl/icu/source/i18n/regexcmp.h | 234 + intl/icu/source/i18n/regexcst.h | 570 ++ intl/icu/source/i18n/regexcst.pl | 335 + intl/icu/source/i18n/regexcst.txt | 505 ++ intl/icu/source/i18n/regeximp.cpp | 120 + intl/icu/source/i18n/regeximp.h | 414 + intl/icu/source/i18n/regexst.cpp | 172 + intl/icu/source/i18n/regexst.h | 60 + intl/icu/source/i18n/regextxt.cpp | 48 + intl/icu/source/i18n/regextxt.h | 50 + intl/icu/source/i18n/region.cpp | 772 ++ intl/icu/source/i18n/region_impl.h | 49 + intl/icu/source/i18n/reldatefmt.cpp | 1424 ++++ intl/icu/source/i18n/reldtfmt.cpp | 595 ++ intl/icu/source/i18n/reldtfmt.h | 338 + intl/icu/source/i18n/rematch.cpp | 5733 ++++++++++++++ intl/icu/source/i18n/remtrans.cpp | 71 + intl/icu/source/i18n/remtrans.h | 80 + intl/icu/source/i18n/repattrn.cpp | 875 +++ intl/icu/source/i18n/rulebasedcollator.cpp | 1656 ++++ intl/icu/source/i18n/scientificnumberformatter.cpp | 305 + intl/icu/source/i18n/scriptset.cpp | 313 + intl/icu/source/i18n/scriptset.h | 89 + intl/icu/source/i18n/search.cpp | 445 ++ intl/icu/source/i18n/selfmt.cpp | 197 + intl/icu/source/i18n/selfmtimpl.h | 94 + intl/icu/source/i18n/sharedbreakiterator.cpp | 29 + intl/icu/source/i18n/sharedbreakiterator.h | 49 + intl/icu/source/i18n/sharedcalendar.h | 42 + intl/icu/source/i18n/shareddateformatsymbols.h | 47 + intl/icu/source/i18n/sharednumberformat.h | 41 + intl/icu/source/i18n/sharedpluralrules.h | 40 + intl/icu/source/i18n/simpletz.cpp | 1263 +++ intl/icu/source/i18n/smpdtfmt.cpp | 4413 +++++++++++ intl/icu/source/i18n/smpdtfst.cpp | 137 + intl/icu/source/i18n/smpdtfst.h | 52 + intl/icu/source/i18n/sortkey.cpp | 287 + intl/icu/source/i18n/sources.txt | 243 + intl/icu/source/i18n/standardplural.cpp | 163 + intl/icu/source/i18n/standardplural.h | 134 + intl/icu/source/i18n/string_segment.cpp | 145 + intl/icu/source/i18n/string_segment.h | 134 + intl/icu/source/i18n/strmatch.cpp | 296 + intl/icu/source/i18n/strmatch.h | 252 + intl/icu/source/i18n/strrepl.cpp | 329 + intl/icu/source/i18n/strrepl.h | 163 + intl/icu/source/i18n/stsearch.cpp | 484 ++ intl/icu/source/i18n/taiwncal.cpp | 183 + intl/icu/source/i18n/taiwncal.h | 184 + intl/icu/source/i18n/timezone.cpp | 1740 +++++ intl/icu/source/i18n/titletrn.cpp | 170 + intl/icu/source/i18n/titletrn.h | 92 + intl/icu/source/i18n/tmunit.cpp | 131 + intl/icu/source/i18n/tmutamt.cpp | 78 + intl/icu/source/i18n/tmutfmt.cpp | 806 ++ intl/icu/source/i18n/tolowtrn.cpp | 67 + intl/icu/source/i18n/tolowtrn.h | 76 + intl/icu/source/i18n/toupptrn.cpp | 67 + intl/icu/source/i18n/toupptrn.h | 76 + intl/icu/source/i18n/translit.cpp | 1678 ++++ intl/icu/source/i18n/transreg.cpp | 1406 ++++ intl/icu/source/i18n/transreg.h | 468 ++ intl/icu/source/i18n/tridpars.cpp | 932 +++ intl/icu/source/i18n/tridpars.h | 363 + intl/icu/source/i18n/tzfmt.cpp | 2913 +++++++ intl/icu/source/i18n/tzgnames.cpp | 1327 ++++ intl/icu/source/i18n/tzgnames.h | 67 + intl/icu/source/i18n/tznames.cpp | 507 ++ intl/icu/source/i18n/tznames_impl.cpp | 2308 ++++++ intl/icu/source/i18n/tznames_impl.h | 267 + intl/icu/source/i18n/tzrule.cpp | 629 ++ intl/icu/source/i18n/tztrans.cpp | 148 + intl/icu/source/i18n/ucal.cpp | 870 +++ intl/icu/source/i18n/ucln_in.cpp | 65 + intl/icu/source/i18n/ucln_in.h | 74 + intl/icu/source/i18n/ucol.cpp | 627 ++ intl/icu/source/i18n/ucol_imp.h | 139 + intl/icu/source/i18n/ucol_res.cpp | 701 ++ intl/icu/source/i18n/ucol_sit.cpp | 659 ++ intl/icu/source/i18n/ucoleitr.cpp | 531 ++ intl/icu/source/i18n/ucsdet.cpp | 205 + intl/icu/source/i18n/udat.cpp | 1364 ++++ intl/icu/source/i18n/udateintervalformat.cpp | 176 + intl/icu/source/i18n/udatpg.cpp | 334 + intl/icu/source/i18n/ufieldpositer.cpp | 61 + intl/icu/source/i18n/uitercollationiterator.cpp | 450 ++ intl/icu/source/i18n/uitercollationiterator.h | 162 + intl/icu/source/i18n/ulistformatter.cpp | 160 + intl/icu/source/i18n/ulocdata.cpp | 389 + intl/icu/source/i18n/umsg.cpp | 708 ++ intl/icu/source/i18n/umsg_imp.h | 47 + intl/icu/source/i18n/unesctrn.cpp | 293 + intl/icu/source/i18n/unesctrn.h | 112 + intl/icu/source/i18n/uni2name.cpp | 123 + intl/icu/source/i18n/uni2name.h | 89 + intl/icu/source/i18n/unicode/alphaindex.h | 766 ++ intl/icu/source/i18n/unicode/basictz.h | 248 + intl/icu/source/i18n/unicode/calendar.h | 2567 ++++++ intl/icu/source/i18n/unicode/choicfmt.h | 601 ++ intl/icu/source/i18n/unicode/coleitr.h | 411 + intl/icu/source/i18n/unicode/coll.h | 1294 ++++ .../icu/source/i18n/unicode/compactdecimalformat.h | 196 + intl/icu/source/i18n/unicode/curramt.h | 133 + intl/icu/source/i18n/unicode/currpinf.h | 274 + intl/icu/source/i18n/unicode/currunit.h | 146 + intl/icu/source/i18n/unicode/datefmt.h | 969 +++ intl/icu/source/i18n/unicode/dcfmtsym.h | 614 ++ intl/icu/source/i18n/unicode/decimfmt.h | 2212 ++++++ intl/icu/source/i18n/unicode/displayoptions.h | 274 + intl/icu/source/i18n/unicode/dtfmtsym.h | 1032 +++ intl/icu/source/i18n/unicode/dtitvfmt.h | 1212 +++ intl/icu/source/i18n/unicode/dtitvinf.h | 524 ++ intl/icu/source/i18n/unicode/dtptngen.h | 668 ++ intl/icu/source/i18n/unicode/dtrule.h | 256 + intl/icu/source/i18n/unicode/fieldpos.h | 298 + intl/icu/source/i18n/unicode/fmtable.h | 759 ++ intl/icu/source/i18n/unicode/format.h | 311 + intl/icu/source/i18n/unicode/formattednumber.h | 215 + intl/icu/source/i18n/unicode/formattedvalue.h | 318 + intl/icu/source/i18n/unicode/fpositer.h | 124 + intl/icu/source/i18n/unicode/gender.h | 122 + intl/icu/source/i18n/unicode/gregocal.h | 755 ++ intl/icu/source/i18n/unicode/listformatter.h | 286 + intl/icu/source/i18n/unicode/measfmt.h | 398 + intl/icu/source/i18n/unicode/measunit.h | 3805 +++++++++ intl/icu/source/i18n/unicode/measure.h | 166 + intl/icu/source/i18n/unicode/msgfmt.h | 1117 +++ intl/icu/source/i18n/unicode/nounit.h | 86 + intl/icu/source/i18n/unicode/numberformatter.h | 2787 +++++++ .../icu/source/i18n/unicode/numberrangeformatter.h | 777 ++ intl/icu/source/i18n/unicode/numfmt.h | 1288 +++ intl/icu/source/i18n/unicode/numsys.h | 220 + intl/icu/source/i18n/unicode/plurfmt.h | 606 ++ intl/icu/source/i18n/unicode/plurrule.h | 591 ++ intl/icu/source/i18n/unicode/rbnf.h | 1142 +++ intl/icu/source/i18n/unicode/rbtz.h | 373 + intl/icu/source/i18n/unicode/regex.h | 1882 +++++ intl/icu/source/i18n/unicode/region.h | 229 + intl/icu/source/i18n/unicode/reldatefmt.h | 751 ++ .../i18n/unicode/scientificnumberformatter.h | 222 + intl/icu/source/i18n/unicode/search.h | 580 ++ intl/icu/source/i18n/unicode/selfmt.h | 374 + .../source/i18n/unicode/simplenumberformatter.h | 329 + intl/icu/source/i18n/unicode/simpletz.h | 940 +++ intl/icu/source/i18n/unicode/smpdtfmt.h | 1672 ++++ intl/icu/source/i18n/unicode/sortkey.h | 344 + intl/icu/source/i18n/unicode/stsearch.h | 509 ++ intl/icu/source/i18n/unicode/tblcoll.h | 886 +++ intl/icu/source/i18n/unicode/timezone.h | 1038 +++ intl/icu/source/i18n/unicode/tmunit.h | 142 + intl/icu/source/i18n/unicode/tmutamt.h | 176 + intl/icu/source/i18n/unicode/tmutfmt.h | 238 + intl/icu/source/i18n/unicode/translit.h | 1593 ++++ intl/icu/source/i18n/unicode/tzfmt.h | 1102 +++ intl/icu/source/i18n/unicode/tznames.h | 419 + intl/icu/source/i18n/unicode/tzrule.h | 820 ++ intl/icu/source/i18n/unicode/tztrans.h | 201 + intl/icu/source/i18n/unicode/ucal.h | 1747 +++++ intl/icu/source/i18n/unicode/ucol.h | 1515 ++++ intl/icu/source/i18n/unicode/ucoleitr.h | 276 + intl/icu/source/i18n/unicode/ucsdet.h | 422 + intl/icu/source/i18n/unicode/udat.h | 1741 +++++ intl/icu/source/i18n/unicode/udateintervalformat.h | 332 + intl/icu/source/i18n/unicode/udatpg.h | 751 ++ intl/icu/source/i18n/unicode/udisplayoptions.h | 325 + intl/icu/source/i18n/unicode/ufieldpositer.h | 123 + intl/icu/source/i18n/unicode/uformattable.h | 290 + intl/icu/source/i18n/unicode/uformattednumber.h | 224 + intl/icu/source/i18n/unicode/uformattedvalue.h | 445 ++ intl/icu/source/i18n/unicode/ugender.h | 86 + intl/icu/source/i18n/unicode/ulistformatter.h | 333 + intl/icu/source/i18n/unicode/ulocdata.h | 299 + intl/icu/source/i18n/unicode/umsg.h | 628 ++ intl/icu/source/i18n/unicode/unirepl.h | 103 + intl/icu/source/i18n/unicode/unum.h | 1509 ++++ intl/icu/source/i18n/unicode/unumberformatter.h | 576 ++ intl/icu/source/i18n/unicode/unumberoptions.h | 173 + .../source/i18n/unicode/unumberrangeformatter.h | 471 ++ intl/icu/source/i18n/unicode/unumsys.h | 176 + intl/icu/source/i18n/unicode/upluralrules.h | 249 + intl/icu/source/i18n/unicode/uregex.h | 1617 ++++ intl/icu/source/i18n/unicode/uregion.h | 252 + intl/icu/source/i18n/unicode/ureldatefmt.h | 510 ++ intl/icu/source/i18n/unicode/usearch.h | 911 +++ .../source/i18n/unicode/usimplenumberformatter.h | 305 + intl/icu/source/i18n/unicode/uspoof.h | 1577 ++++ intl/icu/source/i18n/unicode/utmscale.h | 490 ++ intl/icu/source/i18n/unicode/utrans.h | 660 ++ intl/icu/source/i18n/unicode/vtzone.h | 471 ++ intl/icu/source/i18n/units_complexconverter.cpp | 275 + intl/icu/source/i18n/units_complexconverter.h | 136 + intl/icu/source/i18n/units_converter.cpp | 634 ++ intl/icu/source/i18n/units_converter.h | 226 + intl/icu/source/i18n/units_data.cpp | 488 ++ intl/icu/source/i18n/units_data.h | 219 + intl/icu/source/i18n/units_router.cpp | 149 + intl/icu/source/i18n/units_router.h | 166 + intl/icu/source/i18n/unum.cpp | 988 +++ intl/icu/source/i18n/unumsys.cpp | 88 + intl/icu/source/i18n/upluralrules.cpp | 182 + intl/icu/source/i18n/uregex.cpp | 1980 +++++ intl/icu/source/i18n/uregexc.cpp | 42 + intl/icu/source/i18n/uregion.cpp | 117 + intl/icu/source/i18n/usearch.cpp | 2531 ++++++ intl/icu/source/i18n/uspoof.cpp | 840 ++ intl/icu/source/i18n/uspoof_build.cpp | 108 + intl/icu/source/i18n/uspoof_conf.cpp | 477 ++ intl/icu/source/i18n/uspoof_conf.h | 135 + intl/icu/source/i18n/uspoof_impl.cpp | 959 +++ intl/icu/source/i18n/uspoof_impl.h | 344 + intl/icu/source/i18n/usrchimp.h | 243 + intl/icu/source/i18n/utf16collationiterator.cpp | 494 ++ intl/icu/source/i18n/utf16collationiterator.h | 186 + intl/icu/source/i18n/utf8collationiterator.cpp | 529 ++ intl/icu/source/i18n/utf8collationiterator.h | 174 + intl/icu/source/i18n/utmscale.cpp | 114 + intl/icu/source/i18n/utrans.cpp | 533 ++ intl/icu/source/i18n/vtzone.cpp | 2608 +++++++ intl/icu/source/i18n/vzone.cpp | 187 + intl/icu/source/i18n/vzone.h | 361 + intl/icu/source/i18n/windtfmt.cpp | 409 + intl/icu/source/i18n/windtfmt.h | 139 + intl/icu/source/i18n/winnmfmt.cpp | 461 ++ intl/icu/source/i18n/winnmfmt.h | 167 + intl/icu/source/i18n/wintzimpl.cpp | 161 + intl/icu/source/i18n/wintzimpl.h | 39 + intl/icu/source/i18n/zonemeta.cpp | 928 +++ intl/icu/source/i18n/zonemeta.h | 125 + intl/icu/source/i18n/zrule.cpp | 151 + intl/icu/source/i18n/zrule.h | 279 + intl/icu/source/i18n/ztrans.cpp | 103 + intl/icu/source/i18n/ztrans.h | 172 + 523 files changed, 269599 insertions(+) create mode 100644 intl/icu/source/i18n/BUILD.bazel create mode 100644 intl/icu/source/i18n/Makefile.in create mode 100644 intl/icu/source/i18n/alphaindex.cpp create mode 100644 intl/icu/source/i18n/anytrans.cpp create mode 100644 intl/icu/source/i18n/anytrans.h create mode 100644 intl/icu/source/i18n/astro.cpp create mode 100644 intl/icu/source/i18n/astro.h create mode 100644 intl/icu/source/i18n/basictz.cpp create mode 100644 intl/icu/source/i18n/bocsu.cpp create mode 100644 intl/icu/source/i18n/bocsu.h create mode 100644 intl/icu/source/i18n/brktrans.cpp create mode 100644 intl/icu/source/i18n/brktrans.h create mode 100644 intl/icu/source/i18n/buddhcal.cpp create mode 100644 intl/icu/source/i18n/buddhcal.h create mode 100644 intl/icu/source/i18n/calendar.cpp create mode 100644 intl/icu/source/i18n/casetrn.cpp create mode 100644 intl/icu/source/i18n/casetrn.h create mode 100644 intl/icu/source/i18n/cecal.cpp create mode 100644 intl/icu/source/i18n/cecal.h create mode 100644 intl/icu/source/i18n/chnsecal.cpp create mode 100644 intl/icu/source/i18n/chnsecal.h create mode 100644 intl/icu/source/i18n/choicfmt.cpp create mode 100644 intl/icu/source/i18n/coleitr.cpp create mode 100644 intl/icu/source/i18n/coll.cpp create mode 100644 intl/icu/source/i18n/collation.cpp create mode 100644 intl/icu/source/i18n/collation.h create mode 100644 intl/icu/source/i18n/collationbuilder.cpp create mode 100644 intl/icu/source/i18n/collationbuilder.h create mode 100644 intl/icu/source/i18n/collationcompare.cpp create mode 100644 intl/icu/source/i18n/collationcompare.h create mode 100644 intl/icu/source/i18n/collationdata.cpp create mode 100644 intl/icu/source/i18n/collationdata.h create mode 100644 intl/icu/source/i18n/collationdatabuilder.cpp create mode 100644 intl/icu/source/i18n/collationdatabuilder.h create mode 100644 intl/icu/source/i18n/collationdatareader.cpp create mode 100644 intl/icu/source/i18n/collationdatareader.h create mode 100644 intl/icu/source/i18n/collationdatawriter.cpp create mode 100644 intl/icu/source/i18n/collationdatawriter.h create mode 100644 intl/icu/source/i18n/collationfastlatin.cpp create mode 100644 intl/icu/source/i18n/collationfastlatin.h create mode 100644 intl/icu/source/i18n/collationfastlatinbuilder.cpp create mode 100644 intl/icu/source/i18n/collationfastlatinbuilder.h create mode 100644 intl/icu/source/i18n/collationfcd.cpp create mode 100644 intl/icu/source/i18n/collationfcd.h create mode 100644 intl/icu/source/i18n/collationiterator.cpp create mode 100644 intl/icu/source/i18n/collationiterator.h create mode 100644 intl/icu/source/i18n/collationkeys.cpp create mode 100644 intl/icu/source/i18n/collationkeys.h create mode 100644 intl/icu/source/i18n/collationroot.cpp create mode 100644 intl/icu/source/i18n/collationroot.h create mode 100644 intl/icu/source/i18n/collationrootelements.cpp create mode 100644 intl/icu/source/i18n/collationrootelements.h create mode 100644 intl/icu/source/i18n/collationruleparser.cpp create mode 100644 intl/icu/source/i18n/collationruleparser.h create mode 100644 intl/icu/source/i18n/collationsets.cpp create mode 100644 intl/icu/source/i18n/collationsets.h create mode 100644 intl/icu/source/i18n/collationsettings.cpp create mode 100644 intl/icu/source/i18n/collationsettings.h create mode 100644 intl/icu/source/i18n/collationtailoring.cpp create mode 100644 intl/icu/source/i18n/collationtailoring.h create mode 100644 intl/icu/source/i18n/collationweights.cpp create mode 100644 intl/icu/source/i18n/collationweights.h create mode 100644 intl/icu/source/i18n/collunsafe.h create mode 100644 intl/icu/source/i18n/compactdecimalformat.cpp create mode 100644 intl/icu/source/i18n/coptccal.cpp create mode 100644 intl/icu/source/i18n/coptccal.h create mode 100644 intl/icu/source/i18n/cpdtrans.cpp create mode 100644 intl/icu/source/i18n/cpdtrans.h create mode 100644 intl/icu/source/i18n/csdetect.cpp create mode 100644 intl/icu/source/i18n/csdetect.h create mode 100644 intl/icu/source/i18n/csmatch.cpp create mode 100644 intl/icu/source/i18n/csmatch.h create mode 100644 intl/icu/source/i18n/csr2022.cpp create mode 100644 intl/icu/source/i18n/csr2022.h create mode 100644 intl/icu/source/i18n/csrecog.cpp create mode 100644 intl/icu/source/i18n/csrecog.h create mode 100644 intl/icu/source/i18n/csrmbcs.cpp create mode 100644 intl/icu/source/i18n/csrmbcs.h create mode 100644 intl/icu/source/i18n/csrsbcs.cpp create mode 100644 intl/icu/source/i18n/csrsbcs.h create mode 100644 intl/icu/source/i18n/csrucode.cpp create mode 100644 intl/icu/source/i18n/csrucode.h create mode 100644 intl/icu/source/i18n/csrutf8.cpp create mode 100644 intl/icu/source/i18n/csrutf8.h create mode 100644 intl/icu/source/i18n/curramt.cpp create mode 100644 intl/icu/source/i18n/currfmt.cpp create mode 100644 intl/icu/source/i18n/currfmt.h create mode 100644 intl/icu/source/i18n/currpinf.cpp create mode 100644 intl/icu/source/i18n/currunit.cpp create mode 100644 intl/icu/source/i18n/dangical.cpp create mode 100644 intl/icu/source/i18n/dangical.h create mode 100644 intl/icu/source/i18n/datefmt.cpp create mode 100644 intl/icu/source/i18n/dayperiodrules.cpp create mode 100644 intl/icu/source/i18n/dayperiodrules.h create mode 100644 intl/icu/source/i18n/dcfmtsym.cpp create mode 100644 intl/icu/source/i18n/decContext.cpp create mode 100644 intl/icu/source/i18n/decContext.h create mode 100644 intl/icu/source/i18n/decNumber.cpp create mode 100644 intl/icu/source/i18n/decNumber.h create mode 100644 intl/icu/source/i18n/decNumberLocal.h create mode 100644 intl/icu/source/i18n/decimfmt.cpp create mode 100644 intl/icu/source/i18n/displayoptions.cpp create mode 100644 intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp create mode 100644 intl/icu/source/i18n/double-conversion-bignum-dtoa.h create mode 100644 intl/icu/source/i18n/double-conversion-bignum.cpp create mode 100644 intl/icu/source/i18n/double-conversion-bignum.h create mode 100644 intl/icu/source/i18n/double-conversion-cached-powers.cpp create mode 100644 intl/icu/source/i18n/double-conversion-cached-powers.h create mode 100644 intl/icu/source/i18n/double-conversion-diy-fp.h create mode 100644 intl/icu/source/i18n/double-conversion-double-to-string.cpp create mode 100644 intl/icu/source/i18n/double-conversion-double-to-string.h create mode 100644 intl/icu/source/i18n/double-conversion-fast-dtoa.cpp create mode 100644 intl/icu/source/i18n/double-conversion-fast-dtoa.h create mode 100644 intl/icu/source/i18n/double-conversion-ieee.h create mode 100644 intl/icu/source/i18n/double-conversion-string-to-double.cpp create mode 100644 intl/icu/source/i18n/double-conversion-string-to-double.h create mode 100644 intl/icu/source/i18n/double-conversion-strtod.cpp create mode 100644 intl/icu/source/i18n/double-conversion-strtod.h create mode 100644 intl/icu/source/i18n/double-conversion-utils.h create mode 100644 intl/icu/source/i18n/double-conversion.h create mode 100644 intl/icu/source/i18n/dt_impl.h create mode 100644 intl/icu/source/i18n/dtfmtsym.cpp create mode 100644 intl/icu/source/i18n/dtitv_impl.h create mode 100644 intl/icu/source/i18n/dtitvfmt.cpp create mode 100644 intl/icu/source/i18n/dtitvinf.cpp create mode 100644 intl/icu/source/i18n/dtptngen.cpp create mode 100644 intl/icu/source/i18n/dtptngen_impl.h create mode 100644 intl/icu/source/i18n/dtrule.cpp create mode 100644 intl/icu/source/i18n/erarules.cpp create mode 100644 intl/icu/source/i18n/erarules.h create mode 100644 intl/icu/source/i18n/esctrn.cpp create mode 100644 intl/icu/source/i18n/esctrn.h create mode 100644 intl/icu/source/i18n/ethpccal.cpp create mode 100644 intl/icu/source/i18n/ethpccal.h create mode 100644 intl/icu/source/i18n/fmtable.cpp create mode 100644 intl/icu/source/i18n/fmtable_cnv.cpp create mode 100644 intl/icu/source/i18n/fmtableimp.h create mode 100644 intl/icu/source/i18n/format.cpp create mode 100644 intl/icu/source/i18n/formatted_string_builder.cpp create mode 100644 intl/icu/source/i18n/formatted_string_builder.h create mode 100644 intl/icu/source/i18n/formattedval_impl.h create mode 100644 intl/icu/source/i18n/formattedval_iterimpl.cpp create mode 100644 intl/icu/source/i18n/formattedval_sbimpl.cpp create mode 100644 intl/icu/source/i18n/formattedvalue.cpp create mode 100644 intl/icu/source/i18n/fphdlimp.cpp create mode 100644 intl/icu/source/i18n/fphdlimp.h create mode 100644 intl/icu/source/i18n/fpositer.cpp create mode 100644 intl/icu/source/i18n/funcrepl.cpp create mode 100644 intl/icu/source/i18n/funcrepl.h create mode 100644 intl/icu/source/i18n/gender.cpp create mode 100644 intl/icu/source/i18n/gregocal.cpp create mode 100644 intl/icu/source/i18n/gregoimp.cpp create mode 100644 intl/icu/source/i18n/gregoimp.h create mode 100644 intl/icu/source/i18n/hebrwcal.cpp create mode 100644 intl/icu/source/i18n/hebrwcal.h create mode 100644 intl/icu/source/i18n/i18n.rc create mode 100644 intl/icu/source/i18n/i18n.vcxproj create mode 100644 intl/icu/source/i18n/i18n.vcxproj.filters create mode 100644 intl/icu/source/i18n/i18n_uwp.vcxproj create mode 100644 intl/icu/source/i18n/indiancal.cpp create mode 100644 intl/icu/source/i18n/indiancal.h create mode 100644 intl/icu/source/i18n/inputext.cpp create mode 100644 intl/icu/source/i18n/inputext.h create mode 100644 intl/icu/source/i18n/islamcal.cpp create mode 100644 intl/icu/source/i18n/islamcal.h create mode 100644 intl/icu/source/i18n/iso8601cal.cpp create mode 100644 intl/icu/source/i18n/iso8601cal.h create mode 100644 intl/icu/source/i18n/japancal.cpp create mode 100644 intl/icu/source/i18n/japancal.h create mode 100644 intl/icu/source/i18n/listformatter.cpp create mode 100644 intl/icu/source/i18n/measfmt.cpp create mode 100644 intl/icu/source/i18n/measunit.cpp create mode 100644 intl/icu/source/i18n/measunit_extra.cpp create mode 100644 intl/icu/source/i18n/measunit_impl.h create mode 100644 intl/icu/source/i18n/measure.cpp create mode 100644 intl/icu/source/i18n/msgfmt.cpp create mode 100644 intl/icu/source/i18n/msgfmt_impl.h create mode 100644 intl/icu/source/i18n/name2uni.cpp create mode 100644 intl/icu/source/i18n/name2uni.h create mode 100644 intl/icu/source/i18n/nfrlist.h create mode 100644 intl/icu/source/i18n/nfrs.cpp create mode 100644 intl/icu/source/i18n/nfrs.h create mode 100644 intl/icu/source/i18n/nfrule.cpp create mode 100644 intl/icu/source/i18n/nfrule.h create mode 100644 intl/icu/source/i18n/nfsubs.cpp create mode 100644 intl/icu/source/i18n/nfsubs.h create mode 100644 intl/icu/source/i18n/nortrans.cpp create mode 100644 intl/icu/source/i18n/nortrans.h create mode 100644 intl/icu/source/i18n/nultrans.cpp create mode 100644 intl/icu/source/i18n/nultrans.h create mode 100644 intl/icu/source/i18n/number_affixutils.cpp create mode 100644 intl/icu/source/i18n/number_affixutils.h create mode 100644 intl/icu/source/i18n/number_asformat.cpp create mode 100644 intl/icu/source/i18n/number_asformat.h create mode 100644 intl/icu/source/i18n/number_capi.cpp create mode 100644 intl/icu/source/i18n/number_compact.cpp create mode 100644 intl/icu/source/i18n/number_compact.h create mode 100644 intl/icu/source/i18n/number_currencysymbols.cpp create mode 100644 intl/icu/source/i18n/number_currencysymbols.h create mode 100644 intl/icu/source/i18n/number_decimalquantity.cpp create mode 100644 intl/icu/source/i18n/number_decimalquantity.h create mode 100644 intl/icu/source/i18n/number_decimfmtprops.cpp create mode 100644 intl/icu/source/i18n/number_decimfmtprops.h create mode 100644 intl/icu/source/i18n/number_decnum.h create mode 100644 intl/icu/source/i18n/number_fluent.cpp create mode 100644 intl/icu/source/i18n/number_formatimpl.cpp create mode 100644 intl/icu/source/i18n/number_formatimpl.h create mode 100644 intl/icu/source/i18n/number_grouping.cpp create mode 100644 intl/icu/source/i18n/number_integerwidth.cpp create mode 100644 intl/icu/source/i18n/number_longnames.cpp create mode 100644 intl/icu/source/i18n/number_longnames.h create mode 100644 intl/icu/source/i18n/number_mapper.cpp create mode 100644 intl/icu/source/i18n/number_mapper.h create mode 100644 intl/icu/source/i18n/number_microprops.h create mode 100644 intl/icu/source/i18n/number_modifiers.cpp create mode 100644 intl/icu/source/i18n/number_modifiers.h create mode 100644 intl/icu/source/i18n/number_multiplier.cpp create mode 100644 intl/icu/source/i18n/number_multiplier.h create mode 100644 intl/icu/source/i18n/number_notation.cpp create mode 100644 intl/icu/source/i18n/number_output.cpp create mode 100644 intl/icu/source/i18n/number_padding.cpp create mode 100644 intl/icu/source/i18n/number_patternmodifier.cpp create mode 100644 intl/icu/source/i18n/number_patternmodifier.h create mode 100644 intl/icu/source/i18n/number_patternstring.cpp create mode 100644 intl/icu/source/i18n/number_patternstring.h create mode 100644 intl/icu/source/i18n/number_rounding.cpp create mode 100644 intl/icu/source/i18n/number_roundingutils.h create mode 100644 intl/icu/source/i18n/number_scientific.cpp create mode 100644 intl/icu/source/i18n/number_scientific.h create mode 100644 intl/icu/source/i18n/number_simple.cpp create mode 100644 intl/icu/source/i18n/number_skeletons.cpp create mode 100644 intl/icu/source/i18n/number_skeletons.h create mode 100644 intl/icu/source/i18n/number_symbolswrapper.cpp create mode 100644 intl/icu/source/i18n/number_types.h create mode 100644 intl/icu/source/i18n/number_usageprefs.cpp create mode 100644 intl/icu/source/i18n/number_usageprefs.h create mode 100644 intl/icu/source/i18n/number_utils.cpp create mode 100644 intl/icu/source/i18n/number_utils.h create mode 100644 intl/icu/source/i18n/number_utypes.h create mode 100644 intl/icu/source/i18n/numfmt.cpp create mode 100644 intl/icu/source/i18n/numparse_affixes.cpp create mode 100644 intl/icu/source/i18n/numparse_affixes.h create mode 100644 intl/icu/source/i18n/numparse_compositions.cpp create mode 100644 intl/icu/source/i18n/numparse_compositions.h create mode 100644 intl/icu/source/i18n/numparse_currency.cpp create mode 100644 intl/icu/source/i18n/numparse_currency.h create mode 100644 intl/icu/source/i18n/numparse_decimal.cpp create mode 100644 intl/icu/source/i18n/numparse_decimal.h create mode 100644 intl/icu/source/i18n/numparse_impl.cpp create mode 100644 intl/icu/source/i18n/numparse_impl.h create mode 100644 intl/icu/source/i18n/numparse_parsednumber.cpp create mode 100644 intl/icu/source/i18n/numparse_scientific.cpp create mode 100644 intl/icu/source/i18n/numparse_scientific.h create mode 100644 intl/icu/source/i18n/numparse_symbols.cpp create mode 100644 intl/icu/source/i18n/numparse_symbols.h create mode 100644 intl/icu/source/i18n/numparse_types.h create mode 100644 intl/icu/source/i18n/numparse_utils.h create mode 100644 intl/icu/source/i18n/numparse_validators.cpp create mode 100644 intl/icu/source/i18n/numparse_validators.h create mode 100644 intl/icu/source/i18n/numrange_capi.cpp create mode 100644 intl/icu/source/i18n/numrange_fluent.cpp create mode 100644 intl/icu/source/i18n/numrange_impl.cpp create mode 100644 intl/icu/source/i18n/numrange_impl.h create mode 100644 intl/icu/source/i18n/numsys.cpp create mode 100644 intl/icu/source/i18n/numsys_impl.h create mode 100644 intl/icu/source/i18n/olsontz.cpp create mode 100644 intl/icu/source/i18n/olsontz.h create mode 100644 intl/icu/source/i18n/persncal.cpp create mode 100644 intl/icu/source/i18n/persncal.h create mode 100644 intl/icu/source/i18n/pluralranges.cpp create mode 100644 intl/icu/source/i18n/pluralranges.h create mode 100644 intl/icu/source/i18n/plurfmt.cpp create mode 100644 intl/icu/source/i18n/plurrule.cpp create mode 100644 intl/icu/source/i18n/plurrule_impl.h create mode 100644 intl/icu/source/i18n/quant.cpp create mode 100644 intl/icu/source/i18n/quant.h create mode 100644 intl/icu/source/i18n/quantityformatter.cpp create mode 100644 intl/icu/source/i18n/quantityformatter.h create mode 100644 intl/icu/source/i18n/rbnf.cpp create mode 100644 intl/icu/source/i18n/rbt.cpp create mode 100644 intl/icu/source/i18n/rbt.h create mode 100644 intl/icu/source/i18n/rbt_data.cpp create mode 100644 intl/icu/source/i18n/rbt_data.h create mode 100644 intl/icu/source/i18n/rbt_pars.cpp create mode 100644 intl/icu/source/i18n/rbt_pars.h create mode 100644 intl/icu/source/i18n/rbt_rule.cpp create mode 100644 intl/icu/source/i18n/rbt_rule.h create mode 100644 intl/icu/source/i18n/rbt_set.cpp create mode 100644 intl/icu/source/i18n/rbt_set.h create mode 100644 intl/icu/source/i18n/rbtz.cpp create mode 100644 intl/icu/source/i18n/regexcmp.cpp create mode 100644 intl/icu/source/i18n/regexcmp.h create mode 100644 intl/icu/source/i18n/regexcst.h create mode 100755 intl/icu/source/i18n/regexcst.pl create mode 100644 intl/icu/source/i18n/regexcst.txt create mode 100644 intl/icu/source/i18n/regeximp.cpp create mode 100644 intl/icu/source/i18n/regeximp.h create mode 100644 intl/icu/source/i18n/regexst.cpp create mode 100644 intl/icu/source/i18n/regexst.h create mode 100644 intl/icu/source/i18n/regextxt.cpp create mode 100644 intl/icu/source/i18n/regextxt.h create mode 100644 intl/icu/source/i18n/region.cpp create mode 100644 intl/icu/source/i18n/region_impl.h create mode 100644 intl/icu/source/i18n/reldatefmt.cpp create mode 100644 intl/icu/source/i18n/reldtfmt.cpp create mode 100644 intl/icu/source/i18n/reldtfmt.h create mode 100644 intl/icu/source/i18n/rematch.cpp create mode 100644 intl/icu/source/i18n/remtrans.cpp create mode 100644 intl/icu/source/i18n/remtrans.h create mode 100644 intl/icu/source/i18n/repattrn.cpp create mode 100644 intl/icu/source/i18n/rulebasedcollator.cpp create mode 100644 intl/icu/source/i18n/scientificnumberformatter.cpp create mode 100644 intl/icu/source/i18n/scriptset.cpp create mode 100644 intl/icu/source/i18n/scriptset.h create mode 100644 intl/icu/source/i18n/search.cpp create mode 100644 intl/icu/source/i18n/selfmt.cpp create mode 100644 intl/icu/source/i18n/selfmtimpl.h create mode 100644 intl/icu/source/i18n/sharedbreakiterator.cpp create mode 100644 intl/icu/source/i18n/sharedbreakiterator.h create mode 100644 intl/icu/source/i18n/sharedcalendar.h create mode 100644 intl/icu/source/i18n/shareddateformatsymbols.h create mode 100644 intl/icu/source/i18n/sharednumberformat.h create mode 100644 intl/icu/source/i18n/sharedpluralrules.h create mode 100644 intl/icu/source/i18n/simpletz.cpp create mode 100644 intl/icu/source/i18n/smpdtfmt.cpp create mode 100644 intl/icu/source/i18n/smpdtfst.cpp create mode 100644 intl/icu/source/i18n/smpdtfst.h create mode 100644 intl/icu/source/i18n/sortkey.cpp create mode 100644 intl/icu/source/i18n/sources.txt create mode 100644 intl/icu/source/i18n/standardplural.cpp create mode 100644 intl/icu/source/i18n/standardplural.h create mode 100644 intl/icu/source/i18n/string_segment.cpp create mode 100644 intl/icu/source/i18n/string_segment.h create mode 100644 intl/icu/source/i18n/strmatch.cpp create mode 100644 intl/icu/source/i18n/strmatch.h create mode 100644 intl/icu/source/i18n/strrepl.cpp create mode 100644 intl/icu/source/i18n/strrepl.h create mode 100644 intl/icu/source/i18n/stsearch.cpp create mode 100644 intl/icu/source/i18n/taiwncal.cpp create mode 100644 intl/icu/source/i18n/taiwncal.h create mode 100644 intl/icu/source/i18n/timezone.cpp create mode 100644 intl/icu/source/i18n/titletrn.cpp create mode 100644 intl/icu/source/i18n/titletrn.h create mode 100644 intl/icu/source/i18n/tmunit.cpp create mode 100644 intl/icu/source/i18n/tmutamt.cpp create mode 100644 intl/icu/source/i18n/tmutfmt.cpp create mode 100644 intl/icu/source/i18n/tolowtrn.cpp create mode 100644 intl/icu/source/i18n/tolowtrn.h create mode 100644 intl/icu/source/i18n/toupptrn.cpp create mode 100644 intl/icu/source/i18n/toupptrn.h create mode 100644 intl/icu/source/i18n/translit.cpp create mode 100644 intl/icu/source/i18n/transreg.cpp create mode 100644 intl/icu/source/i18n/transreg.h create mode 100644 intl/icu/source/i18n/tridpars.cpp create mode 100644 intl/icu/source/i18n/tridpars.h create mode 100644 intl/icu/source/i18n/tzfmt.cpp create mode 100644 intl/icu/source/i18n/tzgnames.cpp create mode 100644 intl/icu/source/i18n/tzgnames.h create mode 100644 intl/icu/source/i18n/tznames.cpp create mode 100644 intl/icu/source/i18n/tznames_impl.cpp create mode 100644 intl/icu/source/i18n/tznames_impl.h create mode 100644 intl/icu/source/i18n/tzrule.cpp create mode 100644 intl/icu/source/i18n/tztrans.cpp create mode 100644 intl/icu/source/i18n/ucal.cpp create mode 100644 intl/icu/source/i18n/ucln_in.cpp create mode 100644 intl/icu/source/i18n/ucln_in.h create mode 100644 intl/icu/source/i18n/ucol.cpp create mode 100644 intl/icu/source/i18n/ucol_imp.h create mode 100644 intl/icu/source/i18n/ucol_res.cpp create mode 100644 intl/icu/source/i18n/ucol_sit.cpp create mode 100644 intl/icu/source/i18n/ucoleitr.cpp create mode 100644 intl/icu/source/i18n/ucsdet.cpp create mode 100644 intl/icu/source/i18n/udat.cpp create mode 100644 intl/icu/source/i18n/udateintervalformat.cpp create mode 100644 intl/icu/source/i18n/udatpg.cpp create mode 100644 intl/icu/source/i18n/ufieldpositer.cpp create mode 100644 intl/icu/source/i18n/uitercollationiterator.cpp create mode 100644 intl/icu/source/i18n/uitercollationiterator.h create mode 100644 intl/icu/source/i18n/ulistformatter.cpp create mode 100644 intl/icu/source/i18n/ulocdata.cpp create mode 100644 intl/icu/source/i18n/umsg.cpp create mode 100644 intl/icu/source/i18n/umsg_imp.h create mode 100644 intl/icu/source/i18n/unesctrn.cpp create mode 100644 intl/icu/source/i18n/unesctrn.h create mode 100644 intl/icu/source/i18n/uni2name.cpp create mode 100644 intl/icu/source/i18n/uni2name.h create mode 100644 intl/icu/source/i18n/unicode/alphaindex.h create mode 100644 intl/icu/source/i18n/unicode/basictz.h create mode 100644 intl/icu/source/i18n/unicode/calendar.h create mode 100644 intl/icu/source/i18n/unicode/choicfmt.h create mode 100644 intl/icu/source/i18n/unicode/coleitr.h create mode 100644 intl/icu/source/i18n/unicode/coll.h create mode 100644 intl/icu/source/i18n/unicode/compactdecimalformat.h create mode 100644 intl/icu/source/i18n/unicode/curramt.h create mode 100644 intl/icu/source/i18n/unicode/currpinf.h create mode 100644 intl/icu/source/i18n/unicode/currunit.h create mode 100644 intl/icu/source/i18n/unicode/datefmt.h create mode 100644 intl/icu/source/i18n/unicode/dcfmtsym.h create mode 100644 intl/icu/source/i18n/unicode/decimfmt.h create mode 100644 intl/icu/source/i18n/unicode/displayoptions.h create mode 100644 intl/icu/source/i18n/unicode/dtfmtsym.h create mode 100644 intl/icu/source/i18n/unicode/dtitvfmt.h create mode 100644 intl/icu/source/i18n/unicode/dtitvinf.h create mode 100644 intl/icu/source/i18n/unicode/dtptngen.h create mode 100644 intl/icu/source/i18n/unicode/dtrule.h create mode 100644 intl/icu/source/i18n/unicode/fieldpos.h create mode 100644 intl/icu/source/i18n/unicode/fmtable.h create mode 100644 intl/icu/source/i18n/unicode/format.h create mode 100644 intl/icu/source/i18n/unicode/formattednumber.h create mode 100644 intl/icu/source/i18n/unicode/formattedvalue.h create mode 100644 intl/icu/source/i18n/unicode/fpositer.h create mode 100644 intl/icu/source/i18n/unicode/gender.h create mode 100644 intl/icu/source/i18n/unicode/gregocal.h create mode 100644 intl/icu/source/i18n/unicode/listformatter.h create mode 100644 intl/icu/source/i18n/unicode/measfmt.h create mode 100644 intl/icu/source/i18n/unicode/measunit.h create mode 100644 intl/icu/source/i18n/unicode/measure.h create mode 100644 intl/icu/source/i18n/unicode/msgfmt.h create mode 100644 intl/icu/source/i18n/unicode/nounit.h create mode 100644 intl/icu/source/i18n/unicode/numberformatter.h create mode 100644 intl/icu/source/i18n/unicode/numberrangeformatter.h create mode 100644 intl/icu/source/i18n/unicode/numfmt.h create mode 100644 intl/icu/source/i18n/unicode/numsys.h create mode 100644 intl/icu/source/i18n/unicode/plurfmt.h create mode 100644 intl/icu/source/i18n/unicode/plurrule.h create mode 100644 intl/icu/source/i18n/unicode/rbnf.h create mode 100644 intl/icu/source/i18n/unicode/rbtz.h create mode 100644 intl/icu/source/i18n/unicode/regex.h create mode 100644 intl/icu/source/i18n/unicode/region.h create mode 100644 intl/icu/source/i18n/unicode/reldatefmt.h create mode 100644 intl/icu/source/i18n/unicode/scientificnumberformatter.h create mode 100644 intl/icu/source/i18n/unicode/search.h create mode 100644 intl/icu/source/i18n/unicode/selfmt.h create mode 100644 intl/icu/source/i18n/unicode/simplenumberformatter.h create mode 100644 intl/icu/source/i18n/unicode/simpletz.h create mode 100644 intl/icu/source/i18n/unicode/smpdtfmt.h create mode 100644 intl/icu/source/i18n/unicode/sortkey.h create mode 100644 intl/icu/source/i18n/unicode/stsearch.h create mode 100644 intl/icu/source/i18n/unicode/tblcoll.h create mode 100644 intl/icu/source/i18n/unicode/timezone.h create mode 100644 intl/icu/source/i18n/unicode/tmunit.h create mode 100644 intl/icu/source/i18n/unicode/tmutamt.h create mode 100644 intl/icu/source/i18n/unicode/tmutfmt.h create mode 100644 intl/icu/source/i18n/unicode/translit.h create mode 100644 intl/icu/source/i18n/unicode/tzfmt.h create mode 100644 intl/icu/source/i18n/unicode/tznames.h create mode 100644 intl/icu/source/i18n/unicode/tzrule.h create mode 100644 intl/icu/source/i18n/unicode/tztrans.h create mode 100644 intl/icu/source/i18n/unicode/ucal.h create mode 100644 intl/icu/source/i18n/unicode/ucol.h create mode 100644 intl/icu/source/i18n/unicode/ucoleitr.h create mode 100644 intl/icu/source/i18n/unicode/ucsdet.h create mode 100644 intl/icu/source/i18n/unicode/udat.h create mode 100644 intl/icu/source/i18n/unicode/udateintervalformat.h create mode 100644 intl/icu/source/i18n/unicode/udatpg.h create mode 100644 intl/icu/source/i18n/unicode/udisplayoptions.h create mode 100644 intl/icu/source/i18n/unicode/ufieldpositer.h create mode 100644 intl/icu/source/i18n/unicode/uformattable.h create mode 100644 intl/icu/source/i18n/unicode/uformattednumber.h create mode 100644 intl/icu/source/i18n/unicode/uformattedvalue.h create mode 100644 intl/icu/source/i18n/unicode/ugender.h create mode 100644 intl/icu/source/i18n/unicode/ulistformatter.h create mode 100644 intl/icu/source/i18n/unicode/ulocdata.h create mode 100644 intl/icu/source/i18n/unicode/umsg.h create mode 100644 intl/icu/source/i18n/unicode/unirepl.h create mode 100644 intl/icu/source/i18n/unicode/unum.h create mode 100644 intl/icu/source/i18n/unicode/unumberformatter.h create mode 100644 intl/icu/source/i18n/unicode/unumberoptions.h create mode 100644 intl/icu/source/i18n/unicode/unumberrangeformatter.h create mode 100644 intl/icu/source/i18n/unicode/unumsys.h create mode 100644 intl/icu/source/i18n/unicode/upluralrules.h create mode 100644 intl/icu/source/i18n/unicode/uregex.h create mode 100644 intl/icu/source/i18n/unicode/uregion.h create mode 100644 intl/icu/source/i18n/unicode/ureldatefmt.h create mode 100644 intl/icu/source/i18n/unicode/usearch.h create mode 100644 intl/icu/source/i18n/unicode/usimplenumberformatter.h create mode 100644 intl/icu/source/i18n/unicode/uspoof.h create mode 100644 intl/icu/source/i18n/unicode/utmscale.h create mode 100644 intl/icu/source/i18n/unicode/utrans.h create mode 100644 intl/icu/source/i18n/unicode/vtzone.h create mode 100644 intl/icu/source/i18n/units_complexconverter.cpp create mode 100644 intl/icu/source/i18n/units_complexconverter.h create mode 100644 intl/icu/source/i18n/units_converter.cpp create mode 100644 intl/icu/source/i18n/units_converter.h create mode 100644 intl/icu/source/i18n/units_data.cpp create mode 100644 intl/icu/source/i18n/units_data.h create mode 100644 intl/icu/source/i18n/units_router.cpp create mode 100644 intl/icu/source/i18n/units_router.h create mode 100644 intl/icu/source/i18n/unum.cpp create mode 100644 intl/icu/source/i18n/unumsys.cpp create mode 100644 intl/icu/source/i18n/upluralrules.cpp create mode 100644 intl/icu/source/i18n/uregex.cpp create mode 100644 intl/icu/source/i18n/uregexc.cpp create mode 100644 intl/icu/source/i18n/uregion.cpp create mode 100644 intl/icu/source/i18n/usearch.cpp create mode 100644 intl/icu/source/i18n/uspoof.cpp create mode 100644 intl/icu/source/i18n/uspoof_build.cpp create mode 100644 intl/icu/source/i18n/uspoof_conf.cpp create mode 100644 intl/icu/source/i18n/uspoof_conf.h create mode 100644 intl/icu/source/i18n/uspoof_impl.cpp create mode 100644 intl/icu/source/i18n/uspoof_impl.h create mode 100644 intl/icu/source/i18n/usrchimp.h create mode 100644 intl/icu/source/i18n/utf16collationiterator.cpp create mode 100644 intl/icu/source/i18n/utf16collationiterator.h create mode 100644 intl/icu/source/i18n/utf8collationiterator.cpp create mode 100644 intl/icu/source/i18n/utf8collationiterator.h create mode 100644 intl/icu/source/i18n/utmscale.cpp create mode 100644 intl/icu/source/i18n/utrans.cpp create mode 100644 intl/icu/source/i18n/vtzone.cpp create mode 100644 intl/icu/source/i18n/vzone.cpp create mode 100644 intl/icu/source/i18n/vzone.h create mode 100644 intl/icu/source/i18n/windtfmt.cpp create mode 100644 intl/icu/source/i18n/windtfmt.h create mode 100644 intl/icu/source/i18n/winnmfmt.cpp create mode 100644 intl/icu/source/i18n/winnmfmt.h create mode 100644 intl/icu/source/i18n/wintzimpl.cpp create mode 100644 intl/icu/source/i18n/wintzimpl.h create mode 100644 intl/icu/source/i18n/zonemeta.cpp create mode 100644 intl/icu/source/i18n/zonemeta.h create mode 100644 intl/icu/source/i18n/zrule.cpp create mode 100644 intl/icu/source/i18n/zrule.h create mode 100644 intl/icu/source/i18n/ztrans.cpp create mode 100644 intl/icu/source/i18n/ztrans.h (limited to 'intl/icu/source/i18n') diff --git a/intl/icu/source/i18n/BUILD.bazel b/intl/icu/source/i18n/BUILD.bazel new file mode 100644 index 0000000000..2d85cdb180 --- /dev/null +++ b/intl/icu/source/i18n/BUILD.bazel @@ -0,0 +1,130 @@ +# © 2021 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +# This file defines Bazel targets for a subset of the ICU4C "i18n" library header and source files. +# The configuration of dependencies among targets is strongly assisted by the +# file in depstest that maintains such information, at +# icu4c/source/test/depstest/dependencies.txt . + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") + +package( + default_visibility = ["//visibility:public"], +) + +# When compiling code in the `common` dir, the constant +# `U_I18n_IMPLEMENTATION` needs to be defined. See +# https://unicode-org.github.io/icu/userguide/howtouseicu#c-with-your-own-build-system . + +# If linker errors occur, then this may be a sign that the dependencies were +# not specified correctly. Use dependencies.txt in depstest for assistance. See +# https://stackoverflow.com/q/66111709/2077918 . + +cc_library( + name = "headers", + hdrs = glob([ + "unicode/*.h", # public + "*.h", # internal + ]), + # We need to add includes in order to preserve existing source files' + # include directives that use traditional paths, not paths relative to + # Bazel workspace: + # https://stackoverflow.com/a/65635893/2077918 + includes = ["."], + local_defines = [ + "U_I18N_IMPLEMENTATION", + ], +) + +cc_library( + name = "collation", + srcs = [ + "bocsu.cpp", + "coleitr.cpp", + "coll.cpp", + "collation.cpp", + "collationcompare.cpp", + "collationdata.cpp", + "collationdatareader.cpp", + "collationdatawriter.cpp", + "collationfastlatin.cpp", + # collationfcd.cpp is generated by genuca; + # probably hard to build genuca without depending on the old version. + "collationfcd.cpp", + "collationiterator.cpp", + "collationkeys.cpp", + "collationroot.cpp", + "collationrootelements.cpp", + "collationsets.cpp", + "collationsettings.cpp", + "collationtailoring.cpp", + "rulebasedcollator.cpp", + "sortkey.cpp", + "ucol.cpp", + "ucol_res.cpp", + "ucol_sit.cpp", + "ucoleitr.cpp", + "uitercollationiterator.cpp", + "utf16collationiterator.cpp", + "utf8collationiterator.cpp", + ], + includes = ["."], + deps = [ + ":headers", + ":uclean_i18n", + "//icu4c/source/common:bytestream", + "//icu4c/source/common:normalizer2", + "//icu4c/source/common:platform", + "//icu4c/source/common:propname", + "//icu4c/source/common:resourcebundle", + "//icu4c/source/common:service_registration", + "//icu4c/source/common:ucharstrieiterator", + "//icu4c/source/common:uiter", + "//icu4c/source/common:ulist", + "//icu4c/source/common:unifiedcache", + "//icu4c/source/common:uset", + "//icu4c/source/common:usetiter", + "//icu4c/source/common:utrie2", + "//icu4c/source/common:uvector32", + "//icu4c/source/common:uvector64", + ], + local_defines = [ + "U_I18N_IMPLEMENTATION", + ], +) + +cc_library( + name = "collation_builder", + srcs = [ + "collationbuilder.cpp", + "collationdatabuilder.cpp", + "collationfastlatinbuilder.cpp", + "collationruleparser.cpp", + "collationweights.cpp", + ], + includes = ["."], + deps = [ + ":collation", + "//icu4c/source/common:canonical_iterator", + "//icu4c/source/common:ucharstriebuilder", + "//icu4c/source/common:uset_props" + ], + local_defines = [ + "U_I18N_IMPLEMENTATION", + ], +) + +cc_library( + name = "uclean_i18n", + srcs = [ + "ucln_in.cpp", + ], + hdrs = ["ucln_in.h"], + includes = ["."], + deps = [ + "//icu4c/source/common:platform", + ], + local_defines = [ + "U_I18N_IMPLEMENTATION", + ], +) diff --git a/intl/icu/source/i18n/Makefile.in b/intl/icu/source/i18n/Makefile.in new file mode 100644 index 0000000000..ce7fe006e6 --- /dev/null +++ b/intl/icu/source/i18n/Makefile.in @@ -0,0 +1,180 @@ +# Copyright (C) 2016 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html +#****************************************************************************** +# +# Copyright (C) 1998-2016, International Business Machines +# Corporation and others. All Rights Reserved. +# +#****************************************************************************** +## Makefile.in for ICU - icui18n.so +## Stephen F. Booth + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = .. + +## All the flags and other definitions are included here. +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = i18n + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(IMPORT_LIB) $(MIDDLE_IMPORT_LIB) $(FINAL_IMPORT_LIB) + +## Target information + +TARGET_STUBNAME=$(I18N_STUBNAME) + +ifneq ($(ENABLE_STATIC),) +TARGET = $(LIBDIR)/$(LIBSICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(A) +endif + +ifneq ($(ENABLE_SHARED),) +SO_TARGET = $(LIBDIR)/$(LIBICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(SO) +ALL_SO_TARGETS = $(SO_TARGET) $(MIDDLE_SO_TARGET) $(FINAL_SO_TARGET) $(SHARED_OBJECT) + +ifeq ($(ENABLE_SO_VERSION_DATA),1) +SO_VERSION_DATA = i18n.res +endif + +ifeq ($(OS390BATCH),1) +BATCH_TARGET = $(BATCH_I18N_TARGET) +BATCH_LIBS = $(BATCH_LIBICUUC) -lm +endif # OS390BATCH + +endif # ENABLE_SHARED + +ALL_TARGETS = $(TARGET) $(ALL_SO_TARGETS) $(BATCH_TARGET) + +DYNAMICCPPFLAGS = $(SHAREDLIBCPPFLAGS) +DYNAMICCFLAGS = $(SHAREDLIBCFLAGS) +DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS) +CFLAGS += $(LIBCFLAGS) +CXXFLAGS += $(LIBCXXFLAGS) +ifeq ($(OS390BATCH),1) +CFLAGS += -WI +CXXFLAGS += -WI +endif + +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common $(LIBCPPFLAGS) $(CPPFLAGSICUI18N) +DEFS += -DU_I18N_IMPLEMENTATION +LDFLAGS += $(LDFLAGSICUI18N) +LIBS = $(LIBICUUC) $(DEFAULT_LIBS) + +SOURCES = $(shell cat $(srcdir)/sources.txt) +OBJECTS = $(SOURCES:.cpp=.o) + +## Header files to install +HEADERS = $(srcdir)/unicode/*.h + +STATIC_OBJECTS = $(OBJECTS:.o=.$(STATIC_O)) + +DEPS = $(OBJECTS:.o=.d) + +-include Makefile.local + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local install-library install-headers dist \ +dist-local check check-local check-exhaustive + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +check-exhaustive: check + +all-local: $(ALL_TARGETS) + +install-local: install-headers install-library + +install-library: all-local + $(MKINSTALLDIRS) $(DESTDIR)$(libdir) +ifneq ($(ENABLE_STATIC),) + $(INSTALL-L) $(TARGET) $(DESTDIR)$(libdir) +endif +ifneq ($(ENABLE_SHARED),) +# For MinGW, do we want the DLL to go in the bin location? +ifeq ($(MINGW_MOVEDLLSTOBINDIR),YES) + $(MKINSTALLDIRS) $(DESTDIR)$(bindir) + $(INSTALL-L) $(FINAL_SO_TARGET) $(DESTDIR)$(bindir) +else + $(INSTALL-L) $(FINAL_SO_TARGET) $(DESTDIR)$(libdir) +ifneq ($(FINAL_SO_TARGET),$(SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(SO_TARGET)) +ifneq ($(FINAL_SO_TARGET),$(MIDDLE_SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(MIDDLE_SO_TARGET)) +endif +endif +endif +ifneq ($(IMPORT_LIB_EXT),) + $(INSTALL-L) $(FINAL_IMPORT_LIB) $(DESTDIR)$(libdir) +ifneq ($(IMPORT_LIB),$(FINAL_IMPORT_LIB)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(IMPORT_LIB)) +endif +ifneq ($(MIDDLE_IMPORT_LIB),$(FINAL_IMPORT_LIB)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_IMPORT_LIB)) && ln -s $(notdir $(FINAL_IMPORT_LIB)) $(notdir $(MIDDLE_IMPORT_LIB)) +endif +endif +endif + +install-headers: + $(MKINSTALLDIRS) $(DESTDIR)$(includedir)/unicode + @for file in $(HEADERS); do \ + echo "$(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode"; \ + $(INSTALL_DATA) $$file $(DESTDIR)$(includedir)/unicode || exit; \ + done + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(OBJECTS) $(STATIC_OBJECTS) $(ALL_TARGETS) $(SO_VERSION_DATA) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +ifneq ($(ENABLE_STATIC),) +$(TARGET): $(STATIC_OBJECTS) + $(AR) $(ARFLAGS) $(AR_OUTOPT)$@ $^ + $(RANLIB) $@ +endif + +ifneq ($(ENABLE_SHARED),) +$(SHARED_OBJECT): $(OBJECTS) $(SO_VERSION_DATA) + $(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(LIBS) +ifeq ($(ENABLE_RPATH),YES) +ifneq ($(wildcard $(libdir)/$(MIDDLE_SO_TARGET)),) + $(warning RPATH warning: --enable-rpath means test programs may use existing $(libdir)/$(MIDDLE_SO_TARGET)) +endif +endif + +ifeq ($(OS390BATCH),1) +$(BATCH_TARGET):$(OBJECTS) + $(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(BATCH_LIBS) +endif # OS390BATCH +endif # ENABLE_SHARED + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif diff --git a/intl/icu/source/i18n/alphaindex.cpp b/intl/icu/source/i18n/alphaindex.cpp new file mode 100644 index 0000000000..1b49d3b544 --- /dev/null +++ b/intl/icu/source/i18n/alphaindex.cpp @@ -0,0 +1,1235 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2009-2014, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/alphaindex.h" +#include "unicode/coll.h" +#include "unicode/localpointer.h" +#include "unicode/normalizer2.h" +#include "unicode/tblcoll.h" +#include "unicode/uchar.h" +#include "unicode/ulocdata.h" +#include "unicode/uniset.h" +#include "unicode/uobject.h" +#include "unicode/usetiter.h" +#include "unicode/utf16.h" + +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" +#include "uvector.h" +#include "uvectr64.h" + +//#include +//#include + +U_NAMESPACE_BEGIN + +namespace { + +/** + * Prefix string for Chinese index buckets. + * See http://unicode.org/repos/cldr/trunk/specs/ldml/tr35-collation.html#Collation_Indexes + */ +const char16_t BASE[1] = { 0xFDD0 }; +const int32_t BASE_LENGTH = 1; + +UBool isOneLabelBetterThanOther(const Normalizer2 &nfkdNormalizer, + const UnicodeString &one, const UnicodeString &other); + +} // namespace + +static int32_t U_CALLCONV +collatorComparator(const void *context, const void *left, const void *right); + +static int32_t U_CALLCONV +recordCompareFn(const void *context, const void *left, const void *right); + +// UVector support function, delete a Record. +static void U_CALLCONV +alphaIndex_deleteRecord(void *obj) { + delete static_cast(obj); +} + +namespace { + +UnicodeString *ownedString(const UnicodeString &s, LocalPointer &owned, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return nullptr; } + if (owned.isValid()) { + return owned.orphan(); + } + UnicodeString *p = new UnicodeString(s); + if (p == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } + return p; +} + +inline UnicodeString *getString(const UVector &list, int32_t i) { + return static_cast(list[i]); +} + +inline AlphabeticIndex::Bucket *getBucket(const UVector &list, int32_t i) { + return static_cast(list[i]); +} + +inline AlphabeticIndex::Record *getRecord(const UVector &list, int32_t i) { + return static_cast(list[i]); +} + +/** + * Like Java Collections.binarySearch(List, String, Comparator). + * + * @return the index>=0 where the item was found, + * or the index<0 for inserting the string at ~index in sorted order + */ +int32_t binarySearch(const UVector &list, const UnicodeString &s, const Collator &coll) { + if (list.size() == 0) { return ~0; } + int32_t start = 0; + int32_t limit = list.size(); + for (;;) { + int32_t i = (start + limit) / 2; + const UnicodeString *si = static_cast(list.elementAt(i)); + UErrorCode errorCode = U_ZERO_ERROR; + UCollationResult cmp = coll.compare(s, *si, errorCode); + if (cmp == UCOL_EQUAL) { + return i; + } else if (cmp < 0) { + if (i == start) { + return ~start; // insert s before *si + } + limit = i; + } else { + if (i == start) { + return ~(start + 1); // insert s after *si + } + start = i; + } + } +} + +} // namespace + +// The BucketList is not in the anonymous namespace because only Clang +// seems to support its use in other classes from there. +// However, we also don't need U_I18N_API because it is not used from outside the i18n library. +class BucketList : public UObject { +public: + BucketList(UVector *bucketList, UVector *publicBucketList) + : bucketList_(bucketList), immutableVisibleList_(publicBucketList) { + int32_t displayIndex = 0; + for (int32_t i = 0; i < publicBucketList->size(); ++i) { + getBucket(*publicBucketList, i)->displayIndex_ = displayIndex++; + } + } + + // The virtual destructor must not be inline. + // See ticket #8454 for details. + virtual ~BucketList(); + + int32_t getBucketCount() const { + return immutableVisibleList_->size(); + } + + int32_t getBucketIndex(const UnicodeString &name, const Collator &collatorPrimaryOnly, + UErrorCode &errorCode) { + // binary search + int32_t start = 0; + int32_t limit = bucketList_->size(); + while ((start + 1) < limit) { + int32_t i = (start + limit) / 2; + const AlphabeticIndex::Bucket *bucket = getBucket(*bucketList_, i); + UCollationResult nameVsBucket = + collatorPrimaryOnly.compare(name, bucket->lowerBoundary_, errorCode); + if (nameVsBucket < 0) { + limit = i; + } else { + start = i; + } + } + const AlphabeticIndex::Bucket *bucket = getBucket(*bucketList_, start); + if (bucket->displayBucket_ != nullptr) { + bucket = bucket->displayBucket_; + } + return bucket->displayIndex_; + } + + /** All of the buckets, visible and invisible. */ + UVector *bucketList_; + /** Just the visible buckets. */ + UVector *immutableVisibleList_; +}; + +BucketList::~BucketList() { + delete bucketList_; + if (immutableVisibleList_ != bucketList_) { + delete immutableVisibleList_; + } +} + +AlphabeticIndex::ImmutableIndex::~ImmutableIndex() { + delete buckets_; + delete collatorPrimaryOnly_; +} + +int32_t +AlphabeticIndex::ImmutableIndex::getBucketCount() const { + return buckets_->getBucketCount(); +} + +int32_t +AlphabeticIndex::ImmutableIndex::getBucketIndex( + const UnicodeString &name, UErrorCode &errorCode) const { + return buckets_->getBucketIndex(name, *collatorPrimaryOnly_, errorCode); +} + +const AlphabeticIndex::Bucket * +AlphabeticIndex::ImmutableIndex::getBucket(int32_t index) const { + if (0 <= index && index < buckets_->getBucketCount()) { + return icu::getBucket(*buckets_->immutableVisibleList_, index); + } else { + return nullptr; + } +} + +AlphabeticIndex::AlphabeticIndex(const Locale &locale, UErrorCode &status) + : inputList_(nullptr), + labelsIterIndex_(-1), itemsIterIndex_(0), currentBucket_(nullptr), + maxLabelCount_(99), + initialLabels_(nullptr), firstCharsInScripts_(nullptr), + collator_(nullptr), collatorPrimaryOnly_(nullptr), + buckets_(nullptr) { + init(&locale, status); +} + + +AlphabeticIndex::AlphabeticIndex(RuleBasedCollator *collator, UErrorCode &status) + : inputList_(nullptr), + labelsIterIndex_(-1), itemsIterIndex_(0), currentBucket_(nullptr), + maxLabelCount_(99), + initialLabels_(nullptr), firstCharsInScripts_(nullptr), + collator_(collator), collatorPrimaryOnly_(nullptr), + buckets_(nullptr) { + init(nullptr, status); +} + + + +AlphabeticIndex::~AlphabeticIndex() { + delete collator_; + delete collatorPrimaryOnly_; + delete firstCharsInScripts_; + delete buckets_; + delete inputList_; + delete initialLabels_; +} + + +AlphabeticIndex &AlphabeticIndex::addLabels(const UnicodeSet &additions, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + initialLabels_->addAll(additions); + clearBuckets(); + return *this; +} + + +AlphabeticIndex &AlphabeticIndex::addLabels(const Locale &locale, UErrorCode &status) { + addIndexExemplars(locale, status); + clearBuckets(); + return *this; +} + + +AlphabeticIndex::ImmutableIndex *AlphabeticIndex::buildImmutableIndex(UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return nullptr; } + // In C++, the ImmutableIndex must own its copy of the BucketList, + // even if it contains no records, for proper memory management. + // We could clone the buckets_ if they are not nullptr, + // but that would be worth it only if this method is called multiple times, + // or called after using the old-style bucket iterator API. + LocalPointer immutableBucketList(createBucketList(errorCode)); + LocalPointer coll(collatorPrimaryOnly_->clone()); + if (immutableBucketList.isNull() || coll.isNull()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + ImmutableIndex *immIndex = new ImmutableIndex(immutableBucketList.getAlias(), coll.getAlias()); + if (immIndex == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + // The ImmutableIndex adopted its parameter objects. + immutableBucketList.orphan(); + coll.orphan(); + return immIndex; +} + +int32_t AlphabeticIndex::getBucketCount(UErrorCode &status) { + initBuckets(status); + if (U_FAILURE(status)) { + return 0; + } + return buckets_->getBucketCount(); +} + + +int32_t AlphabeticIndex::getRecordCount(UErrorCode &status) { + if (U_FAILURE(status) || inputList_ == nullptr) { + return 0; + } + return inputList_->size(); +} + +void AlphabeticIndex::initLabels(UVector &indexCharacters, UErrorCode &errorCode) const { + U_ASSERT(indexCharacters.hasDeleter()); + const Normalizer2 *nfkdNormalizer = Normalizer2::getNFKDInstance(errorCode); + if (U_FAILURE(errorCode)) { return; } + + const UnicodeString &firstScriptBoundary = *getString(*firstCharsInScripts_, 0); + const UnicodeString &overflowBoundary = + *getString(*firstCharsInScripts_, firstCharsInScripts_->size() - 1); + + // We make a sorted array of elements. + // Some of the input may be redundant. + // That is, we might have c, ch, d, where "ch" sorts just like "c", "h". + // We filter out those cases. + UnicodeSetIterator iter(*initialLabels_); + while (U_SUCCESS(errorCode) && iter.next()) { + const UnicodeString *item = &iter.getString(); + LocalPointer ownedItem; + UBool checkDistinct; + int32_t itemLength = item->length(); + if (!item->hasMoreChar32Than(0, itemLength, 1)) { + checkDistinct = false; + } else if(item->charAt(itemLength - 1) == 0x2a && // '*' + item->charAt(itemLength - 2) != 0x2a) { + // Use a label if it is marked with one trailing star, + // even if the label string sorts the same when all contractions are suppressed. + ownedItem.adoptInstead(new UnicodeString(*item, 0, itemLength - 1)); + item = ownedItem.getAlias(); + if (item == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + checkDistinct = false; + } else { + checkDistinct = true; + } + if (collatorPrimaryOnly_->compare(*item, firstScriptBoundary, errorCode) < 0) { + // Ignore a primary-ignorable or non-alphabetic index character. + } else if (collatorPrimaryOnly_->compare(*item, overflowBoundary, errorCode) >= 0) { + // Ignore an index character that will land in the overflow bucket. + } else if (checkDistinct && + collatorPrimaryOnly_->compare(*item, separated(*item), errorCode) == 0) { + // Ignore a multi-code point index character that does not sort distinctly + // from the sequence of its separate characters. + } else { + int32_t insertionPoint = binarySearch(indexCharacters, *item, *collatorPrimaryOnly_); + if (insertionPoint < 0) { + indexCharacters.insertElementAt( + ownedString(*item, ownedItem, errorCode), ~insertionPoint, errorCode); + } else { + const UnicodeString &itemAlreadyIn = *getString(indexCharacters, insertionPoint); + if (isOneLabelBetterThanOther(*nfkdNormalizer, *item, itemAlreadyIn)) { + indexCharacters.setElementAt( + ownedString(*item, ownedItem, errorCode), insertionPoint); + } + } + } + } + if (U_FAILURE(errorCode)) { return; } + + // if the result is still too large, cut down to maxLabelCount_ elements, by removing every nth element + + int32_t size = indexCharacters.size() - 1; + if (size > maxLabelCount_) { + int32_t count = 0; + int32_t old = -1; + for (int32_t i = 0; i < indexCharacters.size();) { + ++count; + int32_t bump = count * maxLabelCount_ / size; + if (bump == old) { + indexCharacters.removeElementAt(i); + } else { + old = bump; + ++i; + } + } + } +} + +namespace { + +const UnicodeString &fixLabel(const UnicodeString ¤t, UnicodeString &temp) { + if (!current.startsWith(BASE, BASE_LENGTH)) { + return current; + } + char16_t rest = current.charAt(BASE_LENGTH); + if (0x2800 < rest && rest <= 0x28FF) { // stroke count + int32_t count = rest-0x2800; + temp.setTo((char16_t)(0x30 + count % 10)); + if (count >= 10) { + count /= 10; + temp.insert(0, (char16_t)(0x30 + count % 10)); + if (count >= 10) { + count /= 10; + temp.insert(0, (char16_t)(0x30 + count)); + } + } + return temp.append((char16_t)0x5283); + } + return temp.setTo(current, BASE_LENGTH); +} + +UBool hasMultiplePrimaryWeights( + const RuleBasedCollator &coll, uint32_t variableTop, + const UnicodeString &s, UVector64 &ces, UErrorCode &errorCode) { + ces.removeAllElements(); + coll.internalGetCEs(s, ces, errorCode); + if (U_FAILURE(errorCode)) { return false; } + UBool seenPrimary = false; + for (int32_t i = 0; i < ces.size(); ++i) { + int64_t ce = ces.elementAti(i); + uint32_t p = (uint32_t)(ce >> 32); + if (p > variableTop) { + // not primary ignorable + if (seenPrimary) { + return true; + } + seenPrimary = true; + } + } + return false; +} + +} // namespace + +BucketList *AlphabeticIndex::createBucketList(UErrorCode &errorCode) const { + // Initialize indexCharacters. + UVector indexCharacters(errorCode); + indexCharacters.setDeleter(uprv_deleteUObject); + initLabels(indexCharacters, errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + + // Variables for hasMultiplePrimaryWeights(). + UVector64 ces(errorCode); + uint32_t variableTop; + if (collatorPrimaryOnly_->getAttribute(UCOL_ALTERNATE_HANDLING, errorCode) == UCOL_SHIFTED) { + variableTop = collatorPrimaryOnly_->getVariableTop(errorCode); + } else { + variableTop = 0; + } + UBool hasInvisibleBuckets = false; + + // Helper arrays for Chinese Pinyin collation. + Bucket *asciiBuckets[26] = { + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr + }; + Bucket *pinyinBuckets[26] = { + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr + }; + UBool hasPinyin = false; + + LocalPointer bucketList(new UVector(errorCode), errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + bucketList->setDeleter(uprv_deleteUObject); + + // underflow bucket + LocalPointer bucket(new Bucket(getUnderflowLabel(), emptyString_, U_ALPHAINDEX_UNDERFLOW), errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + bucketList->adoptElement(bucket.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + + UnicodeString temp; + + // fix up the list, adding underflow, additions, overflow + // Insert inflow labels as needed. + int32_t scriptIndex = -1; + const UnicodeString *scriptUpperBoundary = &emptyString_; + for (int32_t i = 0; i < indexCharacters.size(); ++i) { + UnicodeString ¤t = *getString(indexCharacters, i); + if (collatorPrimaryOnly_->compare(current, *scriptUpperBoundary, errorCode) >= 0) { + // We crossed the script boundary into a new script. + const UnicodeString &inflowBoundary = *scriptUpperBoundary; + UBool skippedScript = false; + for (;;) { + scriptUpperBoundary = getString(*firstCharsInScripts_, ++scriptIndex); + if (collatorPrimaryOnly_->compare(current, *scriptUpperBoundary, errorCode) < 0) { + break; + } + skippedScript = true; + } + if (skippedScript && bucketList->size() > 1) { + // We are skipping one or more scripts, + // and we are not just getting out of the underflow label. + bucket.adoptInsteadAndCheckErrorCode( + new Bucket(getInflowLabel(), inflowBoundary, U_ALPHAINDEX_INFLOW), errorCode); + bucketList->adoptElement(bucket.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + } + } + // Add a bucket with the current label. + bucket.adoptInsteadAndCheckErrorCode( + new Bucket(fixLabel(current, temp), current, U_ALPHAINDEX_NORMAL), errorCode); + bucketList->adoptElement(bucket.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + // Remember ASCII and Pinyin buckets for Pinyin redirects. + char16_t c; + if (current.length() == 1 && 0x41 <= (c = current.charAt(0)) && c <= 0x5A) { // A-Z + asciiBuckets[c - 0x41] = (Bucket *)bucketList->lastElement(); + } else if (current.length() == BASE_LENGTH + 1 && current.startsWith(BASE, BASE_LENGTH) && + 0x41 <= (c = current.charAt(BASE_LENGTH)) && c <= 0x5A) { + pinyinBuckets[c - 0x41] = (Bucket *)bucketList->lastElement(); + hasPinyin = true; + } + // Check for multiple primary weights. + if (!current.startsWith(BASE, BASE_LENGTH) && + hasMultiplePrimaryWeights(*collatorPrimaryOnly_, variableTop, current, + ces, errorCode) && + current.charAt(current.length() - 1) != 0xFFFF /* !current.endsWith("\uffff") */) { + // "AE-ligature" or "Sch" etc. + for (int32_t j = bucketList->size() - 2;; --j) { + Bucket *singleBucket = getBucket(*bucketList, j); + if (singleBucket->labelType_ != U_ALPHAINDEX_NORMAL) { + // There is no single-character bucket since the last + // underflow or inflow label. + break; + } + if (singleBucket->displayBucket_ == nullptr && + !hasMultiplePrimaryWeights(*collatorPrimaryOnly_, variableTop, + singleBucket->lowerBoundary_, + ces, errorCode)) { + // Add an invisible bucket that redirects strings greater than the expansion + // to the previous single-character bucket. + // For example, after ... Q R S Sch we add Sch\uFFFF->S + // and after ... Q R S Sch Sch\uFFFF St we add St\uFFFF->S. + bucket.adoptInsteadAndCheckErrorCode(new Bucket(emptyString_, + UnicodeString(current).append((char16_t)0xFFFF), + U_ALPHAINDEX_NORMAL), + errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + bucket->displayBucket_ = singleBucket; + bucketList->adoptElement(bucket.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + hasInvisibleBuckets = true; + break; + } + } + } + } + if (U_FAILURE(errorCode)) { return nullptr; } + if (bucketList->size() == 1) { + // No real labels, show only the underflow label. + BucketList *bl = new BucketList(bucketList.getAlias(), bucketList.getAlias()); + if (bl == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + bucketList.orphan(); + return bl; + } + // overflow bucket + bucket.adoptInsteadAndCheckErrorCode( + new Bucket(getOverflowLabel(), *scriptUpperBoundary, U_ALPHAINDEX_OVERFLOW), errorCode); + bucketList->adoptElement(bucket.orphan(), errorCode); // final + if (U_FAILURE(errorCode)) { return nullptr; } + + if (hasPinyin) { + // Redirect Pinyin buckets. + Bucket *asciiBucket = nullptr; + for (int32_t i = 0; i < 26; ++i) { + if (asciiBuckets[i] != nullptr) { + asciiBucket = asciiBuckets[i]; + } + if (pinyinBuckets[i] != nullptr && asciiBucket != nullptr) { + pinyinBuckets[i]->displayBucket_ = asciiBucket; + hasInvisibleBuckets = true; + } + } + } + + if (U_FAILURE(errorCode)) { return nullptr; } + if (!hasInvisibleBuckets) { + BucketList *bl = new BucketList(bucketList.getAlias(), bucketList.getAlias()); + if (bl == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + bucketList.orphan(); + return bl; + } + // Merge inflow buckets that are visually adjacent. + // Iterate backwards: Merge inflow into overflow rather than the other way around. + int32_t i = bucketList->size() - 1; + Bucket *nextBucket = getBucket(*bucketList, i); + while (--i > 0) { + Bucket *bucket = getBucket(*bucketList, i); + if (bucket->displayBucket_ != nullptr) { + continue; // skip invisible buckets + } + if (bucket->labelType_ == U_ALPHAINDEX_INFLOW) { + if (nextBucket->labelType_ != U_ALPHAINDEX_NORMAL) { + bucket->displayBucket_ = nextBucket; + continue; + } + } + nextBucket = bucket; + } + + LocalPointer publicBucketList(new UVector(errorCode), errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + // Do not call publicBucketList->setDeleter(): + // This vector shares its objects with the bucketList. + for (int32_t j = 0; j < bucketList->size(); ++j) { + Bucket *bucket = getBucket(*bucketList, j); + if (bucket->displayBucket_ == nullptr) { + publicBucketList->addElement(bucket, errorCode); + } + } + if (U_FAILURE(errorCode)) { return nullptr; } + BucketList *bl = new BucketList(bucketList.getAlias(), publicBucketList.getAlias()); + if (bl == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + bucketList.orphan(); + publicBucketList.orphan(); + return bl; +} + +/** + * Creates an index, and buckets and sorts the list of records into the index. + */ +void AlphabeticIndex::initBuckets(UErrorCode &errorCode) { + if (U_FAILURE(errorCode) || buckets_ != nullptr) { + return; + } + buckets_ = createBucketList(errorCode); + if (U_FAILURE(errorCode) || inputList_ == nullptr || inputList_->isEmpty()) { + return; + } + + // Sort the records by name. + // Stable sort preserves input order of collation duplicates. + inputList_->sortWithUComparator(recordCompareFn, collator_, errorCode); + + // Now, we traverse all of the input, which is now sorted. + // If the item doesn't go in the current bucket, we find the next bucket that contains it. + // This makes the process order n*log(n), since we just sort the list and then do a linear process. + // However, if the user adds an item at a time and then gets the buckets, this isn't efficient, so + // we need to improve it for that case. + + Bucket *currentBucket = getBucket(*buckets_->bucketList_, 0); + int32_t bucketIndex = 1; + Bucket *nextBucket; + const UnicodeString *upperBoundary; + if (bucketIndex < buckets_->bucketList_->size()) { + nextBucket = getBucket(*buckets_->bucketList_, bucketIndex++); + upperBoundary = &nextBucket->lowerBoundary_; + } else { + nextBucket = nullptr; + upperBoundary = nullptr; + } + for (int32_t i = 0; i < inputList_->size(); ++i) { + Record *r = getRecord(*inputList_, i); + // if the current bucket isn't the right one, find the one that is + // We have a special flag for the last bucket so that we don't look any further + while (upperBoundary != nullptr && + collatorPrimaryOnly_->compare(r->name_, *upperBoundary, errorCode) >= 0) { + currentBucket = nextBucket; + // now reset the boundary that we compare against + if (bucketIndex < buckets_->bucketList_->size()) { + nextBucket = getBucket(*buckets_->bucketList_, bucketIndex++); + upperBoundary = &nextBucket->lowerBoundary_; + } else { + upperBoundary = nullptr; + } + } + // now put the record into the bucket. + Bucket *bucket = currentBucket; + if (bucket->displayBucket_ != nullptr) { + bucket = bucket->displayBucket_; + } + if (bucket->records_ == nullptr) { + LocalPointer records(new UVector(errorCode), errorCode); + if (U_FAILURE(errorCode)) { + return; + } + bucket->records_ = records.orphan(); + } + bucket->records_->addElement(r, errorCode); + } +} + +void AlphabeticIndex::clearBuckets() { + if (buckets_ != nullptr) { + delete buckets_; + buckets_ = nullptr; + internalResetBucketIterator(); + } +} + +void AlphabeticIndex::internalResetBucketIterator() { + labelsIterIndex_ = -1; + currentBucket_ = nullptr; +} + + +void AlphabeticIndex::addIndexExemplars(const Locale &locale, UErrorCode &status) { + LocalULocaleDataPointer uld(ulocdata_open(locale.getName(), &status)); + if (U_FAILURE(status)) { + return; + } + + UnicodeSet exemplars; + ulocdata_getExemplarSet(uld.getAlias(), exemplars.toUSet(), 0, ULOCDATA_ES_INDEX, &status); + if (U_SUCCESS(status)) { + initialLabels_->addAll(exemplars); + return; + } + status = U_ZERO_ERROR; // Clear out U_MISSING_RESOURCE_ERROR + + // The locale data did not include explicit Index characters. + // Synthesize a set of them from the locale's standard exemplar characters. + ulocdata_getExemplarSet(uld.getAlias(), exemplars.toUSet(), 0, ULOCDATA_ES_STANDARD, &status); + if (U_FAILURE(status)) { + return; + } + + // question: should we add auxiliary exemplars? + if (exemplars.containsSome(0x61, 0x7A) /* a-z */ || exemplars.isEmpty()) { + exemplars.add(0x61, 0x7A); + } + if (exemplars.containsSome(0xAC00, 0xD7A3)) { // Hangul syllables + // cut down to small list + exemplars.remove(0xAC00, 0xD7A3). + add(0xAC00).add(0xB098).add(0xB2E4).add(0xB77C). + add(0xB9C8).add(0xBC14).add(0xC0AC).add(0xC544). + add(0xC790).add(0xCC28).add(0xCE74).add(0xD0C0). + add(0xD30C).add(0xD558); + } + if (exemplars.containsSome(0x1200, 0x137F)) { // Ethiopic block + // cut down to small list + // make use of the fact that Ethiopic is allocated in 8's, where + // the base is 0 mod 8. + UnicodeSet ethiopic(UnicodeString(u"[ሀለሐመሠረሰሸቀቈቐቘበቨተቸኀኈነኘአከኰኸዀወዐዘዠየደዸጀገጐጘጠጨጰጸፀፈፐፘ]"), status); + ethiopic.retainAll(exemplars); + exemplars.remove(u'ሀ', 0x137F).addAll(ethiopic); + } + + // Upper-case any that aren't already so. + // (We only do this for synthesized index characters.) + UnicodeSetIterator it(exemplars); + UnicodeString upperC; + while (it.next()) { + const UnicodeString &exemplarC = it.getString(); + upperC = exemplarC; + upperC.toUpper(locale); + initialLabels_->add(upperC); + } +} + +UBool AlphabeticIndex::addChineseIndexCharacters(UErrorCode &errorCode) { + UnicodeSet contractions; + collatorPrimaryOnly_->internalAddContractions(BASE[0], contractions, errorCode); + if (U_FAILURE(errorCode) || contractions.isEmpty()) { return false; } + initialLabels_->addAll(contractions); + UnicodeSetIterator iter(contractions); + while (iter.next()) { + const UnicodeString &s = iter.getString(); + U_ASSERT (s.startsWith(BASE, BASE_LENGTH)); + char16_t c = s.charAt(s.length() - 1); + if (0x41 <= c && c <= 0x5A) { // A-Z + // There are Pinyin labels, add ASCII A-Z labels as well. + initialLabels_->add(0x41, 0x5A); // A-Z + break; + } + } + return true; +} + + +/* + * Return the string with interspersed CGJs. Input must have more than 2 codepoints. + */ +static const char16_t CGJ = 0x034F; +UnicodeString AlphabeticIndex::separated(const UnicodeString &item) { + UnicodeString result; + if (item.length() == 0) { + return result; + } + int32_t i = 0; + for (;;) { + UChar32 cp = item.char32At(i); + result.append(cp); + i = item.moveIndex32(i, 1); + if (i >= item.length()) { + break; + } + result.append(CGJ); + } + return result; +} + + +bool AlphabeticIndex::operator==(const AlphabeticIndex& /* other */) const { + return false; +} + + +bool AlphabeticIndex::operator!=(const AlphabeticIndex& /* other */) const { + return false; +} + + +const RuleBasedCollator &AlphabeticIndex::getCollator() const { + return *collator_; +} + + +const UnicodeString &AlphabeticIndex::getInflowLabel() const { + return inflowLabel_; +} + +const UnicodeString &AlphabeticIndex::getOverflowLabel() const { + return overflowLabel_; +} + + +const UnicodeString &AlphabeticIndex::getUnderflowLabel() const { + return underflowLabel_; +} + + +AlphabeticIndex &AlphabeticIndex::setInflowLabel(const UnicodeString &label, UErrorCode &/*status*/) { + inflowLabel_ = label; + clearBuckets(); + return *this; +} + + +AlphabeticIndex &AlphabeticIndex::setOverflowLabel(const UnicodeString &label, UErrorCode &/*status*/) { + overflowLabel_ = label; + clearBuckets(); + return *this; +} + + +AlphabeticIndex &AlphabeticIndex::setUnderflowLabel(const UnicodeString &label, UErrorCode &/*status*/) { + underflowLabel_ = label; + clearBuckets(); + return *this; +} + + +int32_t AlphabeticIndex::getMaxLabelCount() const { + return maxLabelCount_; +} + + +AlphabeticIndex &AlphabeticIndex::setMaxLabelCount(int32_t maxLabelCount, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (maxLabelCount <= 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + maxLabelCount_ = maxLabelCount; + clearBuckets(); + return *this; +} + + +// +// init() - Common code for constructors. +// + +void AlphabeticIndex::init(const Locale *locale, UErrorCode &status) { + if (U_FAILURE(status)) { return; } + if (locale == nullptr && collator_ == nullptr) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + initialLabels_ = new UnicodeSet(); + if (initialLabels_ == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + inflowLabel_.setTo((char16_t)0x2026); // Ellipsis + overflowLabel_ = inflowLabel_; + underflowLabel_ = inflowLabel_; + + if (collator_ == nullptr) { + Collator *coll = Collator::createInstance(*locale, status); + if (U_FAILURE(status)) { + delete coll; + return; + } + if (coll == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + collator_ = dynamic_cast(coll); + if (collator_ == nullptr) { + delete coll; + status = U_UNSUPPORTED_ERROR; + return; + } + } + collatorPrimaryOnly_ = collator_->clone(); + if (collatorPrimaryOnly_ == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + collatorPrimaryOnly_->setAttribute(UCOL_STRENGTH, UCOL_PRIMARY, status); + firstCharsInScripts_ = firstStringsInScript(status); + if (U_FAILURE(status)) { return; } + firstCharsInScripts_->sortWithUComparator(collatorComparator, collatorPrimaryOnly_, status); + // Guard against a degenerate collator where + // some script boundary strings are primary ignorable. + for (;;) { + if (U_FAILURE(status)) { return; } + if (firstCharsInScripts_->isEmpty()) { + // AlphabeticIndex requires some non-ignorable script boundary strings. + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (collatorPrimaryOnly_->compare( + *static_cast(firstCharsInScripts_->elementAt(0)), + emptyString_, status) == UCOL_EQUAL) { + firstCharsInScripts_->removeElementAt(0); + } else { + break; + } + } + + // Chinese index characters, which are specific to each of the several Chinese tailorings, + // take precedence over the single locale data exemplar set per language. + if (!addChineseIndexCharacters(status) && locale != nullptr) { + addIndexExemplars(*locale, status); + } +} + + +// +// Comparison function for UVector sorting with a collator. +// +static int32_t U_CALLCONV +collatorComparator(const void *context, const void *left, const void *right) { + const UElement *leftElement = static_cast(left); + const UElement *rightElement = static_cast(right); + const UnicodeString *leftString = static_cast(leftElement->pointer); + const UnicodeString *rightString = static_cast(rightElement->pointer); + + if (leftString == rightString) { + // Catches case where both are nullptr + return 0; + } + if (leftString == nullptr) { + return 1; + } + if (rightString == nullptr) { + return -1; + } + const Collator *col = static_cast(context); + UErrorCode errorCode = U_ZERO_ERROR; + return col->compare(*leftString, *rightString, errorCode); +} + +// +// Comparison function for UVector sorting with a collator. +// +static int32_t U_CALLCONV +recordCompareFn(const void *context, const void *left, const void *right) { + const UElement *leftElement = static_cast(left); + const UElement *rightElement = static_cast(right); + const AlphabeticIndex::Record *leftRec = static_cast(leftElement->pointer); + const AlphabeticIndex::Record *rightRec = static_cast(rightElement->pointer); + const Collator *col = static_cast(context); + UErrorCode errorCode = U_ZERO_ERROR; + return col->compare(leftRec->name_, rightRec->name_, errorCode); +} + +UVector *AlphabeticIndex::firstStringsInScript(UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer dest(new UVector(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + dest->setDeleter(uprv_deleteUObject); + // Fetch the script-first-primary contractions which are defined in the root collator. + // They all start with U+FDD1. + UnicodeSet set; + collatorPrimaryOnly_->internalAddContractions(0xFDD1, set, status); + if (U_FAILURE(status)) { + return nullptr; + } + if (set.isEmpty()) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + UnicodeSetIterator iter(set); + while (iter.next()) { + const UnicodeString &boundary = iter.getString(); + uint32_t gcMask = U_GET_GC_MASK(boundary.char32At(1)); + if ((gcMask & (U_GC_L_MASK | U_GC_CN_MASK)) == 0) { + // Ignore boundaries for the special reordering groups. + // Take only those for "real scripts" (where the sample character is a Letter, + // and the one for unassigned implicit weights (Cn). + continue; + } + LocalPointer s(new UnicodeString(boundary), status); + dest->adoptElement(s.orphan(), status); + if (U_FAILURE(status)) { + return nullptr; + } + } + return dest.orphan(); +} + + +namespace { + +/** + * Returns true if one index character string is "better" than the other. + * Shorter NFKD is better, and otherwise NFKD-binary-less-than is + * better, and otherwise binary-less-than is better. + */ +UBool isOneLabelBetterThanOther(const Normalizer2 &nfkdNormalizer, + const UnicodeString &one, const UnicodeString &other) { + // This is called with primary-equal strings, but never with one.equals(other). + UErrorCode status = U_ZERO_ERROR; + UnicodeString n1 = nfkdNormalizer.normalize(one, status); + UnicodeString n2 = nfkdNormalizer.normalize(other, status); + if (U_FAILURE(status)) { return false; } + int32_t result = n1.countChar32() - n2.countChar32(); + if (result != 0) { + return result < 0; + } + result = n1.compareCodePointOrder(n2); + if (result != 0) { + return result < 0; + } + return one.compareCodePointOrder(other) < 0; +} + +} // namespace + +// +// Constructor & Destructor for AlphabeticIndex::Record +// +// Records are internal only, instances are not directly surfaced in the public API. +// This class is mostly struct-like, with all public fields. + +AlphabeticIndex::Record::Record(const UnicodeString &name, const void *data) + : name_(name), data_(data) {} + +AlphabeticIndex::Record::~Record() { +} + + +AlphabeticIndex & AlphabeticIndex::addRecord(const UnicodeString &name, const void *data, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (inputList_ == nullptr) { + LocalPointer inputList(new UVector(status), status); + if (U_FAILURE(status)) { + return *this; + } + inputList_ = inputList.orphan(); + inputList_->setDeleter(alphaIndex_deleteRecord); + } + LocalPointer r(new Record(name, data), status); + inputList_->adoptElement(r.orphan(), status); + if (U_FAILURE(status)) { + return *this; + } + clearBuckets(); + //std::string ss; + //std::string ss2; + //std::cout << "added record: name = \"" << r->name_.toUTF8String(ss) << "\"" << + // " sortingName = \"" << r->sortingName_.toUTF8String(ss2) << "\"" << std::endl; + return *this; +} + + +AlphabeticIndex &AlphabeticIndex::clearRecords(UErrorCode &status) { + if (U_SUCCESS(status) && inputList_ != nullptr && !inputList_->isEmpty()) { + inputList_->removeAllElements(); + clearBuckets(); + } + return *this; +} + +int32_t AlphabeticIndex::getBucketIndex(const UnicodeString &name, UErrorCode &status) { + initBuckets(status); + if (U_FAILURE(status)) { + return 0; + } + return buckets_->getBucketIndex(name, *collatorPrimaryOnly_, status); +} + + +int32_t AlphabeticIndex::getBucketIndex() const { + return labelsIterIndex_; +} + + +UBool AlphabeticIndex::nextBucket(UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (buckets_ == nullptr && currentBucket_ != nullptr) { + status = U_ENUM_OUT_OF_SYNC_ERROR; + return false; + } + initBuckets(status); + if (U_FAILURE(status)) { + return false; + } + ++labelsIterIndex_; + if (labelsIterIndex_ >= buckets_->getBucketCount()) { + labelsIterIndex_ = buckets_->getBucketCount(); + return false; + } + currentBucket_ = getBucket(*buckets_->immutableVisibleList_, labelsIterIndex_); + resetRecordIterator(); + return true; +} + +const UnicodeString &AlphabeticIndex::getBucketLabel() const { + if (currentBucket_ != nullptr) { + return currentBucket_->label_; + } else { + return emptyString_; + } +} + + +UAlphabeticIndexLabelType AlphabeticIndex::getBucketLabelType() const { + if (currentBucket_ != nullptr) { + return currentBucket_->labelType_; + } else { + return U_ALPHAINDEX_NORMAL; + } +} + + +int32_t AlphabeticIndex::getBucketRecordCount() const { + if (currentBucket_ != nullptr && currentBucket_->records_ != nullptr) { + return currentBucket_->records_->size(); + } else { + return 0; + } +} + + +AlphabeticIndex &AlphabeticIndex::resetBucketIterator(UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + internalResetBucketIterator(); + return *this; +} + + +UBool AlphabeticIndex::nextRecord(UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (currentBucket_ == nullptr) { + // We are trying to iterate over the items in a bucket, but there is no + // current bucket from the enumeration of buckets. + status = U_INVALID_STATE_ERROR; + return false; + } + if (buckets_ == nullptr) { + status = U_ENUM_OUT_OF_SYNC_ERROR; + return false; + } + if (currentBucket_->records_ == nullptr) { + return false; + } + ++itemsIterIndex_; + if (itemsIterIndex_ >= currentBucket_->records_->size()) { + itemsIterIndex_ = currentBucket_->records_->size(); + return false; + } + return true; +} + + +const UnicodeString &AlphabeticIndex::getRecordName() const { + const UnicodeString *retStr = &emptyString_; + if (currentBucket_ != nullptr && currentBucket_->records_ != nullptr && + itemsIterIndex_ >= 0 && + itemsIterIndex_ < currentBucket_->records_->size()) { + Record *item = static_cast(currentBucket_->records_->elementAt(itemsIterIndex_)); + retStr = &item->name_; + } + return *retStr; +} + +const void *AlphabeticIndex::getRecordData() const { + const void *retPtr = nullptr; + if (currentBucket_ != nullptr && currentBucket_->records_ != nullptr && + itemsIterIndex_ >= 0 && + itemsIterIndex_ < currentBucket_->records_->size()) { + Record *item = static_cast(currentBucket_->records_->elementAt(itemsIterIndex_)); + retPtr = item->data_; + } + return retPtr; +} + + +AlphabeticIndex & AlphabeticIndex::resetRecordIterator() { + itemsIterIndex_ = -1; + return *this; +} + + + +AlphabeticIndex::Bucket::Bucket(const UnicodeString &label, + const UnicodeString &lowerBoundary, + UAlphabeticIndexLabelType type) + : label_(label), lowerBoundary_(lowerBoundary), labelType_(type), + displayBucket_(nullptr), displayIndex_(-1), + records_(nullptr) { +} + + +AlphabeticIndex::Bucket::~Bucket() { + delete records_; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/anytrans.cpp b/intl/icu/source/i18n/anytrans.cpp new file mode 100644 index 0000000000..4972b68733 --- /dev/null +++ b/intl/icu/source/i18n/anytrans.cpp @@ -0,0 +1,411 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +***************************************************************** +* Copyright (c) 2002-2014, International Business Machines Corporation +* and others. All Rights Reserved. +***************************************************************** +* Date Name Description +* 06/06/2002 aliu Creation. +***************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "unicode/uscript.h" + +#include "anytrans.h" +#include "hash.h" +#include "mutex.h" +#include "nultrans.h" +#include "putilimp.h" +#include "tridpars.h" +#include "uinvchar.h" +#include "uvector.h" + +//------------------------------------------------------------ +// Constants + +static const char16_t TARGET_SEP = 45; // '-' +static const char16_t VARIANT_SEP = 47; // '/' +static const char16_t ANY[] = {0x41,0x6E,0x79,0}; // "Any" +static const char16_t NULL_ID[] = {78,117,108,108,0}; // "Null" +static const char16_t LATIN_PIVOT[] = {0x2D,0x4C,0x61,0x74,0x6E,0x3B,0x4C,0x61,0x74,0x6E,0x2D,0}; // "-Latn;Latn-" + +// initial size for an Any-XXXX transform's cache of script-XXXX transforms +// (will grow as necessary, but we don't expect to have source text with more than 7 scripts) +#define ANY_TRANS_CACHE_INIT_SIZE 7 + +//------------------------------------------------------------ + +U_CDECL_BEGIN +/** + * Deleter function for Transliterator*. + */ +static void U_CALLCONV +_deleteTransliterator(void *obj) { + delete (icu::Transliterator*) obj; +} +U_CDECL_END + +//------------------------------------------------------------ + +U_NAMESPACE_BEGIN + +//------------------------------------------------------------ +// ScriptRunIterator + +/** + * Returns a series of ranges corresponding to scripts. They will be + * of the form: + * + * ccccSScSSccccTTcTcccc - c = common, S = first script, T = second + * | | - first run (start, limit) + * | | - second run (start, limit) + * + * That is, the runs will overlap. The reason for this is so that a + * transliterator can consider common characters both before and after + * the scripts. + */ +class ScriptRunIterator : public UMemory { +private: + const Replaceable& text; + int32_t textStart; + int32_t textLimit; + +public: + /** + * The code of the current run, valid after next() returns. May + * be USCRIPT_INVALID_CODE if and only if the entire text is + * COMMON/INHERITED. + */ + UScriptCode scriptCode; + + /** + * The start of the run, inclusive, valid after next() returns. + */ + int32_t start; + + /** + * The end of the run, exclusive, valid after next() returns. + */ + int32_t limit; + + /** + * Constructs a run iterator over the given text from start + * (inclusive) to limit (exclusive). + */ + ScriptRunIterator(const Replaceable& text, int32_t start, int32_t limit); + + /** + * Returns true if there are any more runs. true is always + * returned at least once. Upon return, the caller should + * examine scriptCode, start, and limit. + */ + UBool next(); + + /** + * Adjusts internal indices for a change in the limit index of the + * given delta. A positive delta means the limit has increased. + */ + void adjustLimit(int32_t delta); + +private: + ScriptRunIterator(const ScriptRunIterator &other); // forbid copying of this class + ScriptRunIterator &operator=(const ScriptRunIterator &other); // forbid copying of this class +}; + +ScriptRunIterator::ScriptRunIterator(const Replaceable& theText, + int32_t myStart, int32_t myLimit) : + text(theText) +{ + textStart = myStart; + textLimit = myLimit; + limit = myStart; +} + +UBool ScriptRunIterator::next() { + UChar32 ch; + UScriptCode s; + UErrorCode ec = U_ZERO_ERROR; + + scriptCode = USCRIPT_INVALID_CODE; // don't know script yet + start = limit; + + // Are we done? + if (start == textLimit) { + return false; + } + + // Move start back to include adjacent COMMON or INHERITED + // characters + while (start > textStart) { + ch = text.char32At(start - 1); // look back + s = uscript_getScript(ch, &ec); + if (s == USCRIPT_COMMON || s == USCRIPT_INHERITED) { + --start; + } else { + break; + } + } + + // Move limit ahead to include COMMON, INHERITED, and characters + // of the current script. + while (limit < textLimit) { + ch = text.char32At(limit); // look ahead + s = uscript_getScript(ch, &ec); + if (s != USCRIPT_COMMON && s != USCRIPT_INHERITED) { + if (scriptCode == USCRIPT_INVALID_CODE) { + scriptCode = s; + } else if (s != scriptCode) { + break; + } + } + ++limit; + } + + // Return true even if the entire text is COMMON / INHERITED, in + // which case scriptCode will be USCRIPT_INVALID_CODE. + return true; +} + +void ScriptRunIterator::adjustLimit(int32_t delta) { + limit += delta; + textLimit += delta; +} + +//------------------------------------------------------------ +// AnyTransliterator + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(AnyTransliterator) + +AnyTransliterator::AnyTransliterator(const UnicodeString& id, + const UnicodeString& theTarget, + const UnicodeString& theVariant, + UScriptCode theTargetScript, + UErrorCode& ec) : + Transliterator(id, nullptr), + targetScript(theTargetScript) +{ + cache = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, ANY_TRANS_CACHE_INIT_SIZE, &ec); + if (U_FAILURE(ec)) { + return; + } + uhash_setValueDeleter(cache, _deleteTransliterator); + + target = theTarget; + if (theVariant.length() > 0) { + target.append(VARIANT_SEP).append(theVariant); + } +} + +AnyTransliterator::~AnyTransliterator() { + uhash_close(cache); +} + +/** + * Copy constructor. + */ +AnyTransliterator::AnyTransliterator(const AnyTransliterator& o) : + Transliterator(o), + target(o.target), + targetScript(o.targetScript) +{ + // Don't copy the cache contents + UErrorCode ec = U_ZERO_ERROR; + cache = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, ANY_TRANS_CACHE_INIT_SIZE, &ec); + if (U_FAILURE(ec)) { + return; + } + uhash_setValueDeleter(cache, _deleteTransliterator); +} + +/** + * Transliterator API. + */ +AnyTransliterator* AnyTransliterator::clone() const { + return new AnyTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void AnyTransliterator::handleTransliterate(Replaceable& text, UTransPosition& pos, + UBool isIncremental) const { + int32_t allStart = pos.start; + int32_t allLimit = pos.limit; + + ScriptRunIterator it(text, pos.contextStart, pos.contextLimit); + + while (it.next()) { + // Ignore runs in the ante context + if (it.limit <= allStart) continue; + + // Try to instantiate transliterator from it.scriptCode to + // our target or target/variant + Transliterator* t = getTransliterator(it.scriptCode); + + if (t == nullptr) { + // We have no transliterator. Do nothing, but keep + // pos.start up to date. + pos.start = it.limit; + continue; + } + + // If the run end is before the transliteration limit, do + // a non-incremental transliteration. Otherwise do an + // incremental one. + UBool incremental = isIncremental && (it.limit >= allLimit); + + pos.start = uprv_max(allStart, it.start); + pos.limit = uprv_min(allLimit, it.limit); + int32_t limit = pos.limit; + t->filteredTransliterate(text, pos, incremental); + int32_t delta = pos.limit - limit; + allLimit += delta; + it.adjustLimit(delta); + + // We're done if we enter the post context + if (it.limit >= allLimit) break; + } + + // Restore limit. pos.start is fine where the last transliterator + // left it, or at the end of the last run. + pos.limit = allLimit; +} + +Transliterator* AnyTransliterator::getTransliterator(UScriptCode source) const { + + if (source == targetScript || source == USCRIPT_INVALID_CODE) { + return nullptr; + } + + Transliterator* t = nullptr; + { + Mutex m(nullptr); + t = (Transliterator*) uhash_iget(cache, (int32_t) source); + } + if (t == nullptr) { + UErrorCode ec = U_ZERO_ERROR; + UnicodeString sourceName(uscript_getShortName(source), -1, US_INV); + UnicodeString id(sourceName); + id.append(TARGET_SEP).append(target); + + t = Transliterator::createInstance(id, UTRANS_FORWARD, ec); + if (U_FAILURE(ec) || t == nullptr) { + delete t; + + // Try to pivot around Latin, our most common script + id = sourceName; + id.append(LATIN_PIVOT, -1).append(target); + t = Transliterator::createInstance(id, UTRANS_FORWARD, ec); + if (U_FAILURE(ec) || t == nullptr) { + delete t; + t = nullptr; + } + } + + if (t != nullptr) { + Transliterator *rt = nullptr; + { + Mutex m(nullptr); + rt = static_cast (uhash_iget(cache, (int32_t) source)); + if (rt == nullptr) { + // Common case, no race to cache this new transliterator. + uhash_iput(cache, (int32_t) source, t, &ec); + } else { + // Race case, some other thread beat us to caching this transliterator. + Transliterator *temp = rt; + rt = t; // Our newly created transliterator that lost the race & now needs deleting. + t = temp; // The transliterator from the cache that we will return. + } + } + delete rt; // will be non-null only in case of races. + } + } + return t; +} + +/** + * Return the script code for a given name, or -1 if not found. + */ +static UScriptCode scriptNameToCode(const UnicodeString& name) { + char buf[128]; + UScriptCode code; + UErrorCode ec = U_ZERO_ERROR; + int32_t nameLen = name.length(); + UBool isInvariant = uprv_isInvariantUString(name.getBuffer(), nameLen); + + if (isInvariant) { + name.extract(0, nameLen, buf, (int32_t)sizeof(buf), US_INV); + buf[127] = 0; // Make sure that we nullptr terminate the string. + } + if (!isInvariant || uscript_getCode(buf, &code, 1, &ec) != 1 || U_FAILURE(ec)) + { + code = USCRIPT_INVALID_CODE; + } + return code; +} + +/** + * Registers standard transliterators with the system. Called by + * Transliterator during initialization. Scan all current targets and + * register those that are scripts T as Any-T/V. + */ +void AnyTransliterator::registerIDs() { + + UErrorCode ec = U_ZERO_ERROR; + Hashtable seen(true, ec); + + int32_t sourceCount = Transliterator::_countAvailableSources(); + for (int32_t s=0; s= 1); + for (int32_t v=0; v +#include +#include "unicode/putil.h" +#include "uhash.h" +#include "umutex.h" +#include "ucln_in.h" +#include "putilimp.h" +#include // for toString() + +#if defined (PI) +#undef PI +#endif + +#ifdef U_DEBUG_ASTRO +# include "uresimp.h" // for debugging + +static void debug_astro_loc(const char *f, int32_t l) +{ + fprintf(stderr, "%s:%d: ", f, l); +} + +static void debug_astro_msg(const char *pat, ...) +{ + va_list ap; + va_start(ap, pat); + vfprintf(stderr, pat, ap); + fflush(stderr); +} +#include "unicode/datefmt.h" +#include "unicode/ustring.h" +static const char * debug_astro_date(UDate d) { + static char gStrBuf[1024]; + static DateFormat *df = nullptr; + if(df == nullptr) { + df = DateFormat::createDateTimeInstance(DateFormat::MEDIUM, DateFormat::MEDIUM, Locale::getUS()); + df->adoptTimeZone(TimeZone::getGMT()->clone()); + } + UnicodeString str; + df->format(d,str); + u_austrncpy(gStrBuf,str.getTerminatedBuffer(),sizeof(gStrBuf)-1); + return gStrBuf; +} + +// must use double parens, i.e.: U_DEBUG_ASTRO_MSG(("four is: %d",4)); +#define U_DEBUG_ASTRO_MSG(x) {debug_astro_loc(__FILE__,__LINE__);debug_astro_msg x;} +#else +#define U_DEBUG_ASTRO_MSG(x) +#endif + +static inline UBool isINVALID(double d) { + return(uprv_isNaN(d)); +} + +static icu::UMutex ccLock; + +U_CDECL_BEGIN +static UBool calendar_astro_cleanup() { + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +/** + * The number of standard hours in one sidereal day. + * Approximately 24.93. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define SIDEREAL_DAY (23.93446960027) + +/** + * The number of sidereal hours in one mean solar day. + * Approximately 24.07. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define SOLAR_DAY (24.065709816) + +/** + * The average number of solar days from one new moon to the next. This is the time + * it takes for the moon to return the same ecliptic longitude as the sun. + * It is longer than the sidereal month because the sun's longitude increases + * during the year due to the revolution of the earth around the sun. + * Approximately 29.53. + * + * @see #SIDEREAL_MONTH + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +const double CalendarAstronomer::SYNODIC_MONTH = 29.530588853; + +/** + * The average number of days it takes + * for the moon to return to the same ecliptic longitude relative to the + * stellar background. This is referred to as the sidereal month. + * It is shorter than the synodic month due to + * the revolution of the earth around the sun. + * Approximately 27.32. + * + * @see #SYNODIC_MONTH + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define SIDEREAL_MONTH 27.32166 + +/** + * The average number number of days between successive vernal equinoxes. + * Due to the precession of the earth's + * axis, this is not precisely the same as the sidereal year. + * Approximately 365.24 + * + * @see #SIDEREAL_YEAR + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define TROPICAL_YEAR 365.242191 + +/** + * The average number of days it takes + * for the sun to return to the same position against the fixed stellar + * background. This is the duration of one orbit of the earth about the sun + * as it would appear to an outside observer. + * Due to the precession of the earth's + * axis, this is not precisely the same as the tropical year. + * Approximately 365.25. + * + * @see #TROPICAL_YEAR + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define SIDEREAL_YEAR 365.25636 + +//------------------------------------------------------------------------- +// Time-related constants +//------------------------------------------------------------------------- + +/** + * The number of milliseconds in one second. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define SECOND_MS U_MILLIS_PER_SECOND + +/** + * The number of milliseconds in one minute. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define MINUTE_MS U_MILLIS_PER_MINUTE + +/** + * The number of milliseconds in one hour. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define HOUR_MS U_MILLIS_PER_HOUR + +/** + * The number of milliseconds in one day. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define DAY_MS U_MILLIS_PER_DAY + +/** + * The start of the julian day numbering scheme used by astronomers, which + * is 1/1/4713 BC (Julian), 12:00 GMT. This is given as the number of milliseconds + * since 1/1/1970 AD (Gregorian), a negative number. + * Note that julian day numbers and + * the Julian calendar are not the same thing. Also note that + * julian days start at noon, not midnight. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +#define JULIAN_EPOCH_MS -210866760000000.0 + + +/** + * Milliseconds value for 0.0 January 2000 AD. + */ +#define EPOCH_2000_MS 946598400000.0 + +//------------------------------------------------------------------------- +// Assorted private data used for conversions +//------------------------------------------------------------------------- + +// My own copies of these so compilers are more likely to optimize them away +const double CalendarAstronomer::PI = 3.14159265358979323846; + +#define CalendarAstronomer_PI2 (CalendarAstronomer::PI*2.0) +#define RAD_HOUR ( 12 / CalendarAstronomer::PI ) // radians -> hours +#define DEG_RAD ( CalendarAstronomer::PI / 180 ) // degrees -> radians +#define RAD_DEG ( 180 / CalendarAstronomer::PI ) // radians -> degrees + +/*** + * Given 'value', add or subtract 'range' until 0 <= 'value' < range. + * The modulus operator. + */ +inline static double normalize(double value, double range) { + return value - range * ClockMath::floorDivide(value, range); +} + +/** + * Normalize an angle so that it's in the range 0 - 2pi. + * For positive angles this is just (angle % 2pi), but the Java + * mod operator doesn't work that way for negative numbers.... + */ +inline static double norm2PI(double angle) { + return normalize(angle, CalendarAstronomer::PI * 2.0); +} + +/** + * Normalize an angle into the range -PI - PI + */ +inline static double normPI(double angle) { + return normalize(angle + CalendarAstronomer::PI, CalendarAstronomer::PI * 2.0) - CalendarAstronomer::PI; +} + +//------------------------------------------------------------------------- +// Constructors +//------------------------------------------------------------------------- + +/** + * Construct a new CalendarAstronomer object that is initialized to + * the current date and time. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::CalendarAstronomer(): + fTime(Calendar::getNow()), fLongitude(0.0), fLatitude(0.0), fGmtOffset(0.0), moonPosition(0,0), moonPositionSet(false) { + clearCache(); +} + +/** + * Construct a new CalendarAstronomer object that is initialized to + * the specified date and time. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::CalendarAstronomer(UDate d): fTime(d), fLongitude(0.0), fLatitude(0.0), fGmtOffset(0.0), moonPosition(0,0), moonPositionSet(false) { + clearCache(); +} + +/** + * Construct a new CalendarAstronomer object with the given + * latitude and longitude. The object's time is set to the current + * date and time. + *

+ * @param longitude The desired longitude, in degrees east of + * the Greenwich meridian. + * + * @param latitude The desired latitude, in degrees. Positive + * values signify North, negative South. + * + * @see java.util.Date#getTime() + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::CalendarAstronomer(double longitude, double latitude) : + fTime(Calendar::getNow()), moonPosition(0,0), moonPositionSet(false) { + fLongitude = normPI(longitude * (double)DEG_RAD); + fLatitude = normPI(latitude * (double)DEG_RAD); + fGmtOffset = (double)(fLongitude * 24. * (double)HOUR_MS / (double)CalendarAstronomer_PI2); + clearCache(); +} + +CalendarAstronomer::~CalendarAstronomer() +{ +} + +//------------------------------------------------------------------------- +// Time and date getters and setters +//------------------------------------------------------------------------- + +/** + * Set the current date and time of this CalendarAstronomer object. All + * astronomical calculations are performed based on this time setting. + * + * @param aTime the date and time, expressed as the number of milliseconds since + * 1/1/1970 0:00 GMT (Gregorian). + * + * @see #setDate + * @see #getTime + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +void CalendarAstronomer::setTime(UDate aTime) { + fTime = aTime; + U_DEBUG_ASTRO_MSG(("setTime(%.1lf, %sL)\n", aTime, debug_astro_date(aTime+fGmtOffset))); + clearCache(); +} + +/** + * Set the current date and time of this CalendarAstronomer object. All + * astronomical calculations are performed based on this time setting. + * + * @param jdn the desired time, expressed as a "julian day number", + * which is the number of elapsed days since + * 1/1/4713 BC (Julian), 12:00 GMT. Note that julian day + * numbers start at noon. To get the jdn for + * the corresponding midnight, subtract 0.5. + * + * @see #getJulianDay + * @see #JULIAN_EPOCH_MS + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +void CalendarAstronomer::setJulianDay(double jdn) { + fTime = (double)(jdn * DAY_MS) + JULIAN_EPOCH_MS; + clearCache(); + julianDay = jdn; +} + +/** + * Get the current time of this CalendarAstronomer object, + * represented as the number of milliseconds since + * 1/1/1970 AD 0:00 GMT (Gregorian). + * + * @see #setTime + * @see #getDate + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +UDate CalendarAstronomer::getTime() { + return fTime; +} + +/** + * Get the current time of this CalendarAstronomer object, + * expressed as a "julian day number", which is the number of elapsed + * days since 1/1/4713 BC (Julian), 12:00 GMT. + * + * @see #setJulianDay + * @see #JULIAN_EPOCH_MS + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getJulianDay() { + if (isINVALID(julianDay)) { + julianDay = (fTime - (double)JULIAN_EPOCH_MS) / (double)DAY_MS; + } + return julianDay; +} + +/** + * Return this object's time expressed in julian centuries: + * the number of centuries after 1/1/1900 AD, 12:00 GMT + * + * @see #getJulianDay + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getJulianCentury() { + if (isINVALID(julianCentury)) { + julianCentury = (getJulianDay() - 2415020.0) / 36525.0; + } + return julianCentury; +} + +/** + * Returns the current Greenwich sidereal time, measured in hours + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getGreenwichSidereal() { + if (isINVALID(siderealTime)) { + // See page 86 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + + double UT = normalize(fTime/(double)HOUR_MS, 24.); + + siderealTime = normalize(getSiderealOffset() + UT*1.002737909, 24.); + } + return siderealTime; +} + +double CalendarAstronomer::getSiderealOffset() { + if (isINVALID(siderealT0)) { + double JD = uprv_floor(getJulianDay() - 0.5) + 0.5; + double S = JD - 2451545.0; + double T = S / 36525.0; + siderealT0 = normalize(6.697374558 + 2400.051336*T + 0.000025862*T*T, 24); + } + return siderealT0; +} + +/** + * Returns the current local sidereal time, measured in hours + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getLocalSidereal() { + return normalize(getGreenwichSidereal() + (fGmtOffset/(double)HOUR_MS), 24.); +} + +/** + * Converts local sidereal time to Universal Time. + * + * @param lst The Local Sidereal Time, in hours since sidereal midnight + * on this object's current date. + * + * @return The corresponding Universal Time, in milliseconds since + * 1 Jan 1970, GMT. + */ +double CalendarAstronomer::lstToUT(double lst) { + // Convert to local mean time + double lt = normalize((lst - getSiderealOffset()) * 0.9972695663, 24); + + // Then find local midnight on this day + double base = (DAY_MS * ClockMath::floorDivide(fTime + fGmtOffset,(double)DAY_MS)) - fGmtOffset; + + //out(" lt =" + lt + " hours"); + //out(" base=" + new Date(base)); + + return base + (long)(lt * HOUR_MS); +} + + +//------------------------------------------------------------------------- +// Coordinate transformations, all based on the current time of this object +//------------------------------------------------------------------------- + +/** + * Convert from ecliptic to equatorial coordinates. + * + * @param ecliptic A point in the sky in ecliptic coordinates. + * @return The corresponding point in equatorial coordinates. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::Equatorial& CalendarAstronomer::eclipticToEquatorial(CalendarAstronomer::Equatorial& result, const CalendarAstronomer::Ecliptic& ecliptic) +{ + return eclipticToEquatorial(result, ecliptic.longitude, ecliptic.latitude); +} + +/** + * Convert from ecliptic to equatorial coordinates. + * + * @param eclipLong The ecliptic longitude + * @param eclipLat The ecliptic latitude + * + * @return The corresponding point in equatorial coordinates. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::Equatorial& CalendarAstronomer::eclipticToEquatorial(CalendarAstronomer::Equatorial& result, double eclipLong, double eclipLat) +{ + // See page 42 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + + double obliq = eclipticObliquity(); + double sinE = ::sin(obliq); + double cosE = cos(obliq); + + double sinL = ::sin(eclipLong); + double cosL = cos(eclipLong); + + double sinB = ::sin(eclipLat); + double cosB = cos(eclipLat); + double tanB = tan(eclipLat); + + result.set(atan2(sinL*cosE - tanB*sinE, cosL), + asin(sinB*cosE + cosB*sinE*sinL) ); + return result; +} + +/** + * Convert from ecliptic longitude to equatorial coordinates. + * + * @param eclipLong The ecliptic longitude + * + * @return The corresponding point in equatorial coordinates. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::Equatorial& CalendarAstronomer::eclipticToEquatorial(CalendarAstronomer::Equatorial& result, double eclipLong) +{ + return eclipticToEquatorial(result, eclipLong, 0); // TODO: optimize +} + +/** + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::Horizon& CalendarAstronomer::eclipticToHorizon(CalendarAstronomer::Horizon& result, double eclipLong) +{ + Equatorial equatorial; + eclipticToEquatorial(equatorial, eclipLong); + + double H = getLocalSidereal()*CalendarAstronomer::PI/12 - equatorial.ascension; // Hour-angle + + double sinH = ::sin(H); + double cosH = cos(H); + double sinD = ::sin(equatorial.declination); + double cosD = cos(equatorial.declination); + double sinL = ::sin(fLatitude); + double cosL = cos(fLatitude); + + double altitude = asin(sinD*sinL + cosD*cosL*cosH); + double azimuth = atan2(-cosD*cosL*sinH, sinD - sinL * ::sin(altitude)); + + result.set(azimuth, altitude); + return result; +} + + +//------------------------------------------------------------------------- +// The Sun +//------------------------------------------------------------------------- + +// +// Parameters of the Sun's orbit as of the epoch Jan 0.0 1990 +// Angles are in radians (after multiplying by CalendarAstronomer::PI/180) +// +#define JD_EPOCH 2447891.5 // Julian day of epoch + +#define SUN_ETA_G (279.403303 * CalendarAstronomer::PI/180) // Ecliptic longitude at epoch +#define SUN_OMEGA_G (282.768422 * CalendarAstronomer::PI/180) // Ecliptic longitude of perigee +#define SUN_E 0.016713 // Eccentricity of orbit +//double sunR0 1.495585e8 // Semi-major axis in KM +//double sunTheta0 (0.533128 * CalendarAstronomer::PI/180) // Angular diameter at R0 + +// The following three methods, which compute the sun parameters +// given above for an arbitrary epoch (whatever time the object is +// set to), make only a small difference as compared to using the +// above constants. E.g., Sunset times might differ by ~12 +// seconds. Furthermore, the eta-g computation is befuddled by +// Duffet-Smith's incorrect coefficients (p.86). I've corrected +// the first-order coefficient but the others may be off too - no +// way of knowing without consulting another source. + +// /** +// * Return the sun's ecliptic longitude at perigee for the current time. +// * See Duffett-Smith, p. 86. +// * @return radians +// */ +// private double getSunOmegaG() { +// double T = getJulianCentury(); +// return (281.2208444 + (1.719175 + 0.000452778*T)*T) * DEG_RAD; +// } + +// /** +// * Return the sun's ecliptic longitude for the current time. +// * See Duffett-Smith, p. 86. +// * @return radians +// */ +// private double getSunEtaG() { +// double T = getJulianCentury(); +// //return (279.6966778 + (36000.76892 + 0.0003025*T)*T) * DEG_RAD; +// // +// // The above line is from Duffett-Smith, and yields manifestly wrong +// // results. The below constant is derived empirically to match the +// // constant he gives for the 1990 EPOCH. +// // +// return (279.6966778 + (-0.3262541582718024 + 0.0003025*T)*T) * DEG_RAD; +// } + +// /** +// * Return the sun's eccentricity of orbit for the current time. +// * See Duffett-Smith, p. 86. +// * @return double +// */ +// private double getSunE() { +// double T = getJulianCentury(); +// return 0.01675104 - (0.0000418 + 0.000000126*T)*T; +// } + +/** + * Find the "true anomaly" (longitude) of an object from + * its mean anomaly and the eccentricity of its orbit. This uses + * an iterative solution to Kepler's equation. + * + * @param meanAnomaly The object's longitude calculated as if it were in + * a regular, circular orbit, measured in radians + * from the point of perigee. + * + * @param eccentricity The eccentricity of the orbit + * + * @return The true anomaly (longitude) measured in radians + */ +static double trueAnomaly(double meanAnomaly, double eccentricity) +{ + // First, solve Kepler's equation iteratively + // Duffett-Smith, p.90 + double delta; + double E = meanAnomaly; + do { + delta = E - eccentricity * ::sin(E) - meanAnomaly; + E = E - delta / (1 - eccentricity * ::cos(E)); + } + while (uprv_fabs(delta) > 1e-5); // epsilon = 1e-5 rad + + return 2.0 * ::atan( ::tan(E/2) * ::sqrt( (1+eccentricity) + /(1-eccentricity) ) ); +} + +/** + * The longitude of the sun at the time specified by this object. + * The longitude is measured in radians along the ecliptic + * from the "first point of Aries," the point at which the ecliptic + * crosses the earth's equatorial plane at the vernal equinox. + *

+ * Currently, this method uses an approximation of the two-body Kepler's + * equation for the earth and the sun. It does not take into account the + * perturbations caused by the other planets, the moon, etc. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getSunLongitude() +{ + // See page 86 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + + if (isINVALID(sunLongitude)) { + getSunLongitude(getJulianDay(), sunLongitude, meanAnomalySun); + } + return sunLongitude; +} + +/** + * TODO Make this public when the entire class is package-private. + */ +/*public*/ void CalendarAstronomer::getSunLongitude(double jDay, double &longitude, double &meanAnomaly) +{ + // See page 86 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + + double day = jDay - JD_EPOCH; // Days since epoch + + // Find the angular distance the sun in a fictitious + // circular orbit has travelled since the epoch. + double epochAngle = norm2PI(CalendarAstronomer_PI2/TROPICAL_YEAR*day); + + // The epoch wasn't at the sun's perigee; find the angular distance + // since perigee, which is called the "mean anomaly" + meanAnomaly = norm2PI(epochAngle + SUN_ETA_G - SUN_OMEGA_G); + + // Now find the "true anomaly", e.g. the real solar longitude + // by solving Kepler's equation for an elliptical orbit + // NOTE: The 3rd ed. of the book lists omega_g and eta_g in different + // equations; omega_g is to be correct. + longitude = norm2PI(trueAnomaly(meanAnomaly, SUN_E) + SUN_OMEGA_G); +} + +/** + * The position of the sun at this object's current date and time, + * in equatorial coordinates. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +CalendarAstronomer::Equatorial& CalendarAstronomer::getSunPosition(CalendarAstronomer::Equatorial& result) { + return eclipticToEquatorial(result, getSunLongitude(), 0); +} + + +/** + * Constant representing the vernal equinox. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "vernal" refers to the northern hemisphere's seasons. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +/*double CalendarAstronomer::VERNAL_EQUINOX() { + return 0; +}*/ + +/** + * Constant representing the summer solstice. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "summer" refers to the northern hemisphere's seasons. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::SUMMER_SOLSTICE() { + return (CalendarAstronomer::PI/2); +} + +/** + * Constant representing the autumnal equinox. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "autumn" refers to the northern hemisphere's seasons. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +/*double CalendarAstronomer::AUTUMN_EQUINOX() { + return (CalendarAstronomer::PI); +}*/ + +/** + * Constant representing the winter solstice. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "winter" refers to the northern hemisphere's seasons. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::WINTER_SOLSTICE() { + return ((CalendarAstronomer::PI*3)/2); +} + +CalendarAstronomer::AngleFunc::~AngleFunc() {} + +/** + * Find the next time at which the sun's ecliptic longitude will have + * the desired value. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +class SunTimeAngleFunc : public CalendarAstronomer::AngleFunc { +public: + virtual ~SunTimeAngleFunc(); + virtual double eval(CalendarAstronomer& a) override { return a.getSunLongitude(); } +}; + +SunTimeAngleFunc::~SunTimeAngleFunc() {} + +UDate CalendarAstronomer::getSunTime(double desired, UBool next) +{ + SunTimeAngleFunc func; + return timeOfAngle( func, + desired, + TROPICAL_YEAR, + MINUTE_MS, + next); +} + +CalendarAstronomer::CoordFunc::~CoordFunc() {} + +class RiseSetCoordFunc : public CalendarAstronomer::CoordFunc { +public: + virtual ~RiseSetCoordFunc(); + virtual void eval(CalendarAstronomer::Equatorial& result, CalendarAstronomer& a) override { a.getSunPosition(result); } +}; + +RiseSetCoordFunc::~RiseSetCoordFunc() {} + +UDate CalendarAstronomer::getSunRiseSet(UBool rise) +{ + UDate t0 = fTime; + + // Make a rough guess: 6am or 6pm local time on the current day + double noon = ClockMath::floorDivide(fTime + fGmtOffset, (double)DAY_MS)*DAY_MS - fGmtOffset + (12*HOUR_MS); + + U_DEBUG_ASTRO_MSG(("Noon=%.2lf, %sL, gmtoff %.2lf\n", noon, debug_astro_date(noon+fGmtOffset), fGmtOffset)); + setTime(noon + ((rise ? -6 : 6) * HOUR_MS)); + U_DEBUG_ASTRO_MSG(("added %.2lf ms as a guess,\n", ((rise ? -6. : 6.) * HOUR_MS))); + + RiseSetCoordFunc func; + double t = riseOrSet(func, + rise, + .533 * DEG_RAD, // Angular Diameter + 34. /60.0 * DEG_RAD, // Refraction correction + MINUTE_MS / 12.); // Desired accuracy + + setTime(t0); + return t; +} + +// Commented out - currently unused. ICU 2.6, Alan +// //------------------------------------------------------------------------- +// // Alternate Sun Rise/Set +// // See Duffett-Smith p.93 +// //------------------------------------------------------------------------- +// +// // This yields worse results (as compared to USNO data) than getSunRiseSet(). +// /** +// * TODO Make this when the entire class is package-private. +// */ +// /*public*/ long getSunRiseSet2(boolean rise) { +// // 1. Calculate coordinates of the sun's center for midnight +// double jd = uprv_floor(getJulianDay() - 0.5) + 0.5; +// double[] sl = getSunLongitude(jd);// double lambda1 = sl[0]; +// Equatorial pos1 = eclipticToEquatorial(lambda1, 0); +// +// // 2. Add ... to lambda to get position 24 hours later +// double lambda2 = lambda1 + 0.985647*DEG_RAD; +// Equatorial pos2 = eclipticToEquatorial(lambda2, 0); +// +// // 3. Calculate LSTs of rising and setting for these two positions +// double tanL = ::tan(fLatitude); +// double H = ::acos(-tanL * ::tan(pos1.declination)); +// double lst1r = (CalendarAstronomer_PI2 + pos1.ascension - H) * 24 / CalendarAstronomer_PI2; +// double lst1s = (pos1.ascension + H) * 24 / CalendarAstronomer_PI2; +// H = ::acos(-tanL * ::tan(pos2.declination)); +// double lst2r = (CalendarAstronomer_PI2-H + pos2.ascension ) * 24 / CalendarAstronomer_PI2; +// double lst2s = (H + pos2.ascension ) * 24 / CalendarAstronomer_PI2; +// if (lst1r > 24) lst1r -= 24; +// if (lst1s > 24) lst1s -= 24; +// if (lst2r > 24) lst2r -= 24; +// if (lst2s > 24) lst2s -= 24; +// +// // 4. Convert LSTs to GSTs. If GST1 > GST2, add 24 to GST2. +// double gst1r = lstToGst(lst1r); +// double gst1s = lstToGst(lst1s); +// double gst2r = lstToGst(lst2r); +// double gst2s = lstToGst(lst2s); +// if (gst1r > gst2r) gst2r += 24; +// if (gst1s > gst2s) gst2s += 24; +// +// // 5. Calculate GST at 0h UT of this date +// double t00 = utToGst(0); +// +// // 6. Calculate GST at 0h on the observer's longitude +// double offset = ::round(fLongitude*12/PI); // p.95 step 6; he _rounds_ to nearest 15 deg. +// double t00p = t00 - offset*1.002737909; +// if (t00p < 0) t00p += 24; // do NOT normalize +// +// // 7. Adjust +// if (gst1r < t00p) { +// gst1r += 24; +// gst2r += 24; +// } +// if (gst1s < t00p) { +// gst1s += 24; +// gst2s += 24; +// } +// +// // 8. +// double gstr = (24.07*gst1r-t00*(gst2r-gst1r))/(24.07+gst1r-gst2r); +// double gsts = (24.07*gst1s-t00*(gst2s-gst1s))/(24.07+gst1s-gst2s); +// +// // 9. Correct for parallax, refraction, and sun's diameter +// double dec = (pos1.declination + pos2.declination) / 2; +// double psi = ::acos(sin(fLatitude) / cos(dec)); +// double x = 0.830725 * DEG_RAD; // parallax+refraction+diameter +// double y = ::asin(sin(x) / ::sin(psi)) * RAD_DEG; +// double delta_t = 240 * y / cos(dec) / 3600; // hours +// +// // 10. Add correction to GSTs, subtract from GSTr +// gstr -= delta_t; +// gsts += delta_t; +// +// // 11. Convert GST to UT and then to local civil time +// double ut = gstToUt(rise ? gstr : gsts); +// //System.out.println((rise?"rise=":"set=") + ut + ", delta_t=" + delta_t); +// long midnight = DAY_MS * (time / DAY_MS); // Find UT midnight on this day +// return midnight + (long) (ut * 3600000); +// } + +// Commented out - currently unused. ICU 2.6, Alan +// /** +// * Convert local sidereal time to Greenwich sidereal time. +// * Section 15. Duffett-Smith p.21 +// * @param lst in hours (0..24) +// * @return GST in hours (0..24) +// */ +// double lstToGst(double lst) { +// double delta = fLongitude * 24 / CalendarAstronomer_PI2; +// return normalize(lst - delta, 24); +// } + +// Commented out - currently unused. ICU 2.6, Alan +// /** +// * Convert UT to GST on this date. +// * Section 12. Duffett-Smith p.17 +// * @param ut in hours +// * @return GST in hours +// */ +// double utToGst(double ut) { +// return normalize(getT0() + ut*1.002737909, 24); +// } + +// Commented out - currently unused. ICU 2.6, Alan +// /** +// * Convert GST to UT on this date. +// * Section 13. Duffett-Smith p.18 +// * @param gst in hours +// * @return UT in hours +// */ +// double gstToUt(double gst) { +// return normalize(gst - getT0(), 24) * 0.9972695663; +// } + +// Commented out - currently unused. ICU 2.6, Alan +// double getT0() { +// // Common computation for UT <=> GST +// +// // Find JD for 0h UT +// double jd = uprv_floor(getJulianDay() - 0.5) + 0.5; +// +// double s = jd - 2451545.0; +// double t = s / 36525.0; +// double t0 = 6.697374558 + (2400.051336 + 0.000025862*t)*t; +// return t0; +// } + +// Commented out - currently unused. ICU 2.6, Alan +// //------------------------------------------------------------------------- +// // Alternate Sun Rise/Set +// // See sci.astro FAQ +// // http://www.faqs.org/faqs/astronomy/faq/part3/section-5.html +// //------------------------------------------------------------------------- +// +// // Note: This method appears to produce inferior accuracy as +// // compared to getSunRiseSet(). +// +// /** +// * TODO Make this when the entire class is package-private. +// */ +// /*public*/ long getSunRiseSet3(boolean rise) { +// +// // Compute day number for 0.0 Jan 2000 epoch +// double d = (double)(time - EPOCH_2000_MS) / DAY_MS; +// +// // Now compute the Local Sidereal Time, LST: +// // +// double LST = 98.9818 + 0.985647352 * d + /*UT*15 + long*/ +// fLongitude*RAD_DEG; +// // +// // (east long. positive). Note that LST is here expressed in degrees, +// // where 15 degrees corresponds to one hour. Since LST really is an angle, +// // it's convenient to use one unit---degrees---throughout. +// +// // COMPUTING THE SUN'S POSITION +// // ---------------------------- +// // +// // To be able to compute the Sun's rise/set times, you need to be able to +// // compute the Sun's position at any time. First compute the "day +// // number" d as outlined above, for the desired moment. Next compute: +// // +// double oblecl = 23.4393 - 3.563E-7 * d; +// // +// double w = 282.9404 + 4.70935E-5 * d; +// double M = 356.0470 + 0.9856002585 * d; +// double e = 0.016709 - 1.151E-9 * d; +// // +// // This is the obliquity of the ecliptic, plus some of the elements of +// // the Sun's apparent orbit (i.e., really the Earth's orbit): w = +// // argument of perihelion, M = mean anomaly, e = eccentricity. +// // Semi-major axis is here assumed to be exactly 1.0 (while not strictly +// // true, this is still an accurate approximation). Next compute E, the +// // eccentric anomaly: +// // +// double E = M + e*(180/PI) * ::sin(M*DEG_RAD) * ( 1.0 + e*cos(M*DEG_RAD) ); +// // +// // where E and M are in degrees. This is it---no further iterations are +// // needed because we know e has a sufficiently small value. Next compute +// // the true anomaly, v, and the distance, r: +// // +// /* r * cos(v) = */ double A = cos(E*DEG_RAD) - e; +// /* r * ::sin(v) = */ double B = ::sqrt(1 - e*e) * ::sin(E*DEG_RAD); +// // +// // and +// // +// // r = sqrt( A*A + B*B ) +// double v = ::atan2( B, A )*RAD_DEG; +// // +// // The Sun's true longitude, slon, can now be computed: +// // +// double slon = v + w; +// // +// // Since the Sun is always at the ecliptic (or at least very very close to +// // it), we can use simplified formulae to convert slon (the Sun's ecliptic +// // longitude) to sRA and sDec (the Sun's RA and Dec): +// // +// // ::sin(slon) * cos(oblecl) +// // tan(sRA) = ------------------------- +// // cos(slon) +// // +// // ::sin(sDec) = ::sin(oblecl) * ::sin(slon) +// // +// // As was the case when computing az, the Azimuth, if possible use an +// // atan2() function to compute sRA. +// +// double sRA = ::atan2(sin(slon*DEG_RAD) * cos(oblecl*DEG_RAD), cos(slon*DEG_RAD))*RAD_DEG; +// +// double sin_sDec = ::sin(oblecl*DEG_RAD) * ::sin(slon*DEG_RAD); +// double sDec = ::asin(sin_sDec)*RAD_DEG; +// +// // COMPUTING RISE AND SET TIMES +// // ---------------------------- +// // +// // To compute when an object rises or sets, you must compute when it +// // passes the meridian and the HA of rise/set. Then the rise time is +// // the meridian time minus HA for rise/set, and the set time is the +// // meridian time plus the HA for rise/set. +// // +// // To find the meridian time, compute the Local Sidereal Time at 0h local +// // time (or 0h UT if you prefer to work in UT) as outlined above---name +// // that quantity LST0. The Meridian Time, MT, will now be: +// // +// // MT = RA - LST0 +// double MT = normalize(sRA - LST, 360); +// // +// // where "RA" is the object's Right Ascension (in degrees!). If negative, +// // add 360 deg to MT. If the object is the Sun, leave the time as it is, +// // but if it's stellar, multiply MT by 365.2422/366.2422, to convert from +// // sidereal to solar time. Now, compute HA for rise/set, name that +// // quantity HA0: +// // +// // ::sin(h0) - ::sin(lat) * ::sin(Dec) +// // cos(HA0) = --------------------------------- +// // cos(lat) * cos(Dec) +// // +// // where h0 is the altitude selected to represent rise/set. For a purely +// // mathematical horizon, set h0 = 0 and simplify to: +// // +// // cos(HA0) = - tan(lat) * tan(Dec) +// // +// // If you want to account for refraction on the atmosphere, set h0 = -35/60 +// // degrees (-35 arc minutes), and if you want to compute the rise/set times +// // for the Sun's upper limb, set h0 = -50/60 (-50 arc minutes). +// // +// double h0 = -50/60 * DEG_RAD; +// +// double HA0 = ::acos( +// (sin(h0) - ::sin(fLatitude) * sin_sDec) / +// (cos(fLatitude) * cos(sDec*DEG_RAD)))*RAD_DEG; +// +// // When HA0 has been computed, leave it as it is for the Sun but multiply +// // by 365.2422/366.2422 for stellar objects, to convert from sidereal to +// // solar time. Finally compute: +// // +// // Rise time = MT - HA0 +// // Set time = MT + HA0 +// // +// // convert the times from degrees to hours by dividing by 15. +// // +// // If you'd like to check that your calculations are accurate or just +// // need a quick result, check the USNO's Sun or Moon Rise/Set Table, +// // . +// +// double result = MT + (rise ? -HA0 : HA0); // in degrees +// +// // Find UT midnight on this day +// long midnight = DAY_MS * (time / DAY_MS); +// +// return midnight + (long) (result * 3600000 / 15); +// } + +//------------------------------------------------------------------------- +// The Moon +//------------------------------------------------------------------------- + +#define moonL0 (318.351648 * CalendarAstronomer::PI/180 ) // Mean long. at epoch +#define moonP0 ( 36.340410 * CalendarAstronomer::PI/180 ) // Mean long. of perigee +#define moonN0 ( 318.510107 * CalendarAstronomer::PI/180 ) // Mean long. of node +#define moonI ( 5.145366 * CalendarAstronomer::PI/180 ) // Inclination of orbit +#define moonE ( 0.054900 ) // Eccentricity of orbit + +// These aren't used right now +#define moonA ( 3.84401e5 ) // semi-major axis (km) +#define moonT0 ( 0.5181 * CalendarAstronomer::PI/180 ) // Angular size at distance A +#define moonPi ( 0.9507 * CalendarAstronomer::PI/180 ) // Parallax at distance A + +/** + * The position of the moon at the time set on this + * object, in equatorial coordinates. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +const CalendarAstronomer::Equatorial& CalendarAstronomer::getMoonPosition() +{ + // + // See page 142 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + // + if (moonPositionSet == false) { + // Calculate the solar longitude. Has the side effect of + // filling in "meanAnomalySun" as well. + getSunLongitude(); + + // + // Find the # of days since the epoch of our orbital parameters. + // TODO: Convert the time of day portion into ephemeris time + // + double day = getJulianDay() - JD_EPOCH; // Days since epoch + + // Calculate the mean longitude and anomaly of the moon, based on + // a circular orbit. Similar to the corresponding solar calculation. + double meanLongitude = norm2PI(13.1763966*PI/180*day + moonL0); + meanAnomalyMoon = norm2PI(meanLongitude - 0.1114041*PI/180 * day - moonP0); + + // + // Calculate the following corrections: + // Evection: the sun's gravity affects the moon's eccentricity + // Annual Eqn: variation in the effect due to earth-sun distance + // A3: correction factor (for ???) + // + double evection = 1.2739*PI/180 * ::sin(2 * (meanLongitude - sunLongitude) + - meanAnomalyMoon); + double annual = 0.1858*PI/180 * ::sin(meanAnomalySun); + double a3 = 0.3700*PI/180 * ::sin(meanAnomalySun); + + meanAnomalyMoon += evection - annual - a3; + + // + // More correction factors: + // center equation of the center correction + // a4 yet another error correction (???) + // + // TODO: Skip the equation of the center correction and solve Kepler's eqn? + // + double center = 6.2886*PI/180 * ::sin(meanAnomalyMoon); + double a4 = 0.2140*PI/180 * ::sin(2 * meanAnomalyMoon); + + // Now find the moon's corrected longitude + moonLongitude = meanLongitude + evection + center - annual + a4; + + // + // And finally, find the variation, caused by the fact that the sun's + // gravitational pull on the moon varies depending on which side of + // the earth the moon is on + // + double variation = 0.6583*CalendarAstronomer::PI/180 * ::sin(2*(moonLongitude - sunLongitude)); + + moonLongitude += variation; + + // + // What we've calculated so far is the moon's longitude in the plane + // of its own orbit. Now map to the ecliptic to get the latitude + // and longitude. First we need to find the longitude of the ascending + // node, the position on the ecliptic where it is crossed by the moon's + // orbit as it crosses from the southern to the northern hemisphere. + // + double nodeLongitude = norm2PI(moonN0 - 0.0529539*PI/180 * day); + + nodeLongitude -= 0.16*PI/180 * ::sin(meanAnomalySun); + + double y = ::sin(moonLongitude - nodeLongitude); + double x = cos(moonLongitude - nodeLongitude); + + moonEclipLong = ::atan2(y*cos(moonI), x) + nodeLongitude; + double moonEclipLat = ::asin(y * ::sin(moonI)); + + eclipticToEquatorial(moonPosition, moonEclipLong, moonEclipLat); + moonPositionSet = true; + } + return moonPosition; +} + +/** + * The "age" of the moon at the time specified in this object. + * This is really the angle between the + * current ecliptic longitudes of the sun and the moon, + * measured in radians. + * + * @see #getMoonPhase + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getMoonAge() { + // See page 147 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + // + // Force the moon's position to be calculated. We're going to use + // some the intermediate results cached during that calculation. + // + getMoonPosition(); + + return norm2PI(moonEclipLong - sunLongitude); +} + +/** + * Calculate the phase of the moon at the time set in this object. + * The returned phase is a double in the range + * 0 <= phase < 1, interpreted as follows: + *

    + *
  • 0.00: New moon + *
  • 0.25: First quarter + *
  • 0.50: Full moon + *
  • 0.75: Last quarter + *
+ * + * @see #getMoonAge + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +double CalendarAstronomer::getMoonPhase() { + // See page 147 of "Practical Astronomy with your Calculator", + // by Peter Duffet-Smith, for details on the algorithm. + return 0.5 * (1 - cos(getMoonAge())); +} + +/** + * Constant representing a new moon. + * For use with {@link #getMoonTime getMoonTime} + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +const CalendarAstronomer::MoonAge CalendarAstronomer::NEW_MOON() { + return CalendarAstronomer::MoonAge(0); +} + +/** + * Constant representing the moon's first quarter. + * For use with {@link #getMoonTime getMoonTime} + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +/*const CalendarAstronomer::MoonAge CalendarAstronomer::FIRST_QUARTER() { + return CalendarAstronomer::MoonAge(CalendarAstronomer::PI/2); +}*/ + +/** + * Constant representing a full moon. + * For use with {@link #getMoonTime getMoonTime} + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +const CalendarAstronomer::MoonAge CalendarAstronomer::FULL_MOON() { + return CalendarAstronomer::MoonAge(CalendarAstronomer::PI); +} +/** + * Constant representing the moon's last quarter. + * For use with {@link #getMoonTime getMoonTime} + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ + +class MoonTimeAngleFunc : public CalendarAstronomer::AngleFunc { +public: + virtual ~MoonTimeAngleFunc(); + virtual double eval(CalendarAstronomer& a) override { return a.getMoonAge(); } +}; + +MoonTimeAngleFunc::~MoonTimeAngleFunc() {} + +/*const CalendarAstronomer::MoonAge CalendarAstronomer::LAST_QUARTER() { + return CalendarAstronomer::MoonAge((CalendarAstronomer::PI*3)/2); +}*/ + +/** + * Find the next or previous time at which the Moon's ecliptic + * longitude will have the desired value. + *

+ * @param desired The desired longitude. + * @param next true if the next occurrence of the phase + * is desired, false for the previous occurrence. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +UDate CalendarAstronomer::getMoonTime(double desired, UBool next) +{ + MoonTimeAngleFunc func; + return timeOfAngle( func, + desired, + SYNODIC_MONTH, + MINUTE_MS, + next); +} + +/** + * Find the next or previous time at which the moon will be in the + * desired phase. + *

+ * @param desired The desired phase of the moon. + * @param next true if the next occurrence of the phase + * is desired, false for the previous occurrence. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +UDate CalendarAstronomer::getMoonTime(const CalendarAstronomer::MoonAge& desired, UBool next) { + return getMoonTime(desired.value, next); +} + +class MoonRiseSetCoordFunc : public CalendarAstronomer::CoordFunc { +public: + virtual ~MoonRiseSetCoordFunc(); + virtual void eval(CalendarAstronomer::Equatorial& result, CalendarAstronomer& a) override { result = a.getMoonPosition(); } +}; + +MoonRiseSetCoordFunc::~MoonRiseSetCoordFunc() {} + +/** + * Returns the time (GMT) of sunrise or sunset on the local date to which + * this calendar is currently set. + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +UDate CalendarAstronomer::getMoonRiseSet(UBool rise) +{ + MoonRiseSetCoordFunc func; + return riseOrSet(func, + rise, + .533 * DEG_RAD, // Angular Diameter + 34 /60.0 * DEG_RAD, // Refraction correction + MINUTE_MS); // Desired accuracy +} + +//------------------------------------------------------------------------- +// Interpolation methods for finding the time at which a given event occurs +//------------------------------------------------------------------------- + +UDate CalendarAstronomer::timeOfAngle(AngleFunc& func, double desired, + double periodDays, double epsilon, UBool next) +{ + // Find the value of the function at the current time + double lastAngle = func.eval(*this); + + // Find out how far we are from the desired angle + double deltaAngle = norm2PI(desired - lastAngle) ; + + // Using the average period, estimate the next (or previous) time at + // which the desired angle occurs. + double deltaT = (deltaAngle + (next ? 0.0 : - CalendarAstronomer_PI2 )) * (periodDays*DAY_MS) / CalendarAstronomer_PI2; + + double lastDeltaT = deltaT; // Liu + UDate startTime = fTime; // Liu + + setTime(fTime + uprv_ceil(deltaT)); + + // Now iterate until we get the error below epsilon. Throughout + // this loop we use normPI to get values in the range -Pi to Pi, + // since we're using them as correction factors rather than absolute angles. + do { + // Evaluate the function at the time we've estimated + double angle = func.eval(*this); + + // Find the # of milliseconds per radian at this point on the curve + double factor = uprv_fabs(deltaT / normPI(angle-lastAngle)); + + // Correct the time estimate based on how far off the angle is + deltaT = normPI(desired - angle) * factor; + + // HACK: + // + // If abs(deltaT) begins to diverge we need to quit this loop. + // This only appears to happen when attempting to locate, for + // example, a new moon on the day of the new moon. E.g.: + // + // This result is correct: + // newMoon(7508(Mon Jul 23 00:00:00 CST 1990,false))= + // Sun Jul 22 10:57:41 CST 1990 + // + // But attempting to make the same call a day earlier causes deltaT + // to diverge: + // CalendarAstronomer.timeOfAngle() diverging: 1.348508727575625E9 -> + // 1.3649828540224032E9 + // newMoon(7507(Sun Jul 22 00:00:00 CST 1990,false))= + // Sun Jul 08 13:56:15 CST 1990 + // + // As a temporary solution, we catch this specific condition and + // adjust our start time by one eighth period days (either forward + // or backward) and try again. + // Liu 11/9/00 + if (uprv_fabs(deltaT) > uprv_fabs(lastDeltaT)) { + double delta = uprv_ceil (periodDays * DAY_MS / 8.0); + setTime(startTime + (next ? delta : -delta)); + return timeOfAngle(func, desired, periodDays, epsilon, next); + } + + lastDeltaT = deltaT; + lastAngle = angle; + + setTime(fTime + uprv_ceil(deltaT)); + } + while (uprv_fabs(deltaT) > epsilon); + + return fTime; +} + +UDate CalendarAstronomer::riseOrSet(CoordFunc& func, UBool rise, + double diameter, double refraction, + double epsilon) +{ + Equatorial pos; + double tanL = ::tan(fLatitude); + double deltaT = 0; + int32_t count = 0; + + // + // Calculate the object's position at the current time, then use that + // position to calculate the time of rising or setting. The position + // will be different at that time, so iterate until the error is allowable. + // + U_DEBUG_ASTRO_MSG(("setup rise=%s, dia=%.3lf, ref=%.3lf, eps=%.3lf\n", + rise?"T":"F", diameter, refraction, epsilon)); + do { + // See "Practical Astronomy With Your Calculator, section 33. + func.eval(pos, *this); + double angle = ::acos(-tanL * ::tan(pos.declination)); + double lst = ((rise ? CalendarAstronomer_PI2-angle : angle) + pos.ascension ) * 24 / CalendarAstronomer_PI2; + + // Convert from LST to Universal Time. + UDate newTime = lstToUT( lst ); + + deltaT = newTime - fTime; + setTime(newTime); + U_DEBUG_ASTRO_MSG(("%d] dT=%.3lf, angle=%.3lf, lst=%.3lf, A=%.3lf/D=%.3lf\n", + count, deltaT, angle, lst, pos.ascension, pos.declination)); + } + while (++ count < 5 && uprv_fabs(deltaT) > epsilon); + + // Calculate the correction due to refraction and the object's angular diameter + double cosD = ::cos(pos.declination); + double psi = ::acos(sin(fLatitude) / cosD); + double x = diameter / 2 + refraction; + double y = ::asin(sin(x) / ::sin(psi)); + long delta = (long)((240 * y * RAD_DEG / cosD)*SECOND_MS); + + return fTime + (rise ? -delta : delta); +} + /** + * Return the obliquity of the ecliptic (the angle between the ecliptic + * and the earth's equator) at the current time. This varies due to + * the precession of the earth's axis. + * + * @return the obliquity of the ecliptic relative to the equator, + * measured in radians. + */ +double CalendarAstronomer::eclipticObliquity() { + if (isINVALID(eclipObliquity)) { + const double epoch = 2451545.0; // 2000 AD, January 1.5 + + double T = (getJulianDay() - epoch) / 36525; + + eclipObliquity = 23.439292 + - 46.815/3600 * T + - 0.0006/3600 * T*T + + 0.00181/3600 * T*T*T; + + eclipObliquity *= DEG_RAD; + } + return eclipObliquity; +} + + +//------------------------------------------------------------------------- +// Private data +//------------------------------------------------------------------------- +void CalendarAstronomer::clearCache() { + const double INVALID = uprv_getNaN(); + + julianDay = INVALID; + julianCentury = INVALID; + sunLongitude = INVALID; + meanAnomalySun = INVALID; + moonLongitude = INVALID; + moonEclipLong = INVALID; + meanAnomalyMoon = INVALID; + eclipObliquity = INVALID; + siderealTime = INVALID; + siderealT0 = INVALID; + moonPositionSet = false; +} + +//private static void out(String s) { +// System.out.println(s); +//} + +//private static String deg(double rad) { +// return Double.toString(rad * RAD_DEG); +//} + +//private static String hours(long ms) { +// return Double.toString((double)ms / HOUR_MS) + " hours"; +//} + +/** + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ +/*UDate CalendarAstronomer::local(UDate localMillis) { + // TODO - srl ? + TimeZone *tz = TimeZone::createDefault(); + int32_t rawOffset; + int32_t dstOffset; + UErrorCode status = U_ZERO_ERROR; + tz->getOffset(localMillis, true, rawOffset, dstOffset, status); + delete tz; + return localMillis - rawOffset; +}*/ + +// Debugging functions +UnicodeString CalendarAstronomer::Ecliptic::toString() const +{ +#ifdef U_DEBUG_ASTRO + char tmp[800]; + snprintf(tmp, sizeof(tmp), "[%.5f,%.5f]", longitude*RAD_DEG, latitude*RAD_DEG); + return UnicodeString(tmp, ""); +#else + return UnicodeString(); +#endif +} + +UnicodeString CalendarAstronomer::Equatorial::toString() const +{ +#ifdef U_DEBUG_ASTRO + char tmp[400]; + snprintf(tmp, sizeof(tmp), "%f,%f", + (ascension*RAD_DEG), (declination*RAD_DEG)); + return UnicodeString(tmp, ""); +#else + return UnicodeString(); +#endif +} + +UnicodeString CalendarAstronomer::Horizon::toString() const +{ +#ifdef U_DEBUG_ASTRO + char tmp[800]; + snprintf(tmp, sizeof(tmp), "[%.5f,%.5f]", altitude*RAD_DEG, azimuth*RAD_DEG); + return UnicodeString(tmp, ""); +#else + return UnicodeString(); +#endif +} + + +// static private String radToHms(double angle) { +// int hrs = (int) (angle*RAD_HOUR); +// int min = (int)((angle*RAD_HOUR - hrs) * 60); +// int sec = (int)((angle*RAD_HOUR - hrs - min/60.0) * 3600); + +// return Integer.toString(hrs) + "h" + min + "m" + sec + "s"; +// } + +// static private String radToDms(double angle) { +// int deg = (int) (angle*RAD_DEG); +// int min = (int)((angle*RAD_DEG - deg) * 60); +// int sec = (int)((angle*RAD_DEG - deg - min/60.0) * 3600); + +// return Integer.toString(deg) + "\u00b0" + min + "'" + sec + "\""; +// } + +// =============== Calendar Cache ================ + +void CalendarCache::createCache(CalendarCache** cache, UErrorCode& status) { + ucln_i18n_registerCleanup(UCLN_I18N_ASTRO_CALENDAR, calendar_astro_cleanup); + if(cache == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + *cache = new CalendarCache(32, status); + if(U_FAILURE(status)) { + delete *cache; + *cache = nullptr; + } + } +} + +int32_t CalendarCache::get(CalendarCache** cache, int32_t key, UErrorCode &status) { + int32_t res; + + if(U_FAILURE(status)) { + return 0; + } + umtx_lock(&ccLock); + + if(*cache == nullptr) { + createCache(cache, status); + if(U_FAILURE(status)) { + umtx_unlock(&ccLock); + return 0; + } + } + + res = uhash_igeti((*cache)->fTable, key); + U_DEBUG_ASTRO_MSG(("%p: GET: [%d] == %d\n", (*cache)->fTable, key, res)); + + umtx_unlock(&ccLock); + return res; +} + +void CalendarCache::put(CalendarCache** cache, int32_t key, int32_t value, UErrorCode &status) { + if(U_FAILURE(status)) { + return; + } + umtx_lock(&ccLock); + + if(*cache == nullptr) { + createCache(cache, status); + if(U_FAILURE(status)) { + umtx_unlock(&ccLock); + return; + } + } + + uhash_iputi((*cache)->fTable, key, value, &status); + U_DEBUG_ASTRO_MSG(("%p: PUT: [%d] := %d\n", (*cache)->fTable, key, value)); + + umtx_unlock(&ccLock); +} + +CalendarCache::CalendarCache(int32_t size, UErrorCode &status) { + fTable = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, size, &status); + U_DEBUG_ASTRO_MSG(("%p: Opening.\n", fTable)); +} + +CalendarCache::~CalendarCache() { + if(fTable != nullptr) { + U_DEBUG_ASTRO_MSG(("%p: Closing.\n", fTable)); + uhash_close(fTable); + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/astro.h b/intl/icu/source/i18n/astro.h new file mode 100644 index 0000000000..372a79ac67 --- /dev/null +++ b/intl/icu/source/i18n/astro.h @@ -0,0 +1,757 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/************************************************************************ + * Copyright (C) 1996-2008, International Business Machines Corporation * + * and others. All Rights Reserved. * + ************************************************************************ + * 2003-nov-07 srl Port from Java + */ + +#ifndef ASTRO_H +#define ASTRO_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "gregoimp.h" // for Math +#include "unicode/unistr.h" + +U_NAMESPACE_BEGIN + +/** + * CalendarAstronomer is a class that can perform the calculations to + * determine the positions of the sun and moon, the time of sunrise and + * sunset, and other astronomy-related data. The calculations it performs + * are in some cases quite complicated, and this utility class saves you + * the trouble of worrying about them. + *

+ * The measurement of time is a very important part of astronomy. Because + * astronomical bodies are constantly in motion, observations are only valid + * at a given moment in time. Accordingly, each CalendarAstronomer + * object has a time property that determines the date + * and time for which its calculations are performed. You can set and + * retrieve this property with {@link #setDate setDate}, {@link #getDate getDate} + * and related methods. + *

+ * Almost all of the calculations performed by this class, or by any + * astronomer, are approximations to various degrees of accuracy. The + * calculations in this class are mostly modelled after those described + * in the book + * + * Practical Astronomy With Your Calculator, by Peter J. + * Duffett-Smith, Cambridge University Press, 1990. This is an excellent + * book, and if you want a greater understanding of how these calculations + * are performed it a very good, readable starting point. + *

+ * WARNING: This class is very early in its development, and + * it is highly likely that its API will change to some degree in the future. + * At the moment, it basically does just enough to support {@link IslamicCalendar} + * and {@link ChineseCalendar}. + * + * @author Laura Werner + * @author Alan Liu + * @internal + */ +class U_I18N_API CalendarAstronomer : public UMemory { +public: + // some classes + +public: + /** + * Represents the position of an object in the sky relative to the ecliptic, + * the plane of the earth's orbit around the Sun. + * This is a spherical coordinate system in which the latitude + * specifies the position north or south of the plane of the ecliptic. + * The longitude specifies the position along the ecliptic plane + * relative to the "First Point of Aries", which is the Sun's position in the sky + * at the Vernal Equinox. + *

+ * Note that Ecliptic objects are immutable and cannot be modified + * once they are constructed. This allows them to be passed and returned by + * value without worrying about whether other code will modify them. + * + * @see CalendarAstronomer.Equatorial + * @see CalendarAstronomer.Horizon + * @internal + */ + class U_I18N_API Ecliptic : public UMemory { + public: + /** + * Constructs an Ecliptic coordinate object. + *

+ * @param lat The ecliptic latitude, measured in radians. + * @param lon The ecliptic longitude, measured in radians. + * @internal + */ + Ecliptic(double lat = 0, double lon = 0) { + latitude = lat; + longitude = lon; + } + + /** + * Setter for Ecliptic Coordinate object + * @param lat The ecliptic latitude, measured in radians. + * @param lon The ecliptic longitude, measured in radians. + * @internal + */ + void set(double lat, double lon) { + latitude = lat; + longitude = lon; + } + + /** + * Return a string representation of this object + * @internal + */ + UnicodeString toString() const; + + /** + * The ecliptic latitude, in radians. This specifies an object's + * position north or south of the plane of the ecliptic, + * with positive angles representing north. + * @internal + */ + double latitude; + + /** + * The ecliptic longitude, in radians. + * This specifies an object's position along the ecliptic plane + * relative to the "First Point of Aries", which is the Sun's position + * in the sky at the Vernal Equinox, + * with positive angles representing east. + *

+ * A bit of trivia: the first point of Aries is currently in the + * constellation Pisces, due to the precession of the earth's axis. + * @internal + */ + double longitude; + }; + + /** + * Represents the position of an + * object in the sky relative to the plane of the earth's equator. + * The Right Ascension specifies the position east or west + * along the equator, relative to the sun's position at the vernal + * equinox. The Declination is the position north or south + * of the equatorial plane. + *

+ * Note that Equatorial objects are immutable and cannot be modified + * once they are constructed. This allows them to be passed and returned by + * value without worrying about whether other code will modify them. + * + * @see CalendarAstronomer.Ecliptic + * @see CalendarAstronomer.Horizon + * @internal + */ + class U_I18N_API Equatorial : public UMemory { + public: + /** + * Constructs an Equatorial coordinate object. + *

+ * @param asc The right ascension, measured in radians. + * @param dec The declination, measured in radians. + * @internal + */ + Equatorial(double asc = 0, double dec = 0) + : ascension(asc), declination(dec) { } + + /** + * Setter + * @param asc The right ascension, measured in radians. + * @param dec The declination, measured in radians. + * @internal + */ + void set(double asc, double dec) { + ascension = asc; + declination = dec; + } + + /** + * Return a string representation of this object, with the + * angles measured in degrees. + * @internal + */ + UnicodeString toString() const; + + /** + * Return a string representation of this object with the right ascension + * measured in hours, minutes, and seconds. + * @internal + */ + //String toHmsString() { + //return radToHms(ascension) + "," + radToDms(declination); + //} + + /** + * The right ascension, in radians. + * This is the position east or west along the equator + * relative to the sun's position at the vernal equinox, + * with positive angles representing East. + * @internal + */ + double ascension; + + /** + * The declination, in radians. + * This is the position north or south of the equatorial plane, + * with positive angles representing north. + * @internal + */ + double declination; + }; + + /** + * Represents the position of an object in the sky relative to + * the local horizon. + * The Altitude represents the object's elevation above the horizon, + * with objects below the horizon having a negative altitude. + * The Azimuth is the geographic direction of the object from the + * observer's position, with 0 representing north. The azimuth increases + * clockwise from north. + *

+ * Note that Horizon objects are immutable and cannot be modified + * once they are constructed. This allows them to be passed and returned by + * value without worrying about whether other code will modify them. + * + * @see CalendarAstronomer.Ecliptic + * @see CalendarAstronomer.Equatorial + * @internal + */ + class U_I18N_API Horizon : public UMemory { + public: + /** + * Constructs a Horizon coordinate object. + *

+ * @param alt The altitude, measured in radians above the horizon. + * @param azim The azimuth, measured in radians clockwise from north. + * @internal + */ + Horizon(double alt=0, double azim=0) + : altitude(alt), azimuth(azim) { } + + /** + * Setter for Ecliptic Coordinate object + * @param alt The altitude, measured in radians above the horizon. + * @param azim The azimuth, measured in radians clockwise from north. + * @internal + */ + void set(double alt, double azim) { + altitude = alt; + azimuth = azim; + } + + /** + * Return a string representation of this object, with the + * angles measured in degrees. + * @internal + */ + UnicodeString toString() const; + + /** + * The object's altitude above the horizon, in radians. + * @internal + */ + double altitude; + + /** + * The object's direction, in radians clockwise from north. + * @internal + */ + double azimuth; + }; + +public: + //------------------------------------------------------------------------- + // Assorted private data used for conversions + //------------------------------------------------------------------------- + + // My own copies of these so compilers are more likely to optimize them away + static const double PI; + + /** + * The average number of solar days from one new moon to the next. This is the time + * it takes for the moon to return the same ecliptic longitude as the sun. + * It is longer than the sidereal month because the sun's longitude increases + * during the year due to the revolution of the earth around the sun. + * Approximately 29.53. + * + * @see #SIDEREAL_MONTH + * @internal + * @deprecated ICU 2.4. This class may be removed or modified. + */ + static const double SYNODIC_MONTH; + + //------------------------------------------------------------------------- + // Constructors + //------------------------------------------------------------------------- + + /** + * Construct a new CalendarAstronomer object that is initialized to + * the current date and time. + * @internal + */ + CalendarAstronomer(); + + /** + * Construct a new CalendarAstronomer object that is initialized to + * the specified date and time. + * @internal + */ + CalendarAstronomer(UDate d); + + /** + * Construct a new CalendarAstronomer object with the given + * latitude and longitude. The object's time is set to the current + * date and time. + *

+ * @param longitude The desired longitude, in degrees east of + * the Greenwich meridian. + * + * @param latitude The desired latitude, in degrees. Positive + * values signify North, negative South. + * + * @see java.util.Date#getTime() + * @internal + */ + CalendarAstronomer(double longitude, double latitude); + + /** + * Destructor + * @internal + */ + ~CalendarAstronomer(); + + //------------------------------------------------------------------------- + // Time and date getters and setters + //------------------------------------------------------------------------- + + /** + * Set the current date and time of this CalendarAstronomer object. All + * astronomical calculations are performed based on this time setting. + * + * @param aTime the date and time, expressed as the number of milliseconds since + * 1/1/1970 0:00 GMT (Gregorian). + * + * @see #setDate + * @see #getTime + * @internal + */ + void setTime(UDate aTime); + + + /** + * Set the current date and time of this CalendarAstronomer object. All + * astronomical calculations are performed based on this time setting. + * + * @param aTime the date and time, expressed as the number of milliseconds since + * 1/1/1970 0:00 GMT (Gregorian). + * + * @see #getTime + * @internal + */ + void setDate(UDate aDate) { setTime(aDate); } + + /** + * Set the current date and time of this CalendarAstronomer object. All + * astronomical calculations are performed based on this time setting. + * + * @param jdn the desired time, expressed as a "julian day number", + * which is the number of elapsed days since + * 1/1/4713 BC (Julian), 12:00 GMT. Note that julian day + * numbers start at noon. To get the jdn for + * the corresponding midnight, subtract 0.5. + * + * @see #getJulianDay + * @see #JULIAN_EPOCH_MS + * @internal + */ + void setJulianDay(double jdn); + + /** + * Get the current time of this CalendarAstronomer object, + * represented as the number of milliseconds since + * 1/1/1970 AD 0:00 GMT (Gregorian). + * + * @see #setTime + * @see #getDate + * @internal + */ + UDate getTime(); + + /** + * Get the current time of this CalendarAstronomer object, + * expressed as a "julian day number", which is the number of elapsed + * days since 1/1/4713 BC (Julian), 12:00 GMT. + * + * @see #setJulianDay + * @see #JULIAN_EPOCH_MS + * @internal + */ + double getJulianDay(); + + /** + * Return this object's time expressed in julian centuries: + * the number of centuries after 1/1/1900 AD, 12:00 GMT + * + * @see #getJulianDay + * @internal + */ + double getJulianCentury(); + + /** + * Returns the current Greenwich sidereal time, measured in hours + * @internal + */ + double getGreenwichSidereal(); + +private: + double getSiderealOffset(); +public: + /** + * Returns the current local sidereal time, measured in hours + * @internal + */ + double getLocalSidereal(); + + /** + * Converts local sidereal time to Universal Time. + * + * @param lst The Local Sidereal Time, in hours since sidereal midnight + * on this object's current date. + * + * @return The corresponding Universal Time, in milliseconds since + * 1 Jan 1970, GMT. + */ + //private: + double lstToUT(double lst); + + /** + * + * Convert from ecliptic to equatorial coordinates. + * + * @param ecliptic The ecliptic + * @param result Fillin result + * @return reference to result + */ + Equatorial& eclipticToEquatorial(Equatorial& result, const Ecliptic& ecliptic); + + /** + * Convert from ecliptic to equatorial coordinates. + * + * @param eclipLong The ecliptic longitude + * @param eclipLat The ecliptic latitude + * + * @return The corresponding point in equatorial coordinates. + * @internal + */ + Equatorial& eclipticToEquatorial(Equatorial& result, double eclipLong, double eclipLat); + + /** + * Convert from ecliptic longitude to equatorial coordinates. + * + * @param eclipLong The ecliptic longitude + * + * @return The corresponding point in equatorial coordinates. + * @internal + */ + Equatorial& eclipticToEquatorial(Equatorial& result, double eclipLong) ; + + /** + * @internal + */ + Horizon& eclipticToHorizon(Horizon& result, double eclipLong) ; + + //------------------------------------------------------------------------- + // The Sun + //------------------------------------------------------------------------- + + /** + * The longitude of the sun at the time specified by this object. + * The longitude is measured in radians along the ecliptic + * from the "first point of Aries," the point at which the ecliptic + * crosses the earth's equatorial plane at the vernal equinox. + *

+ * Currently, this method uses an approximation of the two-body Kepler's + * equation for the earth and the sun. It does not take into account the + * perturbations caused by the other planets, the moon, etc. + * @internal + */ + double getSunLongitude(); + + /** + * TODO Make this public when the entire class is package-private. + */ + /*public*/ void getSunLongitude(double julianDay, double &longitude, double &meanAnomaly); + + /** + * The position of the sun at this object's current date and time, + * in equatorial coordinates. + * @param result fillin for the result + * @internal + */ + Equatorial& getSunPosition(Equatorial& result); + +public: + /** + * Constant representing the vernal equinox. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "vernal" refers to the northern hemisphere's seasons. + * @internal + */ +// static double VERNAL_EQUINOX(); + + /** + * Constant representing the summer solstice. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "summer" refers to the northern hemisphere's seasons. + * @internal + */ + static double SUMMER_SOLSTICE(); + + /** + * Constant representing the autumnal equinox. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "autumn" refers to the northern hemisphere's seasons. + * @internal + */ +// static double AUTUMN_EQUINOX(); + + /** + * Constant representing the winter solstice. + * For use with {@link #getSunTime getSunTime}. + * Note: In this case, "winter" refers to the northern hemisphere's seasons. + * @internal + */ + static double WINTER_SOLSTICE(); + + /** + * Find the next time at which the sun's ecliptic longitude will have + * the desired value. + * @internal + */ + UDate getSunTime(double desired, UBool next); + + /** + * Returns the time (GMT) of sunrise or sunset on the local date to which + * this calendar is currently set. + * + * NOTE: This method only works well if this object is set to a + * time near local noon. Because of variations between the local + * official time zone and the geographic longitude, the + * computation can flop over into an adjacent day if this object + * is set to a time near local midnight. + * + * @internal + */ + UDate getSunRiseSet(UBool rise); + + //------------------------------------------------------------------------- + // The Moon + //------------------------------------------------------------------------- + + /** + * The position of the moon at the time set on this + * object, in equatorial coordinates. + * @internal + * @return const reference to internal field of calendar astronomer. Do not use outside of the lifetime of this astronomer. + */ + const Equatorial& getMoonPosition(); + + /** + * The "age" of the moon at the time specified in this object. + * This is really the angle between the + * current ecliptic longitudes of the sun and the moon, + * measured in radians. + * + * @see #getMoonPhase + * @internal + */ + double getMoonAge(); + + /** + * Calculate the phase of the moon at the time set in this object. + * The returned phase is a double in the range + * 0 <= phase < 1, interpreted as follows: + *

    + *
  • 0.00: New moon + *
  • 0.25: First quarter + *
  • 0.50: Full moon + *
  • 0.75: Last quarter + *
+ * + * @see #getMoonAge + * @internal + */ + double getMoonPhase(); + + class U_I18N_API MoonAge : public UMemory { + public: + MoonAge(double l) + : value(l) { } + void set(double l) { value = l; } + double value; + }; + + /** + * Constant representing a new moon. + * For use with {@link #getMoonTime getMoonTime} + * @internal + */ + static const MoonAge NEW_MOON(); + + /** + * Constant representing the moon's first quarter. + * For use with {@link #getMoonTime getMoonTime} + * @internal + */ +// static const MoonAge FIRST_QUARTER(); + + /** + * Constant representing a full moon. + * For use with {@link #getMoonTime getMoonTime} + * @internal + */ + static const MoonAge FULL_MOON(); + + /** + * Constant representing the moon's last quarter. + * For use with {@link #getMoonTime getMoonTime} + * @internal + */ +// static const MoonAge LAST_QUARTER(); + + /** + * Find the next or previous time at which the Moon's ecliptic + * longitude will have the desired value. + *

+ * @param desired The desired longitude. + * @param next true if the next occurrence of the phase + * is desired, false for the previous occurrence. + * @internal + */ + UDate getMoonTime(double desired, UBool next); + UDate getMoonTime(const MoonAge& desired, UBool next); + + /** + * Returns the time (GMT) of sunrise or sunset on the local date to which + * this calendar is currently set. + * @internal + */ + UDate getMoonRiseSet(UBool rise); + + //------------------------------------------------------------------------- + // Interpolation methods for finding the time at which a given event occurs + //------------------------------------------------------------------------- + + // private + class AngleFunc : public UMemory { + public: + virtual double eval(CalendarAstronomer&) = 0; + virtual ~AngleFunc(); + }; + friend class AngleFunc; + + UDate timeOfAngle(AngleFunc& func, double desired, + double periodDays, double epsilon, UBool next); + + class CoordFunc : public UMemory { + public: + virtual void eval(Equatorial& result, CalendarAstronomer&) = 0; + virtual ~CoordFunc(); + }; + friend class CoordFunc; + + double riseOrSet(CoordFunc& func, UBool rise, + double diameter, double refraction, + double epsilon); + + //------------------------------------------------------------------------- + // Other utility methods + //------------------------------------------------------------------------- +private: + + /** + * Return the obliquity of the ecliptic (the angle between the ecliptic + * and the earth's equator) at the current time. This varies due to + * the precession of the earth's axis. + * + * @return the obliquity of the ecliptic relative to the equator, + * measured in radians. + */ + double eclipticObliquity(); + + //------------------------------------------------------------------------- + // Private data + //------------------------------------------------------------------------- +private: + /** + * Current time in milliseconds since 1/1/1970 AD + * @see java.util.Date#getTime + */ + UDate fTime; + + /* These aren't used yet, but they'll be needed for sunset calculations + * and equatorial to horizon coordinate conversions + */ + double fLongitude; + double fLatitude; + double fGmtOffset; + + // + // The following fields are used to cache calculated results for improved + // performance. These values all depend on the current time setting + // of this object, so the clearCache method is provided. + // + + double julianDay; + double julianCentury; + double sunLongitude; + double meanAnomalySun; + double moonLongitude; + double moonEclipLong; + double meanAnomalyMoon; + double eclipObliquity; + double siderealT0; + double siderealTime; + + void clearCache(); + + Equatorial moonPosition; + UBool moonPositionSet; + + /** + * @internal + */ +// UDate local(UDate localMillis); +}; + +U_NAMESPACE_END + +struct UHashtable; + +U_NAMESPACE_BEGIN + +/** + * Cache of month -> julian day + * @internal + */ +class CalendarCache : public UMemory { +public: + static int32_t get(CalendarCache** cache, int32_t key, UErrorCode &status); + static void put(CalendarCache** cache, int32_t key, int32_t value, UErrorCode &status); + virtual ~CalendarCache(); +private: + CalendarCache(int32_t size, UErrorCode& status); + static void createCache(CalendarCache** cache, UErrorCode& status); + /** + * not implemented + */ + CalendarCache(); + UHashtable *fTable; +}; + +U_NAMESPACE_END + +#endif +#endif diff --git a/intl/icu/source/i18n/basictz.cpp b/intl/icu/source/i18n/basictz.cpp new file mode 100644 index 0000000000..2490fadcc9 --- /dev/null +++ b/intl/icu/source/i18n/basictz.cpp @@ -0,0 +1,539 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2013, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/basictz.h" +#include "gregoimp.h" +#include "uvector.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +#define MILLIS_PER_YEAR (365*24*60*60*1000.0) + +BasicTimeZone::BasicTimeZone() +: TimeZone() { +} + +BasicTimeZone::BasicTimeZone(const UnicodeString &id) +: TimeZone(id) { +} + +BasicTimeZone::BasicTimeZone(const BasicTimeZone& source) +: TimeZone(source) { +} + +BasicTimeZone::~BasicTimeZone() { +} + +UBool +BasicTimeZone::hasEquivalentTransitions(const BasicTimeZone& tz, UDate start, UDate end, + UBool ignoreDstAmount, UErrorCode& status) const { + if (U_FAILURE(status)) { + return false; + } + if (hasSameRules(tz)) { + return true; + } + // Check the offsets at the start time + int32_t raw1, raw2, dst1, dst2; + getOffset(start, false, raw1, dst1, status); + if (U_FAILURE(status)) { + return false; + } + tz.getOffset(start, false, raw2, dst2, status); + if (U_FAILURE(status)) { + return false; + } + if (ignoreDstAmount) { + if ((raw1 + dst1 != raw2 + dst2) + || (dst1 != 0 && dst2 == 0) + || (dst1 == 0 && dst2 != 0)) { + return false; + } + } else { + if (raw1 != raw2 || dst1 != dst2) { + return false; + } + } + // Check transitions in the range + UDate time = start; + TimeZoneTransition tr1, tr2; + while (true) { + UBool avail1 = getNextTransition(time, false, tr1); + UBool avail2 = tz.getNextTransition(time, false, tr2); + + if (ignoreDstAmount) { + // Skip a transition which only differ the amount of DST savings + while (true) { + if (avail1 + && tr1.getTime() <= end + && (tr1.getFrom()->getRawOffset() + tr1.getFrom()->getDSTSavings() + == tr1.getTo()->getRawOffset() + tr1.getTo()->getDSTSavings()) + && (tr1.getFrom()->getDSTSavings() != 0 && tr1.getTo()->getDSTSavings() != 0)) { + getNextTransition(tr1.getTime(), false, tr1); + } else { + break; + } + } + while (true) { + if (avail2 + && tr2.getTime() <= end + && (tr2.getFrom()->getRawOffset() + tr2.getFrom()->getDSTSavings() + == tr2.getTo()->getRawOffset() + tr2.getTo()->getDSTSavings()) + && (tr2.getFrom()->getDSTSavings() != 0 && tr2.getTo()->getDSTSavings() != 0)) { + tz.getNextTransition(tr2.getTime(), false, tr2); + } else { + break; + } + } + } + + UBool inRange1 = (avail1 && tr1.getTime() <= end); + UBool inRange2 = (avail2 && tr2.getTime() <= end); + if (!inRange1 && !inRange2) { + // No more transition in the range + break; + } + if (!inRange1 || !inRange2) { + return false; + } + if (tr1.getTime() != tr2.getTime()) { + return false; + } + if (ignoreDstAmount) { + if (tr1.getTo()->getRawOffset() + tr1.getTo()->getDSTSavings() + != tr2.getTo()->getRawOffset() + tr2.getTo()->getDSTSavings() + || (tr1.getTo()->getDSTSavings() != 0 && tr2.getTo()->getDSTSavings() == 0) + || (tr1.getTo()->getDSTSavings() == 0 && tr2.getTo()->getDSTSavings() != 0)) { + return false; + } + } else { + if (tr1.getTo()->getRawOffset() != tr2.getTo()->getRawOffset() || + tr1.getTo()->getDSTSavings() != tr2.getTo()->getDSTSavings()) { + return false; + } + } + time = tr1.getTime(); + } + return true; +} + +void +BasicTimeZone::getSimpleRulesNear(UDate date, InitialTimeZoneRule*& initial, + AnnualTimeZoneRule*& std, AnnualTimeZoneRule*& dst, UErrorCode& status) const { + initial = nullptr; + std = nullptr; + dst = nullptr; + if (U_FAILURE(status)) { + return; + } + int32_t initialRaw, initialDst; + UnicodeString initialName; + + AnnualTimeZoneRule *ar1 = nullptr; + AnnualTimeZoneRule *ar2 = nullptr; + UnicodeString name; + + UBool avail; + TimeZoneTransition tr; + // Get the next transition + avail = getNextTransition(date, false, tr); + if (avail) { + tr.getFrom()->getName(initialName); + initialRaw = tr.getFrom()->getRawOffset(); + initialDst = tr.getFrom()->getDSTSavings(); + + // Check if the next transition is either DST->STD or STD->DST and + // within roughly 1 year from the specified date + UDate nextTransitionTime = tr.getTime(); + if (((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0) + || (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0)) + && (date + MILLIS_PER_YEAR > nextTransitionTime)) { + + int32_t year, month, dom, dow, doy, mid; + UDate d; + + // Get local wall time for the next transition time + Grego::timeToFields(nextTransitionTime + initialRaw + initialDst, + year, month, dom, dow, doy, mid); + int32_t weekInMonth = Grego::dayOfWeekInMonth(year, month, dom); + // Create DOW rule + DateTimeRule *dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME); + tr.getTo()->getName(name); + + // Note: SimpleTimeZone does not support raw offset change. + // So we always use raw offset of the given time for the rule, + // even raw offset is changed. This will result that the result + // zone to return wrong offset after the transition. + // When we encounter such case, we do not inspect next next + // transition for another rule. + ar1 = new AnnualTimeZoneRule(name, initialRaw, tr.getTo()->getDSTSavings(), + dtr, year, AnnualTimeZoneRule::MAX_YEAR); + + if (tr.getTo()->getRawOffset() == initialRaw) { + // Get the next next transition + avail = getNextTransition(nextTransitionTime, false, tr); + if (avail) { + // Check if the next next transition is either DST->STD or STD->DST + // and within roughly 1 year from the next transition + if (((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0) + || (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0)) + && nextTransitionTime + MILLIS_PER_YEAR > tr.getTime()) { + + // Get local wall time for the next transition time + Grego::timeToFields(tr.getTime() + tr.getFrom()->getRawOffset() + tr.getFrom()->getDSTSavings(), + year, month, dom, dow, doy, mid); + weekInMonth = Grego::dayOfWeekInMonth(year, month, dom); + // Generate another DOW rule + dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME); + tr.getTo()->getName(name); + ar2 = new AnnualTimeZoneRule(name, tr.getTo()->getRawOffset(), tr.getTo()->getDSTSavings(), + dtr, year - 1, AnnualTimeZoneRule::MAX_YEAR); + + // Make sure this rule can be applied to the specified date + avail = ar2->getPreviousStart(date, tr.getFrom()->getRawOffset(), tr.getFrom()->getDSTSavings(), true, d); + if (!avail || d > date + || initialRaw != tr.getTo()->getRawOffset() + || initialDst != tr.getTo()->getDSTSavings()) { + // We cannot use this rule as the second transition rule + delete ar2; + ar2 = nullptr; + } + } + } + } + if (ar2 == nullptr) { + // Try previous transition + avail = getPreviousTransition(date, true, tr); + if (avail) { + // Check if the previous transition is either DST->STD or STD->DST. + // The actual transition time does not matter here. + if ((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0) + || (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0)) { + + // Generate another DOW rule + Grego::timeToFields(tr.getTime() + tr.getFrom()->getRawOffset() + tr.getFrom()->getDSTSavings(), + year, month, dom, dow, doy, mid); + weekInMonth = Grego::dayOfWeekInMonth(year, month, dom); + dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME); + tr.getTo()->getName(name); + + // second rule raw/dst offsets should match raw/dst offsets + // at the given time + ar2 = new AnnualTimeZoneRule(name, initialRaw, initialDst, + dtr, ar1->getStartYear() - 1, AnnualTimeZoneRule::MAX_YEAR); + + // Check if this rule start after the first rule after the specified date + avail = ar2->getNextStart(date, tr.getFrom()->getRawOffset(), tr.getFrom()->getDSTSavings(), false, d); + if (!avail || d <= nextTransitionTime) { + // We cannot use this rule as the second transition rule + delete ar2; + ar2 = nullptr; + } + } + } + } + if (ar2 == nullptr) { + // Cannot find a good pair of AnnualTimeZoneRule + delete ar1; + ar1 = nullptr; + } else { + // The initial rule should represent the rule before the previous transition + ar1->getName(initialName); + initialRaw = ar1->getRawOffset(); + initialDst = ar1->getDSTSavings(); + } + } + } + else { + // Try the previous one + avail = getPreviousTransition(date, true, tr); + if (avail) { + tr.getTo()->getName(initialName); + initialRaw = tr.getTo()->getRawOffset(); + initialDst = tr.getTo()->getDSTSavings(); + } else { + // No transitions in the past. Just use the current offsets + getOffset(date, false, initialRaw, initialDst, status); + if (U_FAILURE(status)) { + return; + } + } + } + // Set the initial rule + initial = new InitialTimeZoneRule(initialName, initialRaw, initialDst); + + // Set the standard and daylight saving rules + if (ar1 != nullptr && ar2 != nullptr) { + if (ar1->getDSTSavings() != 0) { + dst = ar1; + std = ar2; + } else { + std = ar1; + dst = ar2; + } + } +} + +void +BasicTimeZone::getTimeZoneRulesAfter(UDate start, InitialTimeZoneRule*& initial, + UVector*& transitionRules, UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + + const InitialTimeZoneRule *orgini; + TimeZoneTransition tzt; + bool avail; + int32_t ruleCount; + TimeZoneRule *r = nullptr; + UnicodeString name; + int32_t i; + UDate time, t; + UDate firstStart; + UBool bFinalStd = false, bFinalDst = false; + + initial = nullptr; + transitionRules = nullptr; + + // Original transition rules + ruleCount = countTransitionRules(status); + if (U_FAILURE(status)) { + return; + } + LocalPointer orgRules( + new UVector(uprv_deleteUObject, nullptr, ruleCount, status), status); + if (U_FAILURE(status)) { + return; + } + LocalMemory orgtrs( + static_cast(uprv_malloc(sizeof(TimeZoneRule*)*ruleCount))); + if (orgtrs.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + getTimeZoneRules(orgini, &orgtrs[0], ruleCount, status); + if (U_FAILURE(status)) { + return; + } + for (i = 0; i < ruleCount; i++) { + LocalPointer lpRule(orgtrs[i]->clone(), status); + orgRules->adoptElement(lpRule.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } + + avail = getPreviousTransition(start, true, tzt); + if (!avail) { + // No need to filter out rules only applicable to time before the start + initial = orgini->clone(); + if (initial == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + transitionRules = orgRules.orphan(); + return; + } + + LocalMemory done(static_cast(uprv_malloc(sizeof(bool)*ruleCount))); + if (done.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + LocalPointer filteredRules( + new UVector(uprv_deleteUObject, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + + // Create initial rule + tzt.getTo()->getName(name); + LocalPointer res_initial( + new InitialTimeZoneRule(name, tzt.getTo()->getRawOffset(), tzt.getTo()->getDSTSavings()), status); + if (U_FAILURE(status)) { + return; + } + + // Mark rules which does not need to be processed + for (i = 0; i < ruleCount; i++) { + r = (TimeZoneRule*)orgRules->elementAt(i); + avail = r->getNextStart(start, res_initial->getRawOffset(), res_initial->getDSTSavings(), false, time); + done[i] = !avail; + } + + time = start; + while (!bFinalStd || !bFinalDst) { + avail = getNextTransition(time, false, tzt); + if (!avail) { + break; + } + UDate updatedTime = tzt.getTime(); + if (updatedTime == time) { + // Can get here if rules for start & end of daylight time have exactly + // the same time. + // TODO: fix getNextTransition() to prevent it? + status = U_INVALID_STATE_ERROR; + return; + } + time = updatedTime; + + const TimeZoneRule *toRule = tzt.getTo(); + for (i = 0; i < ruleCount; i++) { + r = (TimeZoneRule*)orgRules->elementAt(i); + if (*r == *toRule) { + break; + } + } + if (i >= ruleCount) { + // This case should never happen + status = U_INVALID_STATE_ERROR; + return; + } + if (done[i]) { + continue; + } + const TimeArrayTimeZoneRule *tar = dynamic_cast(toRule); + const AnnualTimeZoneRule *ar; + if (tar != nullptr) { + // Get the previous raw offset and DST savings before the very first start time + TimeZoneTransition tzt0; + t = start; + while (true) { + avail = getNextTransition(t, false, tzt0); + if (!avail) { + break; + } + if (*(tzt0.getTo()) == *tar) { + break; + } + t = tzt0.getTime(); + } + if (avail) { + // Check if the entire start times to be added + tar->getFirstStart(tzt.getFrom()->getRawOffset(), tzt.getFrom()->getDSTSavings(), firstStart); + if (firstStart > start) { + // Just add the rule as is + LocalPointer lpTar(tar->clone(), status); + filteredRules->adoptElement(lpTar.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } else { + // Collect transitions after the start time + int32_t startTimes; + DateTimeRule::TimeRuleType timeType; + int32_t idx; + + startTimes = tar->countStartTimes(); + timeType = tar->getTimeType(); + for (idx = 0; idx < startTimes; idx++) { + tar->getStartTimeAt(idx, t); + if (timeType == DateTimeRule::STANDARD_TIME) { + t -= tzt.getFrom()->getRawOffset(); + } + if (timeType == DateTimeRule::WALL_TIME) { + t -= tzt.getFrom()->getDSTSavings(); + } + if (t > start) { + break; + } + } + if (U_FAILURE(status)) { + return; + } + int32_t asize = startTimes - idx; + if (asize > 0) { + LocalMemory newTimes(static_cast(uprv_malloc(sizeof(UDate) * asize))); + if (newTimes.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (int32_t newidx = 0; newidx < asize; newidx++) { + tar->getStartTimeAt(idx + newidx, newTimes[newidx]); + } + tar->getName(name); + LocalPointer newTar(new TimeArrayTimeZoneRule( + name, tar->getRawOffset(), tar->getDSTSavings(), &newTimes[0], asize, timeType), status); + filteredRules->adoptElement(newTar.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } + } + } + } else if ((ar = dynamic_cast(toRule)) != nullptr) { + ar->getFirstStart(tzt.getFrom()->getRawOffset(), tzt.getFrom()->getDSTSavings(), firstStart); + if (firstStart == tzt.getTime()) { + // Just add the rule as is + LocalPointer arClone(ar->clone(), status); + filteredRules->adoptElement(arClone.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } else { + // Calculate the transition year + int32_t year, month, dom, dow, doy, mid; + Grego::timeToFields(tzt.getTime(), year, month, dom, dow, doy, mid); + // Re-create the rule + ar->getName(name); + LocalPointer newAr(new AnnualTimeZoneRule(name, ar->getRawOffset(), ar->getDSTSavings(), + *(ar->getRule()), year, ar->getEndYear()), status); + filteredRules->adoptElement(newAr.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } + // check if this is a final rule + if (ar->getEndYear() == AnnualTimeZoneRule::MAX_YEAR) { + // After bot final standard and dst rules are processed, + // exit this while loop. + if (ar->getDSTSavings() == 0) { + bFinalStd = true; + } else { + bFinalDst = true; + } + } + } + done[i] = true; + } + + // Set the results + initial = res_initial.orphan(); + transitionRules = filteredRules.orphan(); + return; +} + +void +BasicTimeZone::getOffsetFromLocal(UDate /*date*/, UTimeZoneLocalOption /*nonExistingTimeOpt*/, + UTimeZoneLocalOption /*duplicatedTimeOpt*/, + int32_t& /*rawOffset*/, int32_t& /*dstOffset*/, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + status = U_UNSUPPORTED_ERROR; +} + +void BasicTimeZone::getOffsetFromLocal(UDate date, int32_t nonExistingTimeOpt, int32_t duplicatedTimeOpt, + int32_t& rawOffset, int32_t& dstOffset, + UErrorCode& status) const { + getOffsetFromLocal(date, (UTimeZoneLocalOption)nonExistingTimeOpt, + (UTimeZoneLocalOption)duplicatedTimeOpt, rawOffset, dstOffset, status); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/bocsu.cpp b/intl/icu/source/i18n/bocsu.cpp new file mode 100644 index 0000000000..585415643b --- /dev/null +++ b/intl/icu/source/i18n/bocsu.cpp @@ -0,0 +1,144 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2001-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* file name: bocsu.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Author: Markus W. Scherer +* +* Modification history: +* 05/18/2001 weiv Made into separate module +*/ + + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/bytestream.h" +#include "unicode/utf16.h" +#include "bocsu.h" + +/* + * encode one difference value -0x10ffff..+0x10ffff in 1..4 bytes, + * preserving lexical order + */ +static uint8_t * +u_writeDiff(int32_t diff, uint8_t *p) { + if(diff>=SLOPE_REACH_NEG_1) { + if(diff<=SLOPE_REACH_POS_1) { + *p++=(uint8_t)(SLOPE_MIDDLE+diff); + } else if(diff<=SLOPE_REACH_POS_2) { + *p++=(uint8_t)(SLOPE_START_POS_2+(diff/SLOPE_TAIL_COUNT)); + *p++=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + } else if(diff<=SLOPE_REACH_POS_3) { + p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + diff/=SLOPE_TAIL_COUNT; + p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + *p=(uint8_t)(SLOPE_START_POS_3+(diff/SLOPE_TAIL_COUNT)); + p+=3; + } else { + p[3]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + diff/=SLOPE_TAIL_COUNT; + p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + diff/=SLOPE_TAIL_COUNT; + p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); + *p=SLOPE_MAX; + p+=4; + } + } else { + int32_t m; + + if(diff>=SLOPE_REACH_NEG_2) { + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + *p++=(uint8_t)(SLOPE_START_NEG_2+diff); + *p++=(uint8_t)(SLOPE_MIN+m); + } else if(diff>=SLOPE_REACH_NEG_3) { + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + p[2]=(uint8_t)(SLOPE_MIN+m); + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + p[1]=(uint8_t)(SLOPE_MIN+m); + *p=(uint8_t)(SLOPE_START_NEG_3+diff); + p+=3; + } else { + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + p[3]=(uint8_t)(SLOPE_MIN+m); + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + p[2]=(uint8_t)(SLOPE_MIN+m); + NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); + p[1]=(uint8_t)(SLOPE_MIN+m); + *p=SLOPE_MIN; + p+=4; + } + } + return p; +} + +/* + * Encode the code points of a string as + * a sequence of byte-encoded differences (slope detection), + * preserving lexical order. + * + * Optimize the difference-taking for runs of Unicode text within + * small scripts: + * + * Most small scripts are allocated within aligned 128-blocks of Unicode + * code points. Lexical order is preserved if "prev" is always moved + * into the middle of such a block. + * + * Additionally, "prev" is moved from anywhere in the Unihan + * area into the middle of that area. + * Note that the identical-level run in a sort key is generated from + * NFD text - there are never Hangul characters included. + */ +U_CFUNC UChar32 +u_writeIdenticalLevelRun(UChar32 prev, const char16_t *s, int32_t length, icu::ByteSink &sink) { + char scratch[64]; + int32_t capacity; + + int32_t i=0; + while(i=SLOPE_MAX_BYTES in case u_writeDiff() writes that much, + // but we do not want to force the sink.GetAppendBuffer() to allocate + // for a large min_capacity because we might actually only write one byte. + if(capacity<16) { + buffer=scratch; + capacity=(int32_t)sizeof(scratch); + } + p=reinterpret_cast(buffer); + uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES; + while(i=0xa000) { + prev=(prev&~0x7f)-SLOPE_REACH_NEG_1; + } else { + /* + * Unihan U+4e00..U+9fa5: + * double-bytes down from the upper end + */ + prev=0x9fff-SLOPE_REACH_POS_2; + } + + UChar32 c; + U16_NEXT(s, i, length, c); + if(c==0xfffe) { + *p++=2; // merge separator + prev=0; + } else { + p=u_writeDiff(c-prev, p); + prev=c; + } + } + sink.Append(buffer, (int32_t)(p-reinterpret_cast(buffer))); + } + return prev; +} + +#endif /* #if !UCONFIG_NO_COLLATION */ diff --git a/intl/icu/source/i18n/bocsu.h b/intl/icu/source/i18n/bocsu.h new file mode 100644 index 0000000000..631e29aa76 --- /dev/null +++ b/intl/icu/source/i18n/bocsu.h @@ -0,0 +1,161 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2001-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* file name: bocsu.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Author: Markus W. Scherer +* +* Modification history: +* 05/18/2001 weiv Made into separate module +*/ + +#ifndef BOCSU_H +#define BOCSU_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +U_NAMESPACE_BEGIN + +class ByteSink; + +U_NAMESPACE_END + +/* + * "BOCSU" + * Binary Ordered Compression Scheme for Unicode + * + * Specific application: + * + * Encode a Unicode string for the identical level of a sort key. + * Restrictions: + * - byte stream (unsigned 8-bit bytes) + * - lexical order of the identical-level run must be + * the same as code point order for the string + * - avoid byte values 0, 1, 2 + * + * Method: Slope Detection + * Remember the previous code point (initial 0). + * For each cp in the string, encode the difference to the previous one. + * + * With a compact encoding of differences, this yields good results for + * small scripts and UTF-like results otherwise. + * + * Encoding of differences: + * - Similar to a UTF, encoding the length of the byte sequence in the lead bytes. + * - Does not need to be friendly for decoding or random access + * (trail byte values may overlap with lead/single byte values). + * - The signedness must be encoded as the most significant part. + * + * We encode differences with few bytes if their absolute values are small. + * For correct ordering, we must treat the entire value range -10ffff..+10ffff + * in ascending order, which forbids encoding the sign and the absolute value separately. + * Instead, we split the lead byte range in the middle and encode non-negative values + * going up and negative values going down. + * + * For very small absolute values, the difference is added to a middle byte value + * for single-byte encoded differences. + * For somewhat larger absolute values, the difference is divided by the number + * of byte values available, the modulo is used for one trail byte, and the remainder + * is added to a lead byte avoiding the single-byte range. + * For large absolute values, the difference is similarly encoded in three bytes. + * + * This encoding does not use byte values 0, 1, 2, but uses all other byte values + * for lead/single bytes so that the middle range of single bytes is as large + * as possible. + * Note that the lead byte ranges overlap some, but that the sequences as a whole + * are well ordered. I.e., even if the lead byte is the same for sequences of different + * lengths, the trail bytes establish correct order. + * It would be possible to encode slightly larger ranges for each length (>1) by + * subtracting the lower bound of the range. However, that would also slow down the + * calculation. + * + * For the actual string encoding, an optimization moves the previous code point value + * to the middle of its Unicode script block to minimize the differences in + * same-script text runs. + */ + +/* Do not use byte values 0, 1, 2 because they are separators in sort keys. */ +#define SLOPE_MIN 3 +#define SLOPE_MAX 0xff +#define SLOPE_MIDDLE 0x81 + +#define SLOPE_TAIL_COUNT (SLOPE_MAX-SLOPE_MIN+1) + +#define SLOPE_MAX_BYTES 4 + +/* + * Number of lead bytes: + * 1 middle byte for 0 + * 2*80=160 single bytes for !=0 + * 2*42=84 for double-byte values + * 2*3=6 for 3-byte values + * 2*1=2 for 4-byte values + * + * The sum must be <=SLOPE_TAIL_COUNT. + * + * Why these numbers? + * - There should be >=128 single-byte values to cover 128-blocks + * with small scripts. + * - There should be >=20902 single/double-byte values to cover Unihan. + * - It helps CJK Extension B some if there are 3-byte values that cover + * the distance between them and Unihan. + * This also helps to jump among distant places in the BMP. + * - Four-byte values are necessary to cover the rest of Unicode. + * + * Symmetrical lead byte counts are for convenience. + * With an equal distribution of even and odd differences there is also + * no advantage to asymmetrical lead byte counts. + */ +#define SLOPE_SINGLE 80 +#define SLOPE_LEAD_2 42 +#define SLOPE_LEAD_3 3 +#define SLOPE_LEAD_4 1 + +/* The difference value range for single-byters. */ +#define SLOPE_REACH_POS_1 SLOPE_SINGLE +#define SLOPE_REACH_NEG_1 (-SLOPE_SINGLE) + +/* The difference value range for double-byters. */ +#define SLOPE_REACH_POS_2 (SLOPE_LEAD_2*SLOPE_TAIL_COUNT+(SLOPE_LEAD_2-1)) +#define SLOPE_REACH_NEG_2 (-SLOPE_REACH_POS_2-1) + +/* The difference value range for 3-byters. */ +#define SLOPE_REACH_POS_3 (SLOPE_LEAD_3*SLOPE_TAIL_COUNT*SLOPE_TAIL_COUNT+(SLOPE_LEAD_3-1)*SLOPE_TAIL_COUNT+(SLOPE_TAIL_COUNT-1)) +#define SLOPE_REACH_NEG_3 (-SLOPE_REACH_POS_3-1) + +/* The lead byte start values. */ +#define SLOPE_START_POS_2 (SLOPE_MIDDLE+SLOPE_SINGLE+1) +#define SLOPE_START_POS_3 (SLOPE_START_POS_2+SLOPE_LEAD_2) + +#define SLOPE_START_NEG_2 (SLOPE_MIDDLE+SLOPE_REACH_NEG_1) +#define SLOPE_START_NEG_3 (SLOPE_START_NEG_2-SLOPE_LEAD_2) + +/* + * Integer division and modulo with negative numerators + * yields negative modulo results and quotients that are one more than + * what we need here. + */ +#define NEGDIVMOD(n, d, m) UPRV_BLOCK_MACRO_BEGIN { \ + (m)=(n)%(d); \ + (n)/=(d); \ + if((m)<0) { \ + --(n); \ + (m)+=(d); \ + } \ +} UPRV_BLOCK_MACRO_END + +U_CFUNC UChar32 +u_writeIdenticalLevelRun(UChar32 prev, const UChar *s, int32_t length, icu::ByteSink &sink); + +#endif /* #if !UCONFIG_NO_COLLATION */ + +#endif diff --git a/intl/icu/source/i18n/brktrans.cpp b/intl/icu/source/i18n/brktrans.cpp new file mode 100644 index 0000000000..1ec0b2ad44 --- /dev/null +++ b/intl/icu/source/i18n/brktrans.cpp @@ -0,0 +1,195 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2008-2015, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 05/11/2008 Andy Heninger Port from Java +********************************************************************** +*/ + +#include + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION && !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/brkiter.h" +#include "unicode/localpointer.h" +#include "unicode/uchar.h" +#include "unicode/unifilt.h" +#include "unicode/uniset.h" + +#include "brktrans.h" +#include "cmemory.h" +#include "mutex.h" +#include "uprops.h" +#include "uinvchar.h" +#include "util.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(BreakTransliterator) + +static const char16_t SPACE = 32; // ' ' + + +/** + * Constructs a transliterator with the default delimiters '{' and + * '}'. + */ +BreakTransliterator::BreakTransliterator(UnicodeFilter* adoptedFilter) : + Transliterator(UNICODE_STRING("Any-BreakInternal", 17), adoptedFilter), + cachedBI(nullptr), cachedBoundaries(nullptr), fInsertion(SPACE) { + } + + +/** + * Destructor. + */ +BreakTransliterator::~BreakTransliterator() { +} + +/** + * Copy constructor. + */ +BreakTransliterator::BreakTransliterator(const BreakTransliterator& o) : + Transliterator(o), cachedBI(nullptr), cachedBoundaries(nullptr), fInsertion(o.fInsertion) { +} + + +/** + * Transliterator API. + */ +BreakTransliterator* BreakTransliterator::clone() const { + return new BreakTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void BreakTransliterator::handleTransliterate(Replaceable& text, UTransPosition& offsets, + UBool isIncremental ) const { + + UErrorCode status = U_ZERO_ERROR; + LocalPointer bi; + LocalPointer boundaries; + + { + Mutex m; + BreakTransliterator *nonConstThis = const_cast(this); + boundaries = std::move(nonConstThis->cachedBoundaries); + bi = std::move(nonConstThis->cachedBI); + } + if (bi.isNull()) { + bi.adoptInstead(BreakIterator::createWordInstance(Locale::getEnglish(), status)); + } + if (boundaries.isNull()) { + boundaries.adoptInstead(new UVector32(status)); + } + + if (bi.isNull() || boundaries.isNull() || U_FAILURE(status)) { + return; + } + + boundaries->removeAllElements(); + UnicodeString sText = replaceableAsString(text); + bi->setText(sText); + bi->preceding(offsets.start); + + // To make things much easier, we will stack the boundaries, and then insert at the end. + // generally, we won't need too many, since we will be filtered. + + int32_t boundary; + for(boundary = bi->next(); boundary != UBRK_DONE && boundary < offsets.limit; boundary = bi->next()) { + if (boundary == 0) continue; + // HACK: Check to see that preceding item was a letter + + UChar32 cp = sText.char32At(boundary-1); + int type = u_charType(cp); + //System.out.println(Integer.toString(cp,16) + " (before): " + type); + if ((U_MASK(type) & (U_GC_L_MASK | U_GC_M_MASK)) == 0) continue; + + cp = sText.char32At(boundary); + type = u_charType(cp); + //System.out.println(Integer.toString(cp,16) + " (after): " + type); + if ((U_MASK(type) & (U_GC_L_MASK | U_GC_M_MASK)) == 0) continue; + + boundaries->addElement(boundary, status); + // printf("Boundary at %d\n", boundary); + } + + int delta = 0; + int lastBoundary = 0; + + if (boundaries->size() != 0) { // if we found something, adjust + delta = boundaries->size() * fInsertion.length(); + lastBoundary = boundaries->lastElementi(); + + // we do this from the end backwards, so that we don't have to keep updating. + + while (boundaries->size() > 0) { + boundary = boundaries->popi(); + text.handleReplaceBetween(boundary, boundary, fInsertion); + } + } + + // Now fix up the return values + offsets.contextLimit += delta; + offsets.limit += delta; + offsets.start = isIncremental ? lastBoundary + delta : offsets.limit; + + // Return break iterator & boundaries vector to the cache. + { + Mutex m; + BreakTransliterator *nonConstThis = const_cast(this); + if (nonConstThis->cachedBI.isNull()) { + nonConstThis->cachedBI = std::move(bi); + } + if (nonConstThis->cachedBoundaries.isNull()) { + nonConstThis->cachedBoundaries = std::move(boundaries); + } + } + + // TODO: do something with U_FAILURE(status); + // (need to look at transliterators overall, not just here.) +} + +// +// getInsertion() +// +const UnicodeString &BreakTransliterator::getInsertion() const { + return fInsertion; +} + +// +// setInsertion() +// +void BreakTransliterator::setInsertion(const UnicodeString &insertion) { + this->fInsertion = insertion; +} + +// +// replaceableAsString Hack to let break iterators work +// on the replaceable text from transliterators. +// In practice, the only real Replaceable type that we +// will be seeing is UnicodeString, so this function +// will normally be efficient. +// +UnicodeString BreakTransliterator::replaceableAsString(Replaceable &r) { + UnicodeString s; + UnicodeString *rs = dynamic_cast(&r); + if (rs != nullptr) { + s = *rs; + } else { + r.extractBetween(0, r.length(), s); + } + return s; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/brktrans.h b/intl/icu/source/i18n/brktrans.h new file mode 100644 index 0000000000..5dcc8c50c0 --- /dev/null +++ b/intl/icu/source/i18n/brktrans.h @@ -0,0 +1,104 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2008-2015, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 05/11/2008 Andy Heninger Ported from Java +********************************************************************** +*/ +#ifndef BRKTRANS_H +#define BRKTRANS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION && !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/translit.h" + +#include "unicode/localpointer.h" + + +U_NAMESPACE_BEGIN + +class UVector32; + +/** + * A transliterator that pInserts the specified characters at word breaks. + * To restrict it to particular characters, use a filter. + * TODO: this is an internal class, and only temporary. + * Remove it once we have \b notation in Transliterator. + */ +class BreakTransliterator : public Transliterator { +public: + + /** + * Constructs a transliterator. + * @param adoptedFilter the filter for this transliterator. + */ + BreakTransliterator(UnicodeFilter* adoptedFilter = 0); + + /** + * Destructor. + */ + virtual ~BreakTransliterator(); + + /** + * Copy constructor. + */ + BreakTransliterator(const BreakTransliterator&); + + /** + * Transliterator API. + * @return A copy of the object. + */ + virtual BreakTransliterator* clone() const override; + + virtual const UnicodeString &getInsertion() const; + + virtual void setInsertion(const UnicodeString &insertion); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + protected: + + /** + * Implements {@link Transliterator#handleTransliterate}. + * @param text the buffer holding transliterated and + * untransliterated text + * @param offset the start and limit of the text, the position + * of the cursor, and the start and limit of transliteration. + * @param incremental if true, assume more text may be coming after + * pos.contextLimit. Otherwise, assume the text is complete. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + + private: + LocalPointer cachedBI; + LocalPointer cachedBoundaries; + UnicodeString fInsertion; + + static UnicodeString replaceableAsString(Replaceable &r); + + /** + * Assignment operator. + */ + BreakTransliterator& operator=(const BreakTransliterator&); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/buddhcal.cpp b/intl/icu/source/i18n/buddhcal.cpp new file mode 100644 index 0000000000..dc14af00bf --- /dev/null +++ b/intl/icu/source/i18n/buddhcal.cpp @@ -0,0 +1,174 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003-2013, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File BUDDHCAL.CPP +* +* Modification History: +* 05/13/2003 srl copied from gregocal.cpp +* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "buddhcal.h" +#include "unicode/gregocal.h" +#include "umutex.h" +#include + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(BuddhistCalendar) + +//static const int32_t kMaxEra = 0; // only 1 era + +static const int32_t kBuddhistEraStart = -543; // 544 BC (Gregorian) + +static const int32_t kGregorianEpoch = 1970; // used as the default value of EXTENDED_YEAR + +BuddhistCalendar::BuddhistCalendar(const Locale& aLocale, UErrorCode& success) +: GregorianCalendar(aLocale, success) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +BuddhistCalendar::~BuddhistCalendar() +{ +} + +BuddhistCalendar::BuddhistCalendar(const BuddhistCalendar& source) +: GregorianCalendar(source) +{ +} + +BuddhistCalendar& BuddhistCalendar::operator= ( const BuddhistCalendar& right) +{ + GregorianCalendar::operator=(right); + return *this; +} + +BuddhistCalendar* BuddhistCalendar::clone() const +{ + return new BuddhistCalendar(*this); +} + +const char *BuddhistCalendar::getType() const +{ + return "buddhist"; +} + +int32_t BuddhistCalendar::handleGetExtendedYear() +{ + // EXTENDED_YEAR in BuddhistCalendar is a Gregorian year. + // The default value of EXTENDED_YEAR is 1970 (Buddhist 2513) + int32_t year; + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, kGregorianEpoch); + } else { + // extended year is a gregorian year, where 1 = 1AD, 0 = 1BC, -1 = 2BC, etc + year = internalGet(UCAL_YEAR, kGregorianEpoch - kBuddhistEraStart) + + kBuddhistEraStart; + } + return year; +} + +void BuddhistCalendar::handleComputeFields(int32_t julianDay, UErrorCode& status) +{ + GregorianCalendar::handleComputeFields(julianDay, status); + int32_t y = internalGet(UCAL_EXTENDED_YEAR) - kBuddhistEraStart; + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, y); +} + +int32_t BuddhistCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const +{ + if(field == UCAL_ERA) { + return BE; + } else { + return GregorianCalendar::handleGetLimit(field,limitType); + } +} + +#if 0 +void BuddhistCalendar::timeToFields(UDate theTime, UBool quick, UErrorCode& status) +{ + //Calendar::timeToFields(theTime, quick, status); + + int32_t era = internalGet(UCAL_ERA); + int32_t year = internalGet(UCAL_YEAR); + + if(era == GregorianCalendar::BC) { + year = 1-year; + era = BuddhistCalendar::BE; + } else if(era == GregorianCalendar::AD) { + era = BuddhistCalendar::BE; + } else { + status = U_INTERNAL_PROGRAM_ERROR; + } + + year = year - kBuddhistEraStart; + + internalSet(UCAL_ERA, era); + internalSet(UCAL_YEAR, year); +} +#endif + +/** + * The system maintains a static default century start date. This is initialized + * the first time it is used. Once the system default century date and year + * are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gBCInitOnce {}; + + +UBool BuddhistCalendar::haveDefaultCentury() const +{ + return true; +} + +static void U_CALLCONV +initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + BuddhistCalendar calendar(Locale("@calendar=buddhist"),status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + UDate newStart = calendar.getTime(status); + int32_t newYear = calendar.get(UCAL_YEAR, status); + gSystemDefaultCenturyStartYear = newYear; + gSystemDefaultCenturyStart = newStart; + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate BuddhistCalendar::defaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart and systemDefaultCenturyStartYear + umtx_initOnce(gBCInitOnce, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t BuddhistCalendar::defaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStartYear and systemDefaultCenturyStart + umtx_initOnce(gBCInitOnce, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/buddhcal.h b/intl/icu/source/i18n/buddhcal.h new file mode 100644 index 0000000000..01b59341c1 --- /dev/null +++ b/intl/icu/source/i18n/buddhcal.h @@ -0,0 +1,187 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ******************************************************************************** + * Copyright (C) 2003-2013, International Business Machines Corporation + * and others. All Rights Reserved. + ******************************************************************************** + * + * File BUDDHCAL.H + * + * Modification History: + * + * Date Name Description + * 05/13/2003 srl copied from gregocal.h + ******************************************************************************** + */ + +#ifndef BUDDHCAL_H +#define BUDDHCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/gregocal.h" + +U_NAMESPACE_BEGIN + +/** + * Concrete class which provides the Buddhist calendar. + *

+ * BuddhistCalendar is a subclass of GregorianCalendar + * that numbers years since the birth of the Buddha. This is the civil calendar + * in some predominantly Buddhist countries such as Thailand, and it is used for + * religious purposes elsewhere. + *

+ * The Buddhist calendar is identical to the Gregorian calendar in all respects + * except for the year and era. Years are numbered since the birth of the + * Buddha in 543 BC (Gregorian), so that 1 AD (Gregorian) is equivalent to 544 + * BE (Buddhist Era) and 1998 AD is 2541 BE. + *

+ * The Buddhist Calendar has only one allowable era: BE. If the + * calendar is not in lenient mode (see setLenient), dates before + * 1/1/1 BE are rejected as an illegal argument. + *

+ * @internal + */ +class BuddhistCalendar : public GregorianCalendar { +public: + + /** + * Useful constants for BuddhistCalendar. Only one Era. + * @internal + */ + enum EEras { + BE + }; + + /** + * Constructs a BuddhistCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of BuddhistCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + BuddhistCalendar(const Locale& aLocale, UErrorCode& success); + + + /** + * Destructor + * @internal + */ + virtual ~BuddhistCalendar(); + + /** + * Copy constructor + * @param source the object to be copied. + * @internal + */ + BuddhistCalendar(const BuddhistCalendar& source); + + /** + * Default assignment operator + * @param right the object to be copied. + * @internal + */ + BuddhistCalendar& operator=(const BuddhistCalendar& right); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual BuddhistCalendar* clone() const override; + +public: + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "buddhist". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + +private: + BuddhistCalendar(); // default constructor not implemented + + protected: + /** + * Return the extended year defined by the current fields. This will + * use the UCAL_EXTENDED_YEAR field or the UCAL_YEAR and supra-year fields (such + * as UCAL_ERA) specific to the calendar system, depending on which set of + * fields is newer. + * @return the extended year + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + /** + * Subclasses may override this method to compute several fields + * specific to each calendar system. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode& status) override; + /** + * Subclass API for defining limits of different types. + * @param field one of the field numbers + * @param limitType one of MINIMUM, GREATEST_MINIMUM, + * LEAST_MAXIMUM, or MAXIMUM + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Returns true because the Buddhist Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif // _GREGOCAL +//eof + diff --git a/intl/icu/source/i18n/calendar.cpp b/intl/icu/source/i18n/calendar.cpp new file mode 100644 index 0000000000..72d5d10ed5 --- /dev/null +++ b/intl/icu/source/i18n/calendar.cpp @@ -0,0 +1,4077 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2016, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File CALENDAR.CPP +* +* Modification History: +* +* Date Name Description +* 02/03/97 clhuang Creation. +* 04/22/97 aliu Cleaned up, fixed memory leak, made +* setWeekCountData() more robust. +* Moved platform code to TPlatformUtilities. +* 05/01/97 aliu Made equals(), before(), after() arguments const. +* 05/20/97 aliu Changed logic of when to compute fields and time +* to fix bugs. +* 08/12/97 aliu Added equivalentTo. Misc other fixes. +* 07/28/98 stephen Sync up with JDK 1.2 +* 09/02/98 stephen Sync with JDK 1.2 8/31 build (getActualMin/Max) +* 03/17/99 stephen Changed adoptTimeZone() - now fAreFieldsSet is +* set to false to force update of time. +******************************************************************************* +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/gregocal.h" +#include "unicode/basictz.h" +#include "unicode/simpletz.h" +#include "unicode/rbtz.h" +#include "unicode/vtzone.h" +#include "gregoimp.h" +#include "buddhcal.h" +#include "taiwncal.h" +#include "japancal.h" +#include "islamcal.h" +#include "hebrwcal.h" +#include "persncal.h" +#include "indiancal.h" +#include "iso8601cal.h" +#include "chnsecal.h" +#include "coptccal.h" +#include "dangical.h" +#include "ethpccal.h" +#include "unicode/calendar.h" +#include "cpputils.h" +#include "servloc.h" +#include "ucln_in.h" +#include "cstring.h" +#include "locbased.h" +#include "uresimp.h" +#include "ustrenum.h" +#include "uassert.h" +#include "olsontz.h" +#include "sharedcalendar.h" +#include "unifiedcache.h" +#include "ulocimp.h" + +#if !UCONFIG_NO_SERVICE +static icu::ICULocaleService* gService = nullptr; +static icu::UInitOnce gServiceInitOnce {}; + +// INTERNAL - for cleanup +U_CDECL_BEGIN +static UBool calendar_cleanup() { +#if !UCONFIG_NO_SERVICE + if (gService) { + delete gService; + gService = nullptr; + } + gServiceInitOnce.reset(); +#endif + return true; +} +U_CDECL_END +#endif + +// ------------------------------------------ +// +// Registration +// +//------------------------------------------- +//#define U_DEBUG_CALSVC 1 +// + +#if defined( U_DEBUG_CALSVC ) || defined (U_DEBUG_CAL) + +/** + * fldName was removed as a duplicate implementation. + * use udbg_ services instead, + * which depend on include files and library from ../tools/toolutil, the following circular link: + * CPPFLAGS+=-I$(top_srcdir)/tools/toolutil + * LIBS+=$(LIBICUTOOLUTIL) + */ +#include "udbgutil.h" +#include + +/** +* convert a UCalendarDateFields into a string - for debugging +* @param f field enum +* @return static string to the field name +* @internal +*/ + +const char* fldName(UCalendarDateFields f) { + return udbg_enumName(UDBG_UCalendarDateFields, (int32_t)f); +} + +#if UCAL_DEBUG_DUMP +// from CalendarTest::calToStr - but doesn't modify contents. +void ucal_dump(const Calendar &cal) { + cal.dump(); +} + +void Calendar::dump() const { + int i; + fprintf(stderr, "@calendar=%s, timeset=%c, fieldset=%c, allfields=%c, virtualset=%c, t=%.2f", + getType(), fIsTimeSet?'y':'n', fAreFieldsSet?'y':'n', fAreAllFieldsSet?'y':'n', + fAreFieldsVirtuallySet?'y':'n', + fTime); + + // can add more things here: DST, zone, etc. + fprintf(stderr, "\n"); + for(i = 0;i U_I18N_API +const SharedCalendar *LocaleCacheKey::createObject( + const void * /*unusedCreationContext*/, UErrorCode &status) const { + if (U_FAILURE(status)) { + return nullptr; + } + Calendar *calendar = Calendar::makeInstance(fLoc, status); + if (U_FAILURE(status)) { + return nullptr; + } + SharedCalendar *shared = new SharedCalendar(calendar); + if (shared == nullptr) { + delete calendar; + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + shared->addRef(); + return shared; +} + +static ECalType getCalendarType(const char *s) { + for (int i = 0; gCalTypes[i] != nullptr; i++) { + if (uprv_stricmp(s, gCalTypes[i]) == 0) { + return (ECalType)i; + } + } + return CALTYPE_UNKNOWN; +} + +#if !UCONFIG_NO_SERVICE +// Only used with service registration. +static UBool isStandardSupportedKeyword(const char *keyword, UErrorCode& status) { + if(U_FAILURE(status)) { + return false; + } + ECalType calType = getCalendarType(keyword); + return (calType != CALTYPE_UNKNOWN); +} + +// only used with service registration. +static void getCalendarKeyword(const UnicodeString &id, char *targetBuffer, int32_t targetBufferSize) { + UnicodeString calendarKeyword = UNICODE_STRING_SIMPLE("calendar="); + int32_t calKeyLen = calendarKeyword.length(); + int32_t keyLen = 0; + + int32_t keywordIdx = id.indexOf((char16_t)0x003D); /* '=' */ + if (id[0] == 0x40/*'@'*/ + && id.compareBetween(1, keywordIdx+1, calendarKeyword, 0, calKeyLen) == 0) + { + keyLen = id.extract(keywordIdx+1, id.length(), targetBuffer, targetBufferSize, US_INV); + } + targetBuffer[keyLen] = 0; +} +#endif + +static ECalType getCalendarTypeForLocale(const char *locid) { + UErrorCode status = U_ZERO_ERROR; + ECalType calType = CALTYPE_UNKNOWN; + + //TODO: ULOC_FULL_NAME is out of date and too small.. + char canonicalName[256]; + + // Canonicalize, so that an old-style variant will be transformed to keywords. + // e.g ja_JP_TRADITIONAL -> ja_JP@calendar=japanese + // NOTE: Since ICU-20187, ja_JP_TRADITIONAL no longer canonicalizes, and + // the Gregorian calendar is returned instead. + int32_t canonicalLen = uloc_canonicalize(locid, canonicalName, sizeof(canonicalName) - 1, &status); + if (U_FAILURE(status)) { + return CALTYPE_GREGORIAN; + } + canonicalName[canonicalLen] = 0; // terminate + + char calTypeBuf[32]; + int32_t calTypeBufLen; + + calTypeBufLen = uloc_getKeywordValue(canonicalName, "calendar", calTypeBuf, sizeof(calTypeBuf) - 1, &status); + if (U_SUCCESS(status)) { + calTypeBuf[calTypeBufLen] = 0; + calType = getCalendarType(calTypeBuf); + if (calType != CALTYPE_UNKNOWN) { + return calType; + } + } + status = U_ZERO_ERROR; + + // when calendar keyword is not available or not supported, read supplementalData + // to get the default calendar type for the locale's region + char region[ULOC_COUNTRY_CAPACITY]; + (void)ulocimp_getRegionForSupplementalData(canonicalName, true, region, sizeof(region), &status); + if (U_FAILURE(status)) { + return CALTYPE_GREGORIAN; + } + + // Read preferred calendar values from supplementalData calendarPreference + UResourceBundle *rb = ures_openDirect(nullptr, "supplementalData", &status); + ures_getByKey(rb, "calendarPreferenceData", rb, &status); + UResourceBundle *order = ures_getByKey(rb, region, nullptr, &status); + if (status == U_MISSING_RESOURCE_ERROR && rb != nullptr) { + status = U_ZERO_ERROR; + order = ures_getByKey(rb, "001", nullptr, &status); + } + + calTypeBuf[0] = 0; + if (U_SUCCESS(status) && order != nullptr) { + // the first calendar type is the default for the region + int32_t len = 0; + const char16_t *uCalType = ures_getStringByIndex(order, 0, &len, &status); + if (len < (int32_t)sizeof(calTypeBuf)) { + u_UCharsToChars(uCalType, calTypeBuf, len); + *(calTypeBuf + len) = 0; // terminate; + calType = getCalendarType(calTypeBuf); + } + } + + ures_close(order); + ures_close(rb); + + if (calType == CALTYPE_UNKNOWN) { + // final fallback + calType = CALTYPE_GREGORIAN; + } + return calType; +} + +static Calendar *createStandardCalendar(ECalType calType, const Locale &loc, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer cal; + + switch (calType) { + case CALTYPE_GREGORIAN: + cal.adoptInsteadAndCheckErrorCode(new GregorianCalendar(loc, status), status); + break; + case CALTYPE_JAPANESE: + cal.adoptInsteadAndCheckErrorCode(new JapaneseCalendar(loc, status), status); + break; + case CALTYPE_BUDDHIST: + cal.adoptInsteadAndCheckErrorCode(new BuddhistCalendar(loc, status), status); + break; + case CALTYPE_ROC: + cal.adoptInsteadAndCheckErrorCode(new TaiwanCalendar(loc, status), status); + break; + case CALTYPE_PERSIAN: + cal.adoptInsteadAndCheckErrorCode(new PersianCalendar(loc, status), status); + break; + case CALTYPE_ISLAMIC_TBLA: + cal.adoptInsteadAndCheckErrorCode(new IslamicTBLACalendar(loc, status), status); + break; + case CALTYPE_ISLAMIC_CIVIL: + cal.adoptInsteadAndCheckErrorCode(new IslamicCivilCalendar(loc, status), status); + break; + case CALTYPE_ISLAMIC_RGSA: + cal.adoptInsteadAndCheckErrorCode(new IslamicRGSACalendar(loc, status), status); + break; + case CALTYPE_ISLAMIC: + cal.adoptInsteadAndCheckErrorCode(new IslamicCalendar(loc, status), status); + break; + case CALTYPE_ISLAMIC_UMALQURA: + cal.adoptInsteadAndCheckErrorCode(new IslamicUmalquraCalendar(loc, status), status); + break; + case CALTYPE_HEBREW: + cal.adoptInsteadAndCheckErrorCode(new HebrewCalendar(loc, status), status); + break; + case CALTYPE_CHINESE: + cal.adoptInsteadAndCheckErrorCode(new ChineseCalendar(loc, status), status); + break; + case CALTYPE_INDIAN: + cal.adoptInsteadAndCheckErrorCode(new IndianCalendar(loc, status), status); + break; + case CALTYPE_COPTIC: + cal.adoptInsteadAndCheckErrorCode(new CopticCalendar(loc, status), status); + break; + case CALTYPE_ETHIOPIC: + cal.adoptInsteadAndCheckErrorCode(new EthiopicCalendar(loc, status), status); + break; + case CALTYPE_ETHIOPIC_AMETE_ALEM: + cal.adoptInsteadAndCheckErrorCode(new EthiopicAmeteAlemCalendar(loc, status), status); + break; + case CALTYPE_ISO8601: + cal.adoptInsteadAndCheckErrorCode(new ISO8601Calendar(loc, status), status); + break; + case CALTYPE_DANGI: + cal.adoptInsteadAndCheckErrorCode(new DangiCalendar(loc, status), status); + break; + default: + status = U_UNSUPPORTED_ERROR; + } + return cal.orphan(); +} + + +#if !UCONFIG_NO_SERVICE + +// ------------------------------------- + +/** +* a Calendar Factory which creates the "basic" calendar types, that is, those +* shipped with ICU. +*/ +class BasicCalendarFactory : public LocaleKeyFactory { +public: + /** + * @param calendarType static const string (caller owns storage - will be aliased) to calendar type + */ + BasicCalendarFactory() + : LocaleKeyFactory(LocaleKeyFactory::INVISIBLE) { } + + virtual ~BasicCalendarFactory(); + +protected: + //virtual UBool isSupportedID( const UnicodeString& id, UErrorCode& status) const { + // if(U_FAILURE(status)) { + // return false; + // } + // char keyword[ULOC_FULLNAME_CAPACITY]; + // getCalendarKeyword(id, keyword, (int32_t)sizeof(keyword)); + // return isStandardSupportedKeyword(keyword, status); + //} + + virtual void updateVisibleIDs(Hashtable& result, UErrorCode& status) const override + { + if (U_SUCCESS(status)) { + for(int32_t i=0;gCalTypes[i] != nullptr;i++) { + UnicodeString id((char16_t)0x40); /* '@' a variant character */ + id.append(UNICODE_STRING_SIMPLE("calendar=")); + id.append(UnicodeString(gCalTypes[i], -1, US_INV)); + result.put(id, (void*)this, status); + } + } + } + + virtual UObject* create(const ICUServiceKey& key, const ICUService* /*service*/, UErrorCode& status) const override { + if (U_FAILURE(status)) { + return nullptr; + } +#ifdef U_DEBUG_CALSVC + if(dynamic_cast(&key) == nullptr) { + fprintf(stderr, "::create - not a LocaleKey!\n"); + } +#endif + const LocaleKey* lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + Locale curLoc; // current locale + Locale canLoc; // Canonical locale + + lkey->currentLocale(curLoc); + lkey->canonicalLocale(canLoc); + + char keyword[ULOC_FULLNAME_CAPACITY]; + UnicodeString str; + + key.currentID(str); + getCalendarKeyword(str, keyword, (int32_t) sizeof(keyword)); + +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "BasicCalendarFactory::create() - cur %s, can %s\n", (const char*)curLoc.getName(), (const char*)canLoc.getName()); +#endif + + if(!isStandardSupportedKeyword(keyword,status)) { // Do we handle this type? +#ifdef U_DEBUG_CALSVC + + fprintf(stderr, "BasicCalendarFactory - not handling %s.[%s]\n", (const char*) curLoc.getName(), tmp ); +#endif + return nullptr; + } + + return createStandardCalendar(getCalendarType(keyword), canLoc, status); + } +}; + +BasicCalendarFactory::~BasicCalendarFactory() {} + +/** +* A factory which looks up the DefaultCalendar resource to determine which class of calendar to use +*/ + +class DefaultCalendarFactory : public ICUResourceBundleFactory { +public: + DefaultCalendarFactory() : ICUResourceBundleFactory() { } + virtual ~DefaultCalendarFactory(); +protected: + virtual UObject* create(const ICUServiceKey& key, const ICUService* /*service*/, UErrorCode& status) const override { + if (U_FAILURE(status)) { + return nullptr; + } + + const LocaleKey *lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + Locale loc; + lkey->currentLocale(loc); + + UnicodeString *ret = new UnicodeString(); + if (ret == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + ret->append((char16_t)0x40); // '@' is a variant character + ret->append(UNICODE_STRING("calendar=", 9)); + ret->append(UnicodeString(gCalTypes[getCalendarTypeForLocale(loc.getName())], -1, US_INV)); + } + return ret; + } +}; + +DefaultCalendarFactory::~DefaultCalendarFactory() {} + +// ------------------------------------- +class CalendarService : public ICULocaleService { +public: + CalendarService() + : ICULocaleService(UNICODE_STRING_SIMPLE("Calendar")) + { + UErrorCode status = U_ZERO_ERROR; + registerFactory(new DefaultCalendarFactory(), status); + } + + virtual ~CalendarService(); + + virtual UObject* cloneInstance(UObject* instance) const override { + UnicodeString *s = dynamic_cast(instance); + if(s != nullptr) { + return s->clone(); + } else { +#ifdef U_DEBUG_CALSVC_F + UErrorCode status2 = U_ZERO_ERROR; + fprintf(stderr, "Cloning a %s calendar with tz=%ld\n", ((Calendar*)instance)->getType(), ((Calendar*)instance)->get(UCAL_ZONE_OFFSET, status2)); +#endif + return ((Calendar*)instance)->clone(); + } + } + + virtual UObject* handleDefault(const ICUServiceKey& key, UnicodeString* /*actualID*/, UErrorCode& status) const override { + if (U_FAILURE(status)) { + return nullptr; + } + LocaleKey& lkey = static_cast(const_cast(key)); + //int32_t kind = lkey.kind(); + + Locale loc; + lkey.canonicalLocale(loc); + +#ifdef U_DEBUG_CALSVC + Locale loc2; + lkey.currentLocale(loc2); + fprintf(stderr, "CalSvc:handleDefault for currentLoc %s, canloc %s\n", (const char*)loc.getName(), (const char*)loc2.getName()); +#endif + Calendar *nc = new GregorianCalendar(loc, status); + if (nc == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nc; + } + +#ifdef U_DEBUG_CALSVC + UErrorCode status2 = U_ZERO_ERROR; + fprintf(stderr, "New default calendar has tz=%d\n", ((Calendar*)nc)->get(UCAL_ZONE_OFFSET, status2)); +#endif + return nc; + } + + virtual UBool isDefault() const override { + return countFactories() == 1; + } +}; + +CalendarService::~CalendarService() {} + +// ------------------------------------- + +static inline UBool +isCalendarServiceUsed() { + return !gServiceInitOnce.isReset(); +} + +// ------------------------------------- + +static void U_CALLCONV +initCalendarService(UErrorCode &status) +{ +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "Spinning up Calendar Service\n"); +#endif + if (U_FAILURE(status)) { + return; + } + ucln_i18n_registerCleanup(UCLN_I18N_CALENDAR, calendar_cleanup); + gService = new CalendarService(); + if (gService == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "Registering classes..\n"); +#endif + + // Register all basic instances. + gService->registerFactory(new BasicCalendarFactory(),status); + +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "Done..\n"); +#endif + + if(U_FAILURE(status)) { +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "err (%s) registering classes, deleting service.....\n", u_errorName(status)); +#endif + delete gService; + gService = nullptr; + } + } + +static ICULocaleService* +getCalendarService(UErrorCode &status) +{ + umtx_initOnce(gServiceInitOnce, &initCalendarService, status); + return gService; +} + +URegistryKey Calendar::registerFactory(ICUServiceFactory* toAdopt, UErrorCode& status) +{ + return getCalendarService(status)->registerFactory(toAdopt, status); +} + +UBool Calendar::unregister(URegistryKey key, UErrorCode& status) { + return getCalendarService(status)->unregister(key, status); +} +#endif /* UCONFIG_NO_SERVICE */ + +// ------------------------------------- + +static const int32_t kCalendarLimits[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest min Least max Greatest max + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // ERA + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // YEAR + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // MONTH + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // WEEK_OF_YEAR + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // WEEK_OF_MONTH + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // DAY_OF_MONTH + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // DAY_OF_YEAR + { 1, 1, 7, 7 }, // DAY_OF_WEEK + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // DAY_OF_WEEK_IN_MONTH + { 0, 0, 1, 1 }, // AM_PM + { 0, 0, 11, 11 }, // HOUR + { 0, 0, 23, 23 }, // HOUR_OF_DAY + { 0, 0, 59, 59 }, // MINUTE + { 0, 0, 59, 59 }, // SECOND + { 0, 0, 999, 999 }, // MILLISECOND + {-16*kOneHour, -16*kOneHour, 12*kOneHour, 30*kOneHour }, // ZONE_OFFSET + { -1*kOneHour, -1*kOneHour, 2*kOneHour, 2*kOneHour }, // DST_OFFSET + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // YEAR_WOY + { 1, 1, 7, 7 }, // DOW_LOCAL + {/*N/A*/-1, /*N/A*/-1, /*N/A*/-1, /*N/A*/-1}, // EXTENDED_YEAR + { -0x7F000000, -0x7F000000, 0x7F000000, 0x7F000000 }, // JULIAN_DAY + { 0, 0, 24*kOneHour-1, 24*kOneHour-1 }, // MILLISECONDS_IN_DAY + { 0, 0, 1, 1 }, // IS_LEAP_MONTH + { 0, 0, 11, 11 } // ORDINAL_MONTH +}; + +// Resource bundle tags read by this class +static const char gCalendar[] = "calendar"; +static const char gMonthNames[] = "monthNames"; +static const char gGregorian[] = "gregorian"; + +// Data flow in Calendar +// --------------------- + +// The current time is represented in two ways by Calendar: as UTC +// milliseconds from the epoch start (1 January 1970 0:00 UTC), and as local +// fields such as MONTH, HOUR, AM_PM, etc. It is possible to compute the +// millis from the fields, and vice versa. The data needed to do this +// conversion is encapsulated by a TimeZone object owned by the Calendar. +// The data provided by the TimeZone object may also be overridden if the +// user sets the ZONE_OFFSET and/or DST_OFFSET fields directly. The class +// keeps track of what information was most recently set by the caller, and +// uses that to compute any other information as needed. + +// If the user sets the fields using set(), the data flow is as follows. +// This is implemented by the Calendar subclass's computeTime() method. +// During this process, certain fields may be ignored. The disambiguation +// algorithm for resolving which fields to pay attention to is described +// above. + +// local fields (YEAR, MONTH, DATE, HOUR, MINUTE, etc.) +// | +// | Using Calendar-specific algorithm +// V +// local standard millis +// | +// | Using TimeZone or user-set ZONE_OFFSET / DST_OFFSET +// V +// UTC millis (in time data member) + +// If the user sets the UTC millis using setTime(), the data flow is as +// follows. This is implemented by the Calendar subclass's computeFields() +// method. + +// UTC millis (in time data member) +// | +// | Using TimeZone getOffset() +// V +// local standard millis +// | +// | Using Calendar-specific algorithm +// V +// local fields (YEAR, MONTH, DATE, HOUR, MINUTE, etc.) + +// In general, a round trip from fields, through local and UTC millis, and +// back out to fields is made when necessary. This is implemented by the +// complete() method. Resolving a partial set of fields into a UTC millis +// value allows all remaining fields to be generated from that value. If +// the Calendar is lenient, the fields are also renormalized to standard +// ranges when they are regenerated. + +// ------------------------------------- + +Calendar::Calendar(UErrorCode& success) +: UObject(), +fIsTimeSet(false), +fAreFieldsSet(false), +fAreAllFieldsSet(false), +fAreFieldsVirtuallySet(false), +fNextStamp((int32_t)kMinimumUserStamp), +fTime(0), +fLenient(true), +fZone(nullptr), +fRepeatedWallTime(UCAL_WALLTIME_LAST), +fSkippedWallTime(UCAL_WALLTIME_LAST) +{ + validLocale[0] = 0; + actualLocale[0] = 0; + clear(); + if (U_FAILURE(success)) { + return; + } + fZone = TimeZone::createDefault(); + if (fZone == nullptr) { + success = U_MEMORY_ALLOCATION_ERROR; + } + setWeekData(Locale::getDefault(), nullptr, success); +} + +// ------------------------------------- + +Calendar::Calendar(TimeZone* zone, const Locale& aLocale, UErrorCode& success) +: UObject(), +fIsTimeSet(false), +fAreFieldsSet(false), +fAreAllFieldsSet(false), +fAreFieldsVirtuallySet(false), +fNextStamp((int32_t)kMinimumUserStamp), +fTime(0), +fLenient(true), +fZone(nullptr), +fRepeatedWallTime(UCAL_WALLTIME_LAST), +fSkippedWallTime(UCAL_WALLTIME_LAST) +{ + validLocale[0] = 0; + actualLocale[0] = 0; + if (U_FAILURE(success)) { + delete zone; + return; + } + if(zone == 0) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because timezone cannot be 0\n", + __FILE__, __LINE__); +#endif + success = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + clear(); + fZone = zone; + setWeekData(aLocale, nullptr, success); +} + +// ------------------------------------- + +Calendar::Calendar(const TimeZone& zone, const Locale& aLocale, UErrorCode& success) +: UObject(), +fIsTimeSet(false), +fAreFieldsSet(false), +fAreAllFieldsSet(false), +fAreFieldsVirtuallySet(false), +fNextStamp((int32_t)kMinimumUserStamp), +fTime(0), +fLenient(true), +fZone(nullptr), +fRepeatedWallTime(UCAL_WALLTIME_LAST), +fSkippedWallTime(UCAL_WALLTIME_LAST) +{ + validLocale[0] = 0; + actualLocale[0] = 0; + if (U_FAILURE(success)) { + return; + } + clear(); + fZone = zone.clone(); + if (fZone == nullptr) { + success = U_MEMORY_ALLOCATION_ERROR; + } + setWeekData(aLocale, nullptr, success); +} + +// ------------------------------------- + +Calendar::~Calendar() +{ + delete fZone; +} + +// ------------------------------------- + +Calendar::Calendar(const Calendar &source) +: UObject(source) +{ + fZone = nullptr; + *this = source; +} + +// ------------------------------------- + +Calendar & +Calendar::operator=(const Calendar &right) +{ + if (this != &right) { + uprv_arrayCopy(right.fFields, fFields, UCAL_FIELD_COUNT); + uprv_arrayCopy(right.fIsSet, fIsSet, UCAL_FIELD_COUNT); + uprv_arrayCopy(right.fStamp, fStamp, UCAL_FIELD_COUNT); + fTime = right.fTime; + fIsTimeSet = right.fIsTimeSet; + fAreAllFieldsSet = right.fAreAllFieldsSet; + fAreFieldsSet = right.fAreFieldsSet; + fAreFieldsVirtuallySet = right.fAreFieldsVirtuallySet; + fLenient = right.fLenient; + fRepeatedWallTime = right.fRepeatedWallTime; + fSkippedWallTime = right.fSkippedWallTime; + delete fZone; + fZone = nullptr; + if (right.fZone != nullptr) { + fZone = right.fZone->clone(); + } + fFirstDayOfWeek = right.fFirstDayOfWeek; + fMinimalDaysInFirstWeek = right.fMinimalDaysInFirstWeek; + fWeekendOnset = right.fWeekendOnset; + fWeekendOnsetMillis = right.fWeekendOnsetMillis; + fWeekendCease = right.fWeekendCease; + fWeekendCeaseMillis = right.fWeekendCeaseMillis; + fNextStamp = right.fNextStamp; + uprv_strncpy(validLocale, right.validLocale, sizeof(validLocale)); + uprv_strncpy(actualLocale, right.actualLocale, sizeof(actualLocale)); + validLocale[sizeof(validLocale)-1] = 0; + actualLocale[sizeof(validLocale)-1] = 0; + } + + return *this; +} + +// ------------------------------------- + +Calendar* U_EXPORT2 +Calendar::createInstance(UErrorCode& success) +{ + return createInstance(TimeZone::createDefault(), Locale::getDefault(), success); +} + +// ------------------------------------- + +Calendar* U_EXPORT2 +Calendar::createInstance(const TimeZone& zone, UErrorCode& success) +{ + return createInstance(zone, Locale::getDefault(), success); +} + +// ------------------------------------- + +Calendar* U_EXPORT2 +Calendar::createInstance(const Locale& aLocale, UErrorCode& success) +{ + return createInstance(TimeZone::forLocaleOrDefault(aLocale), aLocale, success); +} + +// ------------------------------------- Adopting + +// Note: this is the bottleneck that actually calls the service routines. + +Calendar * U_EXPORT2 +Calendar::makeInstance(const Locale& aLocale, UErrorCode& success) { + if (U_FAILURE(success)) { + return nullptr; + } + + Locale actualLoc; + UObject* u = nullptr; + +#if !UCONFIG_NO_SERVICE + if (isCalendarServiceUsed()) { + u = getCalendarService(success)->get(aLocale, LocaleKey::KIND_ANY, &actualLoc, success); + } + else +#endif + { + u = createStandardCalendar(getCalendarTypeForLocale(aLocale.getName()), aLocale, success); + } + Calendar* c = nullptr; + + if(U_FAILURE(success) || !u) { + if(U_SUCCESS(success)) { // Propagate some kind of err + success = U_INTERNAL_PROGRAM_ERROR; + } + return nullptr; + } + +#if !UCONFIG_NO_SERVICE + const UnicodeString* str = dynamic_cast(u); + if(str != nullptr) { + // It's a unicode string telling us what type of calendar to load ("gregorian", etc) + // Create a Locale over this string + Locale l(""); + LocaleUtility::initLocaleFromName(*str, l); + +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "Calendar::createInstance(%s), looking up [%s]\n", aLocale.getName(), l.getName()); +#endif + + Locale actualLoc2; + delete u; + u = nullptr; + + // Don't overwrite actualLoc, since the actual loc from this call + // may be something like "@calendar=gregorian" -- TODO investigate + // further... + c = (Calendar*)getCalendarService(success)->get(l, LocaleKey::KIND_ANY, &actualLoc2, success); + + if(U_FAILURE(success) || !c) { + if(U_SUCCESS(success)) { + success = U_INTERNAL_PROGRAM_ERROR; // Propagate some err + } + return nullptr; + } + + str = dynamic_cast(c); + if(str != nullptr) { + // recursed! Second lookup returned a UnicodeString. + // Perhaps DefaultCalendar{} was set to another locale. +#ifdef U_DEBUG_CALSVC + char tmp[200]; + // Extract a char* out of it.. + int32_t len = str->length(); + int32_t actLen = sizeof(tmp)-1; + if(len > actLen) { + len = actLen; + } + str->extract(0,len,tmp); + tmp[len]=0; + + fprintf(stderr, "err - recursed, 2nd lookup was unistring %s\n", tmp); +#endif + success = U_MISSING_RESOURCE_ERROR; // requested a calendar type which could NOT be found. + delete c; + return nullptr; + } +#ifdef U_DEBUG_CALSVC + fprintf(stderr, "%p: setting week count data to locale %s, actual locale %s\n", c, (const char*)aLocale.getName(), (const char *)actualLoc.getName()); +#endif + c->setWeekData(aLocale, c->getType(), success); // set the correct locale (this was an indirect calendar) + + char keyword[ULOC_FULLNAME_CAPACITY] = ""; + UErrorCode tmpStatus = U_ZERO_ERROR; + l.getKeywordValue("calendar", keyword, ULOC_FULLNAME_CAPACITY, tmpStatus); + if (U_SUCCESS(tmpStatus) && uprv_strcmp(keyword, "iso8601") == 0) { + c->setFirstDayOfWeek(UCAL_MONDAY); + c->setMinimalDaysInFirstWeek(4); + } + } + else +#endif /* UCONFIG_NO_SERVICE */ + { + // a calendar was returned - we assume the factory did the right thing. + c = (Calendar*)u; + } + + return c; +} + +Calendar* U_EXPORT2 +Calendar::createInstance(TimeZone* zone, const Locale& aLocale, UErrorCode& success) +{ + LocalPointer zonePtr(zone); + const SharedCalendar *shared = nullptr; + UnifiedCache::getByLocale(aLocale, shared, success); + if (U_FAILURE(success)) { + return nullptr; + } + Calendar *c = (*shared)->clone(); + shared->removeRef(); + if (c == nullptr) { + success = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + + // Now, reset calendar to default state: + c->adoptTimeZone(zonePtr.orphan()); // Set the correct time zone + c->setTimeInMillis(getNow(), success); // let the new calendar have the current time. + + return c; +} + +// ------------------------------------- + +Calendar* U_EXPORT2 +Calendar::createInstance(const TimeZone& zone, const Locale& aLocale, UErrorCode& success) +{ + Calendar* c = createInstance(aLocale, success); + if(U_SUCCESS(success) && c) { + c->setTimeZone(zone); + } + return c; +} + +// ------------------------------------- + +void U_EXPORT2 +Calendar::getCalendarTypeFromLocale( + const Locale &aLocale, + char *typeBuffer, + int32_t typeBufferSize, + UErrorCode &success) { + const SharedCalendar *shared = nullptr; + UnifiedCache::getByLocale(aLocale, shared, success); + if (U_FAILURE(success)) { + return; + } + uprv_strncpy(typeBuffer, (*shared)->getType(), typeBufferSize); + shared->removeRef(); + if (typeBuffer[typeBufferSize - 1]) { + success = U_BUFFER_OVERFLOW_ERROR; + } +} + +bool +Calendar::operator==(const Calendar& that) const +{ + UErrorCode status = U_ZERO_ERROR; + return isEquivalentTo(that) && + getTimeInMillis(status) == that.getTimeInMillis(status) && + U_SUCCESS(status); +} + +UBool +Calendar::isEquivalentTo(const Calendar& other) const +{ + return typeid(*this) == typeid(other) && + fLenient == other.fLenient && + fRepeatedWallTime == other.fRepeatedWallTime && + fSkippedWallTime == other.fSkippedWallTime && + fFirstDayOfWeek == other.fFirstDayOfWeek && + fMinimalDaysInFirstWeek == other.fMinimalDaysInFirstWeek && + fWeekendOnset == other.fWeekendOnset && + fWeekendOnsetMillis == other.fWeekendOnsetMillis && + fWeekendCease == other.fWeekendCease && + fWeekendCeaseMillis == other.fWeekendCeaseMillis && + *fZone == *other.fZone; +} + +// ------------------------------------- + +UBool +Calendar::equals(const Calendar& when, UErrorCode& status) const +{ + return (this == &when || + getTime(status) == when.getTime(status)); +} + +// ------------------------------------- + +UBool +Calendar::before(const Calendar& when, UErrorCode& status) const +{ + return (this != &when && + getTimeInMillis(status) < when.getTimeInMillis(status)); +} + +// ------------------------------------- + +UBool +Calendar::after(const Calendar& when, UErrorCode& status) const +{ + return (this != &when && + getTimeInMillis(status) > when.getTimeInMillis(status)); +} + +// ------------------------------------- + + +const Locale* U_EXPORT2 +Calendar::getAvailableLocales(int32_t& count) +{ + return Locale::getAvailableLocales(count); +} + +// ------------------------------------- + +StringEnumeration* U_EXPORT2 +Calendar::getKeywordValuesForLocale(const char* key, + const Locale& locale, UBool commonlyUsed, UErrorCode& status) +{ + // This is a wrapper over ucal_getKeywordValuesForLocale + UEnumeration *uenum = ucal_getKeywordValuesForLocale(key, locale.getName(), + commonlyUsed, &status); + if (U_FAILURE(status)) { + uenum_close(uenum); + return nullptr; + } + UStringEnumeration* ustringenum = new UStringEnumeration(uenum); + if (ustringenum == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return ustringenum; +} + +// ------------------------------------- + +UDate U_EXPORT2 +Calendar::getNow() +{ + return uprv_getUTCtime(); // return as milliseconds +} + +// ------------------------------------- + +/** +* Gets this Calendar's current time as a long. +* @return the current time as UTC milliseconds from the epoch. +*/ +double +Calendar::getTimeInMillis(UErrorCode& status) const +{ + if(U_FAILURE(status)) + return 0.0; + + if ( ! fIsTimeSet) + ((Calendar*)this)->updateTime(status); + + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return 0.0; + } + return fTime; +} + +// ------------------------------------- + +/** +* Sets this Calendar's current time from the given long value. +* A status of U_ILLEGAL_ARGUMENT_ERROR is set when millis is +* outside the range permitted by a Calendar object when not in lenient mode. +* when in lenient mode the out of range values are pinned to their respective min/max. +* @param date the new time in UTC milliseconds from the epoch. +*/ +void +Calendar::setTimeInMillis( double millis, UErrorCode& status ) { + if(U_FAILURE(status)) + return; + + if (millis > MAX_MILLIS) { + if(isLenient()) { + millis = MAX_MILLIS; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + } else if (millis < MIN_MILLIS) { + if(isLenient()) { + millis = MIN_MILLIS; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + } + + fTime = millis; + fAreFieldsSet = fAreAllFieldsSet = false; + fIsTimeSet = fAreFieldsVirtuallySet = true; + + for (int32_t i=0; icomplete(status); // Cast away const + return U_SUCCESS(status) ? fFields[field] : 0; +} + +// ------------------------------------- + +void +Calendar::set(UCalendarDateFields field, int32_t value) +{ + if (fAreFieldsVirtuallySet) { + UErrorCode ec = U_ZERO_ERROR; + computeFields(ec); + } + fFields[field] = value; + /* Ensure that the fNextStamp value doesn't go pass max value for int32_t */ + if (fNextStamp == STAMP_MAX) { + recalculateStamp(); + } + fStamp[field] = fNextStamp++; + fIsSet[field] = true; // Remove later + fIsTimeSet = fAreFieldsSet = fAreFieldsVirtuallySet = false; +} + +// ------------------------------------- + +void +Calendar::set(int32_t year, int32_t month, int32_t date) +{ + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); +} + +// ------------------------------------- + +void +Calendar::set(int32_t year, int32_t month, int32_t date, int32_t hour, int32_t minute) +{ + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); + set(UCAL_HOUR_OF_DAY, hour); + set(UCAL_MINUTE, minute); +} + +// ------------------------------------- + +void +Calendar::set(int32_t year, int32_t month, int32_t date, int32_t hour, int32_t minute, int32_t second) +{ + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); + set(UCAL_HOUR_OF_DAY, hour); + set(UCAL_MINUTE, minute); + set(UCAL_SECOND, second); +} + +// ------------------------------------- +int32_t Calendar::getRelatedYear(UErrorCode &status) const +{ + return get(UCAL_EXTENDED_YEAR, status); +} + +// ------------------------------------- +void Calendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year); +} + +// ------------------------------------- + +void +Calendar::clear() +{ + for (int32_t i=0; i bestStamp) { + bestStamp = fStamp[i]; + } + } + return bestStamp; +} + + +// ------------------------------------- + +void +Calendar::complete(UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + if (!fIsTimeSet) { + updateTime(status); + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return; + } + } + if (!fAreFieldsSet) { + computeFields(status); // fills in unset fields + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return; + } + fAreFieldsSet = true; + fAreAllFieldsSet = true; + } +} + +//------------------------------------------------------------------------- +// Protected utility methods for use by subclasses. These are very handy +// for implementing add, roll, and computeFields. +//------------------------------------------------------------------------- + +/** +* Adjust the specified field so that it is within +* the allowable range for the date to which this calendar is set. +* For example, in a Gregorian calendar pinning the {@link #DAY_OF_MONTH DAY_OF_MONTH} +* field for a calendar set to April 31 would cause it to be set +* to April 30. +*

+* Subclassing: +*
+* This utility method is intended for use by subclasses that need to implement +* their own overrides of {@link #roll roll} and {@link #add add}. +*

+* Note: +* pinField is implemented in terms of +* {@link #getActualMinimum getActualMinimum} +* and {@link #getActualMaximum getActualMaximum}. If either of those methods uses +* a slow, iterative algorithm for a particular field, it would be +* unwise to attempt to call pinField for that field. If you +* really do need to do so, you should override this method to do +* something more efficient for that field. +*

+* @param field The calendar field whose value should be pinned. +* +* @see #getActualMinimum +* @see #getActualMaximum +* @stable ICU 2.0 +*/ +void Calendar::pinField(UCalendarDateFields field, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + int32_t max = getActualMaximum(field, status); + int32_t min = getActualMinimum(field, status); + + if (fFields[field] > max) { + set(field, max); + } else if (fFields[field] < min) { + set(field, min); + } +} + + +void Calendar::computeFields(UErrorCode &ec) +{ + if (U_FAILURE(ec)) { + return; + } + // Compute local wall millis + double localMillis = internalGetTime(); + int32_t rawOffset, dstOffset; + getTimeZone().getOffset(localMillis, false, rawOffset, dstOffset, ec); + if (U_FAILURE(ec)) { + return; + } + localMillis += (rawOffset + dstOffset); + + // Mark fields as set. Do this before calling handleComputeFields(). + uint32_t mask = //fInternalSetMask; + (1 << UCAL_ERA) | + (1 << UCAL_YEAR) | + (1 << UCAL_MONTH) | + (1 << UCAL_DAY_OF_MONTH) | // = UCAL_DATE + (1 << UCAL_DAY_OF_YEAR) | + (1 << UCAL_EXTENDED_YEAR) | + (1 << UCAL_ORDINAL_MONTH); + + for (int32_t i=0; i>= 1; + } + + // We used to check for and correct extreme millis values (near + // Long.MIN_VALUE or Long.MAX_VALUE) here. Such values would cause + // overflows from positive to negative (or vice versa) and had to + // be manually tweaked. We no longer need to do this because we + // have limited the range of supported dates to those that have a + // Julian day that fits into an int. This allows us to implement a + // JULIAN_DAY field and also removes some inelegant code. - Liu + // 11/6/00 + + int32_t millisInDay; + int32_t days = ClockMath::floorDivide(localMillis, kOneDay, &millisInDay); + + internalSet(UCAL_JULIAN_DAY,days + kEpochStartAsJulianDay); + +#if defined (U_DEBUG_CAL) + //fprintf(stderr, "%s:%d- Hmm! Jules @ %d, as per %.0lf millis\n", + //__FILE__, __LINE__, fFields[UCAL_JULIAN_DAY], localMillis); +#endif + + computeGregorianAndDOWFields(fFields[UCAL_JULIAN_DAY], ec); + + // Call framework method to have subclass compute its fields. + // These must include, at a minimum, MONTH, DAY_OF_MONTH, + // EXTENDED_YEAR, YEAR, DAY_OF_YEAR. This method will call internalSet(), + // which will update stamp[]. + handleComputeFields(fFields[UCAL_JULIAN_DAY], ec); + + // Compute week-related fields, based on the subclass-computed + // fields computed by handleComputeFields(). + computeWeekFields(ec); + + // Compute time-related fields. These are independent of the date and + // of the subclass algorithm. They depend only on the local zone + // wall milliseconds in day. + if (U_FAILURE(ec)) { + return; + } + + fFields[UCAL_MILLISECONDS_IN_DAY] = millisInDay; + U_ASSERT(getMinimum(UCAL_MILLISECONDS_IN_DAY) <= + fFields[UCAL_MILLISECONDS_IN_DAY]); + U_ASSERT(fFields[UCAL_MILLISECONDS_IN_DAY] <= + getMaximum(UCAL_MILLISECONDS_IN_DAY)); + + fFields[UCAL_MILLISECOND] = millisInDay % 1000; + U_ASSERT(getMinimum(UCAL_MILLISECOND) <= fFields[UCAL_MILLISECOND]); + U_ASSERT(fFields[UCAL_MILLISECOND] <= getMaximum(UCAL_MILLISECOND)); + + millisInDay /= 1000; + fFields[UCAL_SECOND] = millisInDay % 60; + U_ASSERT(getMinimum(UCAL_SECOND) <= fFields[UCAL_SECOND]); + U_ASSERT(fFields[UCAL_SECOND] <= getMaximum(UCAL_SECOND)); + + millisInDay /= 60; + fFields[UCAL_MINUTE] = millisInDay % 60; + U_ASSERT(getMinimum(UCAL_MINUTE) <= fFields[UCAL_MINUTE]); + U_ASSERT(fFields[UCAL_MINUTE] <= getMaximum(UCAL_MINUTE)); + + millisInDay /= 60; + fFields[UCAL_HOUR_OF_DAY] = millisInDay; + U_ASSERT(getMinimum(UCAL_HOUR_OF_DAY) <= fFields[UCAL_HOUR_OF_DAY]); + U_ASSERT(fFields[UCAL_HOUR_OF_DAY] <= getMaximum(UCAL_HOUR_OF_DAY)); + + fFields[UCAL_AM_PM] = millisInDay / 12; // Assume AM == 0 + U_ASSERT(getMinimum(UCAL_AM_PM) <= fFields[UCAL_AM_PM]); + U_ASSERT(fFields[UCAL_AM_PM] <= getMaximum(UCAL_AM_PM)); + + fFields[UCAL_HOUR] = millisInDay % 12; + U_ASSERT(getMinimum(UCAL_HOUR) <= fFields[UCAL_HOUR]); + U_ASSERT(fFields[UCAL_HOUR] <= getMaximum(UCAL_HOUR)); + + fFields[UCAL_ZONE_OFFSET] = rawOffset; + U_ASSERT(getMinimum(UCAL_ZONE_OFFSET) <= fFields[UCAL_ZONE_OFFSET]); + U_ASSERT(fFields[UCAL_ZONE_OFFSET] <= getMaximum(UCAL_ZONE_OFFSET)); + + fFields[UCAL_DST_OFFSET] = dstOffset; + U_ASSERT(getMinimum(UCAL_DST_OFFSET) <= fFields[UCAL_DST_OFFSET]); + U_ASSERT(fFields[UCAL_DST_OFFSET] <= getMaximum(UCAL_DST_OFFSET)); +} + +uint8_t Calendar::julianDayToDayOfWeek(double julian) +{ + // If julian is negative, then julian%7 will be negative, so we adjust + // accordingly. We add 1 because Julian day 0 is Monday. + int8_t dayOfWeek = (int8_t) uprv_fmod(julian + 1, 7); + + uint8_t result = (uint8_t)(dayOfWeek + ((dayOfWeek < 0) ? (7+UCAL_SUNDAY ) : UCAL_SUNDAY)); + return result; +} + +/** +* Compute the Gregorian calendar year, month, and day of month from +* the given Julian day. These values are not stored in fields, but in +* member variables gregorianXxx. Also compute the DAY_OF_WEEK and +* DOW_LOCAL fields. +*/ +void Calendar::computeGregorianAndDOWFields(int32_t julianDay, UErrorCode &ec) +{ + computeGregorianFields(julianDay, ec); + if (U_FAILURE(ec)) { + return; + } + + // Compute day of week: JD 0 = Monday + int32_t dow = julianDayToDayOfWeek(julianDay); + internalSet(UCAL_DAY_OF_WEEK,dow); + + // Calculate 1-based localized day of week + int32_t dowLocal = dow - getFirstDayOfWeek() + 1; + if (dowLocal < 1) { + dowLocal += 7; + } + internalSet(UCAL_DOW_LOCAL,dowLocal); + fFields[UCAL_DOW_LOCAL] = dowLocal; +} + +/** +* Compute the Gregorian calendar year, month, and day of month from the +* Julian day. These values are not stored in fields, but in member +* variables gregorianXxx. They are used for time zone computations and by +* subclasses that are Gregorian derivatives. Subclasses may call this +* method to perform a Gregorian calendar millis->fields computation. +*/ +void Calendar::computeGregorianFields(int32_t julianDay, UErrorCode& ec) { + if (U_FAILURE(ec)) { + return; + } + int32_t gregorianDayOfWeekUnused; + Grego::dayToFields(julianDay - kEpochStartAsJulianDay, fGregorianYear, fGregorianMonth, fGregorianDayOfMonth, gregorianDayOfWeekUnused, fGregorianDayOfYear); +} + +/** +* Compute the fields WEEK_OF_YEAR, YEAR_WOY, WEEK_OF_MONTH, +* DAY_OF_WEEK_IN_MONTH, and DOW_LOCAL from EXTENDED_YEAR, YEAR, +* DAY_OF_WEEK, and DAY_OF_YEAR. The latter fields are computed by the +* subclass based on the calendar system. +* +*

The YEAR_WOY field is computed simplistically. It is equal to YEAR +* most of the time, but at the year boundary it may be adjusted to YEAR-1 +* or YEAR+1 to reflect the overlap of a week into an adjacent year. In +* this case, a simple increment or decrement is performed on YEAR, even +* though this may yield an invalid YEAR value. For instance, if the YEAR +* is part of a calendar system with an N-year cycle field CYCLE, then +* incrementing the YEAR may involve incrementing CYCLE and setting YEAR +* back to 0 or 1. This is not handled by this code, and in fact cannot be +* simply handled without having subclasses define an entire parallel set of +* fields for fields larger than or equal to a year. This additional +* complexity is not warranted, since the intention of the YEAR_WOY field is +* to support ISO 8601 notation, so it will typically be used with a +* proleptic Gregorian calendar, which has no field larger than a year. +*/ +void Calendar::computeWeekFields(UErrorCode &ec) { + if(U_FAILURE(ec)) { + return; + } + int32_t eyear = fFields[UCAL_EXTENDED_YEAR]; + int32_t dayOfWeek = fFields[UCAL_DAY_OF_WEEK]; + int32_t dayOfYear = fFields[UCAL_DAY_OF_YEAR]; + + // WEEK_OF_YEAR start + // Compute the week of the year. For the Gregorian calendar, valid week + // numbers run from 1 to 52 or 53, depending on the year, the first day + // of the week, and the minimal days in the first week. For other + // calendars, the valid range may be different -- it depends on the year + // length. Days at the start of the year may fall into the last week of + // the previous year; days at the end of the year may fall into the + // first week of the next year. ASSUME that the year length is less than + // 7000 days. + int32_t yearOfWeekOfYear = eyear; + int32_t relDow = (dayOfWeek + 7 - getFirstDayOfWeek()) % 7; // 0..6 + int32_t relDowJan1 = (dayOfWeek - dayOfYear + 7001 - getFirstDayOfWeek()) % 7; // 0..6 + int32_t woy = (dayOfYear - 1 + relDowJan1) / 7; // 0..53 + if ((7 - relDowJan1) >= getMinimalDaysInFirstWeek()) { + ++woy; + } + + // Adjust for weeks at the year end that overlap into the previous or + // next calendar year. + if (woy == 0) { + // We are the last week of the previous year. + // Check to see if we are in the last week; if so, we need + // to handle the case in which we are the first week of the + // next year. + + int32_t prevDoy = dayOfYear + handleGetYearLength(eyear - 1); + woy = weekNumber(prevDoy, dayOfWeek); + yearOfWeekOfYear--; + } else { + int32_t lastDoy = handleGetYearLength(eyear); + // Fast check: For it to be week 1 of the next year, the DOY + // must be on or after L-5, where L is yearLength(), then it + // cannot possibly be week 1 of the next year: + // L-5 L + // doy: 359 360 361 362 363 364 365 001 + // dow: 1 2 3 4 5 6 7 + if (dayOfYear >= (lastDoy - 5)) { + int32_t lastRelDow = (relDow + lastDoy - dayOfYear) % 7; + if (lastRelDow < 0) { + lastRelDow += 7; + } + if (((6 - lastRelDow) >= getMinimalDaysInFirstWeek()) && + ((dayOfYear + 7 - relDow) > lastDoy)) { + woy = 1; + yearOfWeekOfYear++; + } + } + } + fFields[UCAL_WEEK_OF_YEAR] = woy; + fFields[UCAL_YEAR_WOY] = yearOfWeekOfYear; + // min/max of years are not constrains for caller, so not assert here. + // WEEK_OF_YEAR end + + int32_t dayOfMonth = fFields[UCAL_DAY_OF_MONTH]; + fFields[UCAL_WEEK_OF_MONTH] = weekNumber(dayOfMonth, dayOfWeek); + U_ASSERT(getMinimum(UCAL_WEEK_OF_MONTH) <= fFields[UCAL_WEEK_OF_MONTH]); + U_ASSERT(fFields[UCAL_WEEK_OF_MONTH] <= getMaximum(UCAL_WEEK_OF_MONTH)); + + fFields[UCAL_DAY_OF_WEEK_IN_MONTH] = (dayOfMonth-1) / 7 + 1; + U_ASSERT(getMinimum(UCAL_DAY_OF_WEEK_IN_MONTH) <= + fFields[UCAL_DAY_OF_WEEK_IN_MONTH]); + U_ASSERT(fFields[UCAL_DAY_OF_WEEK_IN_MONTH] <= + getMaximum(UCAL_DAY_OF_WEEK_IN_MONTH)); + +#if defined (U_DEBUG_CAL) + if(fFields[UCAL_DAY_OF_WEEK_IN_MONTH]==0) fprintf(stderr, "%s:%d: DOWIM %d on %g\n", + __FILE__, __LINE__,fFields[UCAL_DAY_OF_WEEK_IN_MONTH], fTime); +#endif +} + + +int32_t Calendar::weekNumber(int32_t desiredDay, int32_t dayOfPeriod, int32_t dayOfWeek) +{ + // Determine the day of the week of the first day of the period + // in question (either a year or a month). Zero represents the + // first day of the week on this calendar. + int32_t periodStartDayOfWeek = (dayOfWeek - getFirstDayOfWeek() - dayOfPeriod + 1) % 7; + if (periodStartDayOfWeek < 0) periodStartDayOfWeek += 7; + + // Compute the week number. Initially, ignore the first week, which + // may be fractional (or may not be). We add periodStartDayOfWeek in + // order to fill out the first week, if it is fractional. + int32_t weekNo = (desiredDay + periodStartDayOfWeek - 1)/7; + + // If the first week is long enough, then count it. If + // the minimal days in the first week is one, or if the period start + // is zero, we always increment weekNo. + if ((7 - periodStartDayOfWeek) >= getMinimalDaysInFirstWeek()) ++weekNo; + + return weekNo; +} + +void Calendar::handleComputeFields(int32_t /* julianDay */, UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + int32_t month = getGregorianMonth(); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, getGregorianDayOfMonth()); + internalSet(UCAL_DAY_OF_YEAR, getGregorianDayOfYear()); + int32_t eyear = getGregorianYear(); + internalSet(UCAL_EXTENDED_YEAR, eyear); + int32_t era = GregorianCalendar::AD; + if (eyear < 1) { + era = GregorianCalendar::BC; + eyear = 1 - eyear; + } + internalSet(UCAL_ERA, era); + internalSet(UCAL_YEAR, eyear); +} +// ------------------------------------- + + +void Calendar::roll(EDateFields field, int32_t amount, UErrorCode& status) +{ + roll((UCalendarDateFields)field, amount, status); +} + +void Calendar::roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) UPRV_NO_SANITIZE_UNDEFINED { + if (amount == 0) { + return; // Nothing to do + } + + complete(status); + + if(U_FAILURE(status)) { + return; + } + switch (field) { + case UCAL_DAY_OF_MONTH: + case UCAL_AM_PM: + case UCAL_MINUTE: + case UCAL_SECOND: + case UCAL_MILLISECOND: + case UCAL_MILLISECONDS_IN_DAY: + case UCAL_ERA: + // These are the standard roll instructions. These work for all + // simple cases, that is, cases in which the limits are fixed, such + // as the hour, the day of the month, and the era. + { + int32_t min = getActualMinimum(field,status); + int32_t max = getActualMaximum(field,status); + int32_t gap = max - min + 1; + + int32_t value = internalGet(field) + amount; + value = (value - min) % gap; + if (value < 0) { + value += gap; + } + value += min; + + set(field, value); + return; + } + + case UCAL_HOUR: + case UCAL_HOUR_OF_DAY: + // Rolling the hour is difficult on the ONSET and CEASE days of + // daylight savings. For example, if the change occurs at + // 2 AM, we have the following progression: + // ONSET: 12 Std -> 1 Std -> 3 Dst -> 4 Dst + // CEASE: 12 Dst -> 1 Dst -> 1 Std -> 2 Std + // To get around this problem we don't use fields; we manipulate + // the time in millis directly. + { + // Assume min == 0 in calculations below + double start = getTimeInMillis(status); + int32_t oldHour = internalGet(field); + int32_t max = getMaximum(field); + int32_t newHour = (oldHour + amount) % (max + 1); + if (newHour < 0) { + newHour += max + 1; + } + setTimeInMillis(start + kOneHour * (newHour - oldHour),status); + return; + } + + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + // Rolling the month involves both pinning the final value + // and adjusting the DAY_OF_MONTH if necessary. We only adjust the + // DAY_OF_MONTH if, after updating the MONTH field, it is illegal. + // E.g., .roll(MONTH, 1) -> or . + { + int32_t max = getActualMaximum(UCAL_MONTH, status); + int32_t mon = (internalGet(UCAL_MONTH) + amount) % (max+1); + + if (mon < 0) { + mon += (max + 1); + } + set(UCAL_MONTH, mon); + + // Keep the day of month in range. We don't want to spill over + // into the next month; e.g., we don't want jan31 + 1 mo -> feb31 -> + // mar3. + pinField(UCAL_DAY_OF_MONTH,status); + return; + } + + case UCAL_YEAR: + case UCAL_YEAR_WOY: + { + // * If era==0 and years go backwards in time, change sign of amount. + // * Until we have new API per #9393, we temporarily hardcode knowledge of + // which calendars have era 0 years that go backwards. + UBool era0WithYearsThatGoBackwards = false; + int32_t era = get(UCAL_ERA, status); + if (era == 0) { + const char * calType = getType(); + if ( uprv_strcmp(calType,"gregorian")==0 || uprv_strcmp(calType,"roc")==0 || uprv_strcmp(calType,"coptic")==0 ) { + amount = -amount; + era0WithYearsThatGoBackwards = true; + } + } + int32_t newYear = internalGet(field) + amount; + if (era > 0 || newYear >= 1) { + int32_t maxYear = getActualMaximum(field, status); + if (maxYear < 32768) { + // this era has real bounds, roll should wrap years + if (newYear < 1) { + newYear = maxYear - ((-newYear) % maxYear); + } else if (newYear > maxYear) { + newYear = ((newYear - 1) % maxYear) + 1; + } + // else era is unbounded, just pin low year instead of wrapping + } else if (newYear < 1) { + newYear = 1; + } + // else we are in era 0 with newYear < 1; + // calendars with years that go backwards must pin the year value at 0, + // other calendars can have years < 0 in era 0 + } else if (era0WithYearsThatGoBackwards) { + newYear = 1; + } + set(field, newYear); + pinField(UCAL_MONTH,status); + pinField(UCAL_ORDINAL_MONTH,status); + pinField(UCAL_DAY_OF_MONTH,status); + return; + } + + case UCAL_EXTENDED_YEAR: + // Rolling the year can involve pinning the DAY_OF_MONTH. + set(field, internalGet(field) + amount); + pinField(UCAL_MONTH,status); + pinField(UCAL_ORDINAL_MONTH,status); + pinField(UCAL_DAY_OF_MONTH,status); + return; + + case UCAL_WEEK_OF_MONTH: + { + // This is tricky, because during the roll we may have to shift + // to a different day of the week. For example: + + // s m t w r f s + // 1 2 3 4 5 + // 6 7 8 9 10 11 12 + + // When rolling from the 6th or 7th back one week, we go to the + // 1st (assuming that the first partial week counts). The same + // thing happens at the end of the month. + + // The other tricky thing is that we have to figure out whether + // the first partial week actually counts or not, based on the + // minimal first days in the week. And we have to use the + // correct first day of the week to delineate the week + // boundaries. + + // Here's our algorithm. First, we find the real boundaries of + // the month. Then we discard the first partial week if it + // doesn't count in this locale. Then we fill in the ends with + // phantom days, so that the first partial week and the last + // partial week are full weeks. We then have a nice square + // block of weeks. We do the usual rolling within this block, + // as is done elsewhere in this method. If we wind up on one of + // the phantom days that we added, we recognize this and pin to + // the first or the last day of the month. Easy, eh? + + // Normalize the DAY_OF_WEEK so that 0 is the first day of the week + // in this locale. We have dow in 0..6. + int32_t dow = internalGet(UCAL_DAY_OF_WEEK) - getFirstDayOfWeek(); + if (dow < 0) dow += 7; + + // Find the day of the week (normalized for locale) for the first + // of the month. + int32_t fdm = (dow - internalGet(UCAL_DAY_OF_MONTH) + 1) % 7; + if (fdm < 0) fdm += 7; + + // Get the first day of the first full week of the month, + // including phantom days, if any. Figure out if the first week + // counts or not; if it counts, then fill in phantom days. If + // not, advance to the first real full week (skip the partial week). + int32_t start; + if ((7 - fdm) < getMinimalDaysInFirstWeek()) + start = 8 - fdm; // Skip the first partial week + else + start = 1 - fdm; // This may be zero or negative + + // Get the day of the week (normalized for locale) for the last + // day of the month. + int32_t monthLen = getActualMaximum(UCAL_DAY_OF_MONTH, status); + int32_t ldm = (monthLen - internalGet(UCAL_DAY_OF_MONTH) + dow) % 7; + // We know monthLen >= DAY_OF_MONTH so we skip the += 7 step here. + + // Get the limit day for the blocked-off rectangular month; that + // is, the day which is one past the last day of the month, + // after the month has already been filled in with phantom days + // to fill out the last week. This day has a normalized DOW of 0. + int32_t limit = monthLen + 7 - ldm; + + // Now roll between start and (limit - 1). + int32_t gap = limit - start; + int32_t day_of_month = (internalGet(UCAL_DAY_OF_MONTH) + amount*7 - + start) % gap; + if (day_of_month < 0) day_of_month += gap; + day_of_month += start; + + // Finally, pin to the real start and end of the month. + if (day_of_month < 1) day_of_month = 1; + if (day_of_month > monthLen) day_of_month = monthLen; + + // Set the DAY_OF_MONTH. We rely on the fact that this field + // takes precedence over everything else (since all other fields + // are also set at this point). If this fact changes (if the + // disambiguation algorithm changes) then we will have to unset + // the appropriate fields here so that DAY_OF_MONTH is attended + // to. + set(UCAL_DAY_OF_MONTH, day_of_month); + return; + } + case UCAL_WEEK_OF_YEAR: + { + // This follows the outline of WEEK_OF_MONTH, except it applies + // to the whole year. Please see the comment for WEEK_OF_MONTH + // for general notes. + + // Normalize the DAY_OF_WEEK so that 0 is the first day of the week + // in this locale. We have dow in 0..6. + int32_t dow = internalGet(UCAL_DAY_OF_WEEK) - getFirstDayOfWeek(); + if (dow < 0) dow += 7; + + // Find the day of the week (normalized for locale) for the first + // of the year. + int32_t fdy = (dow - internalGet(UCAL_DAY_OF_YEAR) + 1) % 7; + if (fdy < 0) fdy += 7; + + // Get the first day of the first full week of the year, + // including phantom days, if any. Figure out if the first week + // counts or not; if it counts, then fill in phantom days. If + // not, advance to the first real full week (skip the partial week). + int32_t start; + if ((7 - fdy) < getMinimalDaysInFirstWeek()) + start = 8 - fdy; // Skip the first partial week + else + start = 1 - fdy; // This may be zero or negative + + // Get the day of the week (normalized for locale) for the last + // day of the year. + int32_t yearLen = getActualMaximum(UCAL_DAY_OF_YEAR,status); + int32_t ldy = (yearLen - internalGet(UCAL_DAY_OF_YEAR) + dow) % 7; + // We know yearLen >= DAY_OF_YEAR so we skip the += 7 step here. + + // Get the limit day for the blocked-off rectangular year; that + // is, the day which is one past the last day of the year, + // after the year has already been filled in with phantom days + // to fill out the last week. This day has a normalized DOW of 0. + int32_t limit = yearLen + 7 - ldy; + + // Now roll between start and (limit - 1). + int32_t gap = limit - start; + int32_t day_of_year = (internalGet(UCAL_DAY_OF_YEAR) + amount*7 - + start) % gap; + if (day_of_year < 0) day_of_year += gap; + day_of_year += start; + + // Finally, pin to the real start and end of the month. + if (day_of_year < 1) day_of_year = 1; + if (day_of_year > yearLen) day_of_year = yearLen; + + // Make sure that the year and day of year are attended to by + // clearing other fields which would normally take precedence. + // If the disambiguation algorithm is changed, this section will + // have to be updated as well. + set(UCAL_DAY_OF_YEAR, day_of_year); + clear(UCAL_MONTH); + clear(UCAL_ORDINAL_MONTH); + return; + } + case UCAL_DAY_OF_YEAR: + { + // Roll the day of year using millis. Compute the millis for + // the start of the year, and get the length of the year. + double delta = amount * kOneDay; // Scale up from days to millis + double min2 = internalGet(UCAL_DAY_OF_YEAR)-1; + min2 *= kOneDay; + min2 = internalGetTime() - min2; + + // double min2 = internalGetTime() - (internalGet(UCAL_DAY_OF_YEAR) - 1.0) * kOneDay; + double newtime; + + double yearLength = getActualMaximum(UCAL_DAY_OF_YEAR,status); + double oneYear = yearLength; + oneYear *= kOneDay; + newtime = uprv_fmod((internalGetTime() + delta - min2), oneYear); + if (newtime < 0) newtime += oneYear; + setTimeInMillis(newtime + min2, status); + return; + } + case UCAL_DAY_OF_WEEK: + case UCAL_DOW_LOCAL: + { + // Roll the day of week using millis. Compute the millis for + // the start of the week, using the first day of week setting. + // Restrict the millis to [start, start+7days). + double delta = amount * kOneDay; // Scale up from days to millis + // Compute the number of days before the current day in this + // week. This will be a value 0..6. + int32_t leadDays = internalGet(field); + leadDays -= (field == UCAL_DAY_OF_WEEK) ? getFirstDayOfWeek() : 1; + if (leadDays < 0) leadDays += 7; + double min2 = internalGetTime() - leadDays * kOneDay; + double newtime = uprv_fmod((internalGetTime() + delta - min2), kOneWeek); + if (newtime < 0) newtime += kOneWeek; + setTimeInMillis(newtime + min2, status); + return; + } + case UCAL_DAY_OF_WEEK_IN_MONTH: + { + // Roll the day of week in the month using millis. Determine + // the first day of the week in the month, and then the last, + // and then roll within that range. + double delta = amount * kOneWeek; // Scale up from weeks to millis + // Find the number of same days of the week before this one + // in this month. + int32_t preWeeks = (internalGet(UCAL_DAY_OF_MONTH) - 1) / 7; + // Find the number of same days of the week after this one + // in this month. + int32_t postWeeks = (getActualMaximum(UCAL_DAY_OF_MONTH,status) - + internalGet(UCAL_DAY_OF_MONTH)) / 7; + // From these compute the min and gap millis for rolling. + double min2 = internalGetTime() - preWeeks * kOneWeek; + double gap2 = kOneWeek * (preWeeks + postWeeks + 1); // Must add 1! + // Roll within this range + double newtime = uprv_fmod((internalGetTime() + delta - min2), gap2); + if (newtime < 0) newtime += gap2; + setTimeInMillis(newtime + min2, status); + return; + } + case UCAL_JULIAN_DAY: + set(field, internalGet(field) + amount); + return; + default: + // Other fields cannot be rolled by this method +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because of roll on non-rollable field %s\n", + __FILE__, __LINE__,fldName(field)); +#endif + status = U_ILLEGAL_ARGUMENT_ERROR; + } +} + +void Calendar::add(EDateFields field, int32_t amount, UErrorCode& status) +{ + Calendar::add((UCalendarDateFields)field, amount, status); +} + +// ------------------------------------- +void Calendar::add(UCalendarDateFields field, int32_t amount, UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + if (amount == 0) { + return; // Do nothing! + } + + // We handle most fields in the same way. The algorithm is to add + // a computed amount of millis to the current millis. The only + // wrinkle is with DST (and/or a change to the zone's UTC offset, which + // we'll include with DST) -- for some fields, like the DAY_OF_MONTH, + // we don't want the wall time to shift due to changes in DST. If the + // result of the add operation is to move from DST to Standard, or + // vice versa, we need to adjust by an hour forward or back, + // respectively. For such fields we set keepWallTimeInvariant to true. + + // We only adjust the DST for fields larger than an hour. For + // fields smaller than an hour, we cannot adjust for DST without + // causing problems. for instance, if you add one hour to April 5, + // 1998, 1:00 AM, in PST, the time becomes "2:00 AM PDT" (an + // illegal value), but then the adjustment sees the change and + // compensates by subtracting an hour. As a result the time + // doesn't advance at all. + + // For some fields larger than a day, such as a UCAL_MONTH, we pin the + // UCAL_DAY_OF_MONTH. This allows .add(UCAL_MONTH, 1) to be + // , rather than => . + + double delta = amount; // delta in ms + UBool keepWallTimeInvariant = true; + + switch (field) { + case UCAL_ERA: + set(field, get(field, status) + amount); + pinField(UCAL_ERA, status); + return; + + case UCAL_YEAR: + case UCAL_YEAR_WOY: + { + // * If era=0 and years go backwards in time, change sign of amount. + // * Until we have new API per #9393, we temporarily hardcode knowledge of + // which calendars have era 0 years that go backwards. + // * Note that for UCAL_YEAR (but not UCAL_YEAR_WOY) we could instead handle + // this by applying the amount to the UCAL_EXTENDED_YEAR field; but since + // we would still need to handle UCAL_YEAR_WOY as below, might as well + // also handle UCAL_YEAR the same way. + int32_t era = get(UCAL_ERA, status); + if (era == 0) { + const char * calType = getType(); + if ( uprv_strcmp(calType,"gregorian")==0 || uprv_strcmp(calType,"roc")==0 || uprv_strcmp(calType,"coptic")==0 ) { + amount = -amount; + } + } + } + // Fall through into normal handling + U_FALLTHROUGH; + case UCAL_EXTENDED_YEAR: + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + { + UBool oldLenient = isLenient(); + setLenient(true); + set(field, get(field, status) + amount); + pinField(UCAL_DAY_OF_MONTH, status); + if(oldLenient==false) { + complete(status); /* force recalculate */ + setLenient(oldLenient); + } + } + return; + + case UCAL_WEEK_OF_YEAR: + case UCAL_WEEK_OF_MONTH: + case UCAL_DAY_OF_WEEK_IN_MONTH: + delta *= kOneWeek; + break; + + case UCAL_AM_PM: + delta *= 12 * kOneHour; + break; + + case UCAL_DAY_OF_MONTH: + case UCAL_DAY_OF_YEAR: + case UCAL_DAY_OF_WEEK: + case UCAL_DOW_LOCAL: + case UCAL_JULIAN_DAY: + delta *= kOneDay; + break; + + case UCAL_HOUR_OF_DAY: + case UCAL_HOUR: + delta *= kOneHour; + keepWallTimeInvariant = false; + break; + + case UCAL_MINUTE: + delta *= kOneMinute; + keepWallTimeInvariant = false; + break; + + case UCAL_SECOND: + delta *= kOneSecond; + keepWallTimeInvariant = false; + break; + + case UCAL_MILLISECOND: + case UCAL_MILLISECONDS_IN_DAY: + keepWallTimeInvariant = false; + break; + + default: +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because field %s not addable", + __FILE__, __LINE__, fldName(field)); +#endif + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + // throw new IllegalArgumentException("Calendar.add(" + fieldName(field) + + // ") not supported"); + } + + // In order to keep the wall time invariant (for fields where this is + // appropriate), check the combined DST & ZONE offset before and + // after the add() operation. If it changes, then adjust the millis + // to compensate. + int32_t prevOffset = 0; + int32_t prevWallTime = 0; + if (keepWallTimeInvariant) { + prevOffset = get(UCAL_DST_OFFSET, status) + get(UCAL_ZONE_OFFSET, status); + prevWallTime = get(UCAL_MILLISECONDS_IN_DAY, status); + } + + setTimeInMillis(getTimeInMillis(status) + delta, status); + + if (keepWallTimeInvariant) { + int32_t newWallTime = get(UCAL_MILLISECONDS_IN_DAY, status); + if (newWallTime != prevWallTime) { + // There is at least one zone transition between the base + // time and the result time. As the result, wall time has + // changed. + UDate t = internalGetTime(); + int32_t newOffset = get(UCAL_DST_OFFSET, status) + get(UCAL_ZONE_OFFSET, status); + if (newOffset != prevOffset) { + // When the difference of the previous UTC offset and + // the new UTC offset exceeds 1 full day, we do not want + // to roll over/back the date. For now, this only happens + // in Samoa (Pacific/Apia) on Dec 30, 2011. See ticket:9452. + int32_t adjAmount = prevOffset - newOffset; + adjAmount = adjAmount >= 0 ? adjAmount % (int32_t)kOneDay : -(-adjAmount % (int32_t)kOneDay); + if (adjAmount != 0) { + setTimeInMillis(t + adjAmount, status); + newWallTime = get(UCAL_MILLISECONDS_IN_DAY, status); + } + if (newWallTime != prevWallTime) { + // The result wall time or adjusted wall time was shifted because + // the target wall time does not exist on the result date. + switch (fSkippedWallTime) { + case UCAL_WALLTIME_FIRST: + if (adjAmount > 0) { + setTimeInMillis(t, status); + } + break; + case UCAL_WALLTIME_LAST: + if (adjAmount < 0) { + setTimeInMillis(t, status); + } + break; + case UCAL_WALLTIME_NEXT_VALID: + UDate tmpT = adjAmount > 0 ? internalGetTime() : t; + UDate immediatePrevTrans; + UBool hasTransition = getImmediatePreviousZoneTransition(tmpT, &immediatePrevTrans, status); + if (U_SUCCESS(status) && hasTransition) { + setTimeInMillis(immediatePrevTrans, status); + } + break; + } + } + } + } + } +} + +// ------------------------------------- +int32_t Calendar::fieldDifference(UDate when, EDateFields field, UErrorCode& status) { + return fieldDifference(when, (UCalendarDateFields) field, status); +} + +int32_t Calendar::fieldDifference(UDate targetMs, UCalendarDateFields field, UErrorCode& ec) { + if (U_FAILURE(ec)) return 0; + int32_t min = 0; + double startMs = getTimeInMillis(ec); + // Always add from the start millis. This accommodates + // operations like adding years from February 29, 2000 up to + // February 29, 2004. If 1, 1, 1, 1 is added to the year + // field, the DOM gets pinned to 28 and stays there, giving an + // incorrect DOM difference of 1. We have to add 1, reset, 2, + // reset, 3, reset, 4. + if (startMs < targetMs) { + int32_t max = 1; + // Find a value that is too large + while (U_SUCCESS(ec)) { + setTimeInMillis(startMs, ec); + add(field, max, ec); + double ms = getTimeInMillis(ec); + if (ms == targetMs) { + return max; + } else if (ms > targetMs) { + break; + } else if (max < INT32_MAX) { + min = max; + max <<= 1; + if (max < 0) { + max = INT32_MAX; + } + } else { + // Field difference too large to fit into int32_t +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because field %s's max too large for int32_t\n", + __FILE__, __LINE__, fldName(field)); +#endif + ec = U_ILLEGAL_ARGUMENT_ERROR; + } + } + // Do a binary search + while ((max - min) > 1 && U_SUCCESS(ec)) { + int32_t t = min + (max - min)/2; // make sure intermediate values don't exceed INT32_MAX + setTimeInMillis(startMs, ec); + add(field, t, ec); + double ms = getTimeInMillis(ec); + if (ms == targetMs) { + return t; + } else if (ms > targetMs) { + max = t; + } else { + min = t; + } + } + } else if (startMs > targetMs) { + int32_t max = -1; + // Find a value that is too small + while (U_SUCCESS(ec)) { + setTimeInMillis(startMs, ec); + add(field, max, ec); + double ms = getTimeInMillis(ec); + if (ms == targetMs) { + return max; + } else if (ms < targetMs) { + break; + } else { + min = max; + max = (int32_t)((uint32_t)(max) << 1); + if (max == 0) { + // Field difference too large to fit into int32_t +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because field %s's max too large for int32_t\n", + __FILE__, __LINE__, fldName(field)); +#endif + ec = U_ILLEGAL_ARGUMENT_ERROR; + } + } + } + // Do a binary search + while ((min - max) > 1 && U_SUCCESS(ec)) { + int32_t t = min + (max - min)/2; // make sure intermediate values don't exceed INT32_MAX + setTimeInMillis(startMs, ec); + add(field, t, ec); + double ms = getTimeInMillis(ec); + if (ms == targetMs) { + return t; + } else if (ms < targetMs) { + max = t; + } else { + min = t; + } + } + } + // Set calendar to end point + setTimeInMillis(startMs, ec); + add(field, min, ec); + + /* Test for buffer overflows */ + if(U_FAILURE(ec)) { + return 0; + } + return min; +} + +// ------------------------------------- + +void +Calendar::adoptTimeZone(TimeZone* zone) +{ + // Do nothing if passed-in zone is nullptr + if (zone == nullptr) return; + + // fZone should always be non-null + delete fZone; + fZone = zone; + + // if the zone changes, we need to recompute the time fields + fAreFieldsSet = false; +} + +// ------------------------------------- +void +Calendar::setTimeZone(const TimeZone& zone) +{ + adoptTimeZone(zone.clone()); +} + +// ------------------------------------- + +const TimeZone& +Calendar::getTimeZone() const +{ + U_ASSERT(fZone != nullptr); + return *fZone; +} + +// ------------------------------------- + +TimeZone* +Calendar::orphanTimeZone() +{ + // we let go of the time zone; the new time zone is the system default time zone + TimeZone *defaultZone = TimeZone::createDefault(); + if (defaultZone == nullptr) { + // No error handling available. Must keep fZone non-nullptr, there are many unchecked uses. + return nullptr; + } + TimeZone *z = fZone; + fZone = defaultZone; + return z; +} + +// ------------------------------------- + +void +Calendar::setLenient(UBool lenient) +{ + fLenient = lenient; +} + +// ------------------------------------- + +UBool +Calendar::isLenient() const +{ + return fLenient; +} + +// ------------------------------------- + +void +Calendar::setRepeatedWallTimeOption(UCalendarWallTimeOption option) +{ + if (option == UCAL_WALLTIME_LAST || option == UCAL_WALLTIME_FIRST) { + fRepeatedWallTime = option; + } +} + +// ------------------------------------- + +UCalendarWallTimeOption +Calendar::getRepeatedWallTimeOption() const +{ + return fRepeatedWallTime; +} + +// ------------------------------------- + +void +Calendar::setSkippedWallTimeOption(UCalendarWallTimeOption option) +{ + fSkippedWallTime = option; +} + +// ------------------------------------- + +UCalendarWallTimeOption +Calendar::getSkippedWallTimeOption() const +{ + return fSkippedWallTime; +} + +// ------------------------------------- + +void +Calendar::setFirstDayOfWeek(UCalendarDaysOfWeek value) UPRV_NO_SANITIZE_UNDEFINED { + if (fFirstDayOfWeek != value && + value >= UCAL_SUNDAY && value <= UCAL_SATURDAY) { + fFirstDayOfWeek = value; + fAreFieldsSet = false; + } +} + +// ------------------------------------- + +Calendar::EDaysOfWeek +Calendar::getFirstDayOfWeek() const +{ + return (Calendar::EDaysOfWeek)fFirstDayOfWeek; +} + +UCalendarDaysOfWeek +Calendar::getFirstDayOfWeek(UErrorCode & /*status*/) const +{ + return fFirstDayOfWeek; +} +// ------------------------------------- + +void +Calendar::setMinimalDaysInFirstWeek(uint8_t value) +{ + // Values less than 1 have the same effect as 1; values greater + // than 7 have the same effect as 7. However, we normalize values + // so operator== and so forth work. + if (value < 1) { + value = 1; + } else if (value > 7) { + value = 7; + } + if (fMinimalDaysInFirstWeek != value) { + fMinimalDaysInFirstWeek = value; + fAreFieldsSet = false; + } +} + +// ------------------------------------- + +uint8_t +Calendar::getMinimalDaysInFirstWeek() const +{ + return fMinimalDaysInFirstWeek; +} + +// ------------------------------------- +// weekend functions, just dummy implementations for now (for API freeze) + +UCalendarWeekdayType +Calendar::getDayOfWeekType(UCalendarDaysOfWeek dayOfWeek, UErrorCode &status) const +{ + if (U_FAILURE(status)) { + return UCAL_WEEKDAY; + } + if (dayOfWeek < UCAL_SUNDAY || dayOfWeek > UCAL_SATURDAY) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return UCAL_WEEKDAY; + } + if (fWeekendOnset == fWeekendCease) { + if (dayOfWeek != fWeekendOnset) + return UCAL_WEEKDAY; + return (fWeekendOnsetMillis == 0) ? UCAL_WEEKEND : UCAL_WEEKEND_ONSET; + } + if (fWeekendOnset < fWeekendCease) { + if (dayOfWeek < fWeekendOnset || dayOfWeek > fWeekendCease) { + return UCAL_WEEKDAY; + } + } else { + if (dayOfWeek > fWeekendCease && dayOfWeek < fWeekendOnset) { + return UCAL_WEEKDAY; + } + } + if (dayOfWeek == fWeekendOnset) { + return (fWeekendOnsetMillis == 0) ? UCAL_WEEKEND : UCAL_WEEKEND_ONSET; + } + if (dayOfWeek == fWeekendCease) { + return (fWeekendCeaseMillis >= 86400000) ? UCAL_WEEKEND : UCAL_WEEKEND_CEASE; + } + return UCAL_WEEKEND; +} + +int32_t +Calendar::getWeekendTransition(UCalendarDaysOfWeek dayOfWeek, UErrorCode &status) const +{ + if (U_FAILURE(status)) { + return 0; + } + if (dayOfWeek == fWeekendOnset) { + return fWeekendOnsetMillis; + } else if (dayOfWeek == fWeekendCease) { + return fWeekendCeaseMillis; + } + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; +} + +UBool +Calendar::isWeekend(UDate date, UErrorCode &status) const +{ + if (U_FAILURE(status)) { + return false; + } + // clone the calendar so we don't mess with the real one. + Calendar *work = this->clone(); + if (work == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + UBool result = false; + work->setTime(date, status); + if (U_SUCCESS(status)) { + result = work->isWeekend(); + } + delete work; + return result; +} + +UBool +Calendar::isWeekend() const +{ + UErrorCode status = U_ZERO_ERROR; + UCalendarDaysOfWeek dayOfWeek = (UCalendarDaysOfWeek)get(UCAL_DAY_OF_WEEK, status); + UCalendarWeekdayType dayType = getDayOfWeekType(dayOfWeek, status); + if (U_SUCCESS(status)) { + switch (dayType) { + case UCAL_WEEKDAY: + return false; + case UCAL_WEEKEND: + return true; + case UCAL_WEEKEND_ONSET: + case UCAL_WEEKEND_CEASE: + // Use internalGet() because the above call to get() populated all fields. + { + int32_t millisInDay = internalGet(UCAL_MILLISECONDS_IN_DAY); + int32_t transitionMillis = getWeekendTransition(dayOfWeek, status); + if (U_SUCCESS(status)) { + return (dayType == UCAL_WEEKEND_ONSET)? + (millisInDay >= transitionMillis): + (millisInDay < transitionMillis); + } + // else fall through, return false + U_FALLTHROUGH; + } + default: + break; + } + } + return false; +} + +// ------------------------------------- limits + +int32_t +Calendar::getMinimum(EDateFields field) const { + return getLimit((UCalendarDateFields) field,UCAL_LIMIT_MINIMUM); +} + +int32_t +Calendar::getMinimum(UCalendarDateFields field) const +{ + return getLimit(field,UCAL_LIMIT_MINIMUM); +} + +// ------------------------------------- +int32_t +Calendar::getMaximum(EDateFields field) const +{ + return getLimit((UCalendarDateFields) field,UCAL_LIMIT_MAXIMUM); +} + +int32_t +Calendar::getMaximum(UCalendarDateFields field) const +{ + return getLimit(field,UCAL_LIMIT_MAXIMUM); +} + +// ------------------------------------- +int32_t +Calendar::getGreatestMinimum(EDateFields field) const +{ + return getLimit((UCalendarDateFields)field,UCAL_LIMIT_GREATEST_MINIMUM); +} + +int32_t +Calendar::getGreatestMinimum(UCalendarDateFields field) const +{ + return getLimit(field,UCAL_LIMIT_GREATEST_MINIMUM); +} + +// ------------------------------------- +int32_t +Calendar::getLeastMaximum(EDateFields field) const +{ + return getLimit((UCalendarDateFields) field,UCAL_LIMIT_LEAST_MAXIMUM); +} + +int32_t +Calendar::getLeastMaximum(UCalendarDateFields field) const +{ + return getLimit( field,UCAL_LIMIT_LEAST_MAXIMUM); +} + +// ------------------------------------- +int32_t +Calendar::getActualMinimum(EDateFields field, UErrorCode& status) const +{ + return getActualMinimum((UCalendarDateFields) field, status); +} + +int32_t Calendar::getLimit(UCalendarDateFields field, ELimitType limitType) const { + switch (field) { + case UCAL_DAY_OF_WEEK: + case UCAL_AM_PM: + case UCAL_HOUR: + case UCAL_HOUR_OF_DAY: + case UCAL_MINUTE: + case UCAL_SECOND: + case UCAL_MILLISECOND: + case UCAL_ZONE_OFFSET: + case UCAL_DST_OFFSET: + case UCAL_DOW_LOCAL: + case UCAL_JULIAN_DAY: + case UCAL_MILLISECONDS_IN_DAY: + case UCAL_IS_LEAP_MONTH: + return kCalendarLimits[field][limitType]; + + case UCAL_WEEK_OF_MONTH: + { + int32_t limit; + if (limitType == UCAL_LIMIT_MINIMUM) { + limit = getMinimalDaysInFirstWeek() == 1 ? 1 : 0; + } else if (limitType == UCAL_LIMIT_GREATEST_MINIMUM) { + limit = 1; + } else { + int32_t minDaysInFirst = getMinimalDaysInFirstWeek(); + int32_t daysInMonth = handleGetLimit(UCAL_DAY_OF_MONTH, limitType); + if (limitType == UCAL_LIMIT_LEAST_MAXIMUM) { + limit = (daysInMonth + (7 - minDaysInFirst)) / 7; + } else { // limitType == UCAL_LIMIT_MAXIMUM + limit = (daysInMonth + 6 + (7 - minDaysInFirst)) / 7; + } + } + return limit; + } + default: + return handleGetLimit(field, limitType); + } +} + +int32_t +Calendar::getActualMinimum(UCalendarDateFields field, UErrorCode& status) const +{ + if (U_FAILURE(status)) { + return 0; + } + int32_t fieldValue = getGreatestMinimum(field); + int32_t endValue = getMinimum(field); + + // if we know that the minimum value is always the same, just return it + if (fieldValue == endValue) { + return fieldValue; + } + + // clone the calendar so we don't mess with the real one, and set it to + // accept anything for the field values + Calendar *work = this->clone(); + if (work == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + work->setLenient(true); + + // now try each value from getLeastMaximum() to getMaximum() one by one until + // we get a value that normalizes to another value. The last value that + // normalizes to itself is the actual minimum for the current date + int32_t result = fieldValue; + + do { + work->set(field, fieldValue); + if (work->get(field, status) != fieldValue) { + break; + } + else { + result = fieldValue; + fieldValue--; + } + } while (fieldValue >= endValue); + + delete work; + + /* Test for buffer overflows */ + if(U_FAILURE(status)) { + return 0; + } + return result; +} + +// ------------------------------------- + +UBool +Calendar::inDaylightTime(UErrorCode& status) const +{ + if (U_FAILURE(status) || !getTimeZone().useDaylightTime()) { + return false; + } + + // Force an update of the state of the Calendar. + ((Calendar*)this)->complete(status); // cast away const + + return (UBool)(U_SUCCESS(status) ? (internalGet(UCAL_DST_OFFSET) != 0) : false); +} + +bool +Calendar::inTemporalLeapYear(UErrorCode& status) const +{ + // Default to Gregorian based leap year rule. + return getActualMaximum(UCAL_DAY_OF_YEAR, status) == 366; +} + +// ------------------------------------- + +static const char * const gTemporalMonthCodes[] = { + "M01", "M02", "M03", "M04", "M05", "M06", + "M07", "M08", "M09", "M10", "M11", "M12", nullptr +}; + +const char* +Calendar::getTemporalMonthCode(UErrorCode& status) const +{ + int32_t month = get(UCAL_MONTH, status); + if (U_FAILURE(status)) return nullptr; + U_ASSERT(month < 12); + U_ASSERT(internalGet(UCAL_IS_LEAP_MONTH) == 0); + return gTemporalMonthCodes[month]; +} + +void +Calendar::setTemporalMonthCode(const char* code, UErrorCode& status ) +{ + if (U_FAILURE(status)) return; + int32_t len = static_cast(uprv_strlen(code)); + if (len == 3 && code[0] == 'M') { + for (int m = 0; gTemporalMonthCodes[m] != nullptr; m++) { + if (uprv_strcmp(code, gTemporalMonthCodes[m]) == 0) { + set(UCAL_MONTH, m); + set(UCAL_IS_LEAP_MONTH, 0); + return; + } + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; +} + +// ------------------------------------- + +/** +* Ensure that each field is within its valid range by calling {@link +* #validateField(int)} on each field that has been set. This method +* should only be called if this calendar is not lenient. +* @see #isLenient +* @see #validateField(int) +*/ +void Calendar::validateFields(UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + for (int32_t field = 0; U_SUCCESS(status) && (field < UCAL_FIELD_COUNT); field++) { + if (fStamp[field] >= kMinimumUserStamp) { + validateField((UCalendarDateFields)field, status); + } + } +} + +/** +* Validate a single field of this calendar. Subclasses should +* override this method to validate any calendar-specific fields. +* Generic fields can be handled by +* Calendar.validateField(). +* @see #validateField(int, int, int) +*/ +void Calendar::validateField(UCalendarDateFields field, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + int32_t y; + switch (field) { + case UCAL_DAY_OF_MONTH: + y = handleGetExtendedYear(); + validateField(field, 1, handleGetMonthLength(y, internalGetMonth()), status); + break; + case UCAL_DAY_OF_YEAR: + y = handleGetExtendedYear(); + validateField(field, 1, handleGetYearLength(y), status); + break; + case UCAL_DAY_OF_WEEK_IN_MONTH: + if (internalGet(field) == 0) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because DOW in month cannot be 0\n", + __FILE__, __LINE__); +#endif + status = U_ILLEGAL_ARGUMENT_ERROR; // "DAY_OF_WEEK_IN_MONTH cannot be zero" + return; + } + validateField(field, getMinimum(field), getMaximum(field), status); + break; + default: + validateField(field, getMinimum(field), getMaximum(field), status); + break; + } +} + +/** +* Validate a single field of this calendar given its minimum and +* maximum allowed value. If the field is out of range, throw a +* descriptive IllegalArgumentException. Subclasses may +* use this method in their implementation of {@link +* #validateField(int)}. +*/ +void Calendar::validateField(UCalendarDateFields field, int32_t min, int32_t max, UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + int32_t value = fFields[field]; + if (value < min || value > max) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: ILLEGAL ARG because of field %s out of range %d..%d at %d\n", + __FILE__, __LINE__,fldName(field),min,max,value); +#endif + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } +} + +// ------------------------- + +const UFieldResolutionTable* Calendar::getFieldResolutionTable() const { + return kDatePrecedence; +} + + +UCalendarDateFields Calendar::newerField(UCalendarDateFields defaultField, UCalendarDateFields alternateField) const +{ + if (fStamp[alternateField] > fStamp[defaultField]) { + return alternateField; + } + return defaultField; +} + +UCalendarDateFields Calendar::resolveFields(const UFieldResolutionTable* precedenceTable) const { + int32_t bestField = UCAL_FIELD_COUNT; + int32_t tempBestField; + for (int32_t g=0; precedenceTable[g][0][0] != -1 && (bestField == UCAL_FIELD_COUNT); ++g) { + int32_t bestStamp = kUnset; + for (int32_t l=0; precedenceTable[g][l][0] != -1; ++l) { + int32_t lineStamp = kUnset; + // Skip over first entry if it is negative + for (int32_t i=((precedenceTable[g][l][0]>=kResolveRemap)?1:0); precedenceTable[g][l][i]!=-1; ++i) { + U_ASSERT(precedenceTable[g][l][i] < UCAL_FIELD_COUNT); + int32_t s = fStamp[precedenceTable[g][l][i]]; + // If any field is unset then don't use this line + if (s == kUnset) { + goto linesInGroup; + } else if(s > lineStamp) { + lineStamp = s; + } + } + // Record new maximum stamp & field no. + if (lineStamp > bestStamp) { + tempBestField = precedenceTable[g][l][0]; // First field refers to entire line + if (tempBestField >= kResolveRemap) { + tempBestField &= (kResolveRemap-1); + // This check is needed to resolve some issues with UCAL_YEAR precedence mapping + if (tempBestField != UCAL_DATE || (fStamp[UCAL_WEEK_OF_MONTH] < fStamp[tempBestField])) { + bestField = tempBestField; + } + } else { + bestField = tempBestField; + } + + if (bestField == tempBestField) { + bestStamp = lineStamp; + } + } +linesInGroup: + ; + } + } + return (UCalendarDateFields)bestField; +} + +const UFieldResolutionTable Calendar::kDatePrecedence[] = +{ + { + { UCAL_DAY_OF_MONTH, kResolveSTOP }, + { UCAL_WEEK_OF_YEAR, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_WEEK_OF_YEAR, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_DAY_OF_YEAR, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_MONTH, UCAL_YEAR, kResolveSTOP }, // if YEAR is set over YEAR_WOY use DAY_OF_MONTH + { kResolveRemap | UCAL_WEEK_OF_YEAR, UCAL_YEAR_WOY, kResolveSTOP }, // if YEAR_WOY is set, calc based on WEEK_OF_YEAR + { kResolveSTOP } + }, + { + { UCAL_WEEK_OF_YEAR, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { kResolveSTOP } + }, + {{kResolveSTOP}} +}; + + +const UFieldResolutionTable Calendar::kMonthPrecedence[] = +{ + { + { UCAL_MONTH,kResolveSTOP, kResolveSTOP }, + { UCAL_ORDINAL_MONTH,kResolveSTOP, kResolveSTOP }, + {kResolveSTOP} + }, + {{kResolveSTOP}} +}; + +const UFieldResolutionTable Calendar::kDOWPrecedence[] = +{ + { + { UCAL_DAY_OF_WEEK,kResolveSTOP, kResolveSTOP }, + { UCAL_DOW_LOCAL,kResolveSTOP, kResolveSTOP }, + {kResolveSTOP} + }, + {{kResolveSTOP}} +}; + +// precedence for calculating a year +const UFieldResolutionTable Calendar::kYearPrecedence[] = +{ + { + { UCAL_YEAR, kResolveSTOP }, + { UCAL_EXTENDED_YEAR, kResolveSTOP }, + { UCAL_YEAR_WOY, UCAL_WEEK_OF_YEAR, kResolveSTOP }, // YEAR_WOY is useless without WEEK_OF_YEAR + { kResolveSTOP } + }, + {{kResolveSTOP}} +}; + + +// ------------------------- + + +void Calendar::computeTime(UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (!isLenient()) { + validateFields(status); + if (U_FAILURE(status)) { + return; + } + } + + // Compute the Julian day + int32_t julianDay = computeJulianDay(); + + double millis = Grego::julianDayToMillis(julianDay); + +#if defined (U_DEBUG_CAL) + // int32_t julianInsanityCheck = (int32_t)ClockMath::floorDivide(millis, kOneDay); + // julianInsanityCheck += kEpochStartAsJulianDay; + // if(1 || julianInsanityCheck != julianDay) { + // fprintf(stderr, "%s:%d- D'oh- computed jules %d, to mills (%s)%.lf, recomputed %d\n", + // __FILE__, __LINE__, julianDay, millis<0.0?"NEG":"", millis, julianInsanityCheck); + // } +#endif + + double millisInDay; + + // We only use MILLISECONDS_IN_DAY if it has been set by the user. + // This makes it possible for the caller to set the calendar to a + // time and call clear(MONTH) to reset the MONTH to January. This + // is legacy behavior. Without this, clear(MONTH) has no effect, + // since the internally set JULIAN_DAY is used. + if (fStamp[UCAL_MILLISECONDS_IN_DAY] >= ((int32_t)kMinimumUserStamp) && + newestStamp(UCAL_AM_PM, UCAL_MILLISECOND, kUnset) <= fStamp[UCAL_MILLISECONDS_IN_DAY]) { + millisInDay = internalGet(UCAL_MILLISECONDS_IN_DAY); + } else { + millisInDay = computeMillisInDay(); + } + + UDate t = 0; + if (fStamp[UCAL_ZONE_OFFSET] >= ((int32_t)kMinimumUserStamp) || fStamp[UCAL_DST_OFFSET] >= ((int32_t)kMinimumUserStamp)) { + t = millis + millisInDay - (internalGet(UCAL_ZONE_OFFSET) + internalGet(UCAL_DST_OFFSET)); + } else { + // Compute the time zone offset and DST offset. There are two potential + // ambiguities here. We'll assume a 2:00 am (wall time) switchover time + // for discussion purposes here. + // + // 1. The positive offset change such as transition into DST. + // Here, a designated time of 2:00 am - 2:59 am does not actually exist. + // For this case, skippedWallTime option specifies the behavior. + // For example, 2:30 am is interpreted as; + // - WALLTIME_LAST(default): 3:30 am (DST) (interpreting 2:30 am as 31 minutes after 1:59 am (STD)) + // - WALLTIME_FIRST: 1:30 am (STD) (interpreting 2:30 am as 30 minutes before 3:00 am (DST)) + // - WALLTIME_NEXT_VALID: 3:00 am (DST) (next valid time after 2:30 am on a wall clock) + // 2. The negative offset change such as transition out of DST. + // Here, a designated time of 1:00 am - 1:59 am can be in standard or DST. Both are valid + // representations (the rep jumps from 1:59:59 DST to 1:00:00 Std). + // For this case, repeatedWallTime option specifies the behavior. + // For example, 1:30 am is interpreted as; + // - WALLTIME_LAST(default): 1:30 am (STD) - latter occurrence + // - WALLTIME_FIRST: 1:30 am (DST) - former occurrence + // + // In addition to above, when calendar is strict (not default), wall time falls into + // the skipped time range will be processed as an error case. + // + // These special cases are mostly handled in #computeZoneOffset(long), except WALLTIME_NEXT_VALID + // at positive offset change. The protected method computeZoneOffset(long) is exposed to Calendar + // subclass implementations and marked as @stable. Strictly speaking, WALLTIME_NEXT_VALID + // should be also handled in the same place, but we cannot change the code flow without deprecating + // the protected method. + // + // We use the TimeZone object, unless the user has explicitly set the ZONE_OFFSET + // or DST_OFFSET fields; then we use those fields. + + if (!isLenient() || fSkippedWallTime == UCAL_WALLTIME_NEXT_VALID) { + // When strict, invalidate a wall time falls into a skipped wall time range. + // When lenient and skipped wall time option is WALLTIME_NEXT_VALID, + // the result time will be adjusted to the next valid time (on wall clock). + int32_t zoneOffset = computeZoneOffset(millis, millisInDay, status); + UDate tmpTime = millis + millisInDay - zoneOffset; + + int32_t raw, dst; + fZone->getOffset(tmpTime, false, raw, dst, status); + + if (U_SUCCESS(status)) { + // zoneOffset != (raw + dst) only when the given wall time fall into + // a skipped wall time range caused by positive zone offset transition. + if (zoneOffset != (raw + dst)) { + if (!isLenient()) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + U_ASSERT(fSkippedWallTime == UCAL_WALLTIME_NEXT_VALID); + // Adjust time to the next valid wall clock time. + // At this point, tmpTime is on or after the zone offset transition causing + // the skipped time range. + UDate immediatePrevTransition; + UBool hasTransition = getImmediatePreviousZoneTransition(tmpTime, &immediatePrevTransition, status); + if (U_SUCCESS(status) && hasTransition) { + t = immediatePrevTransition; + } + } + } else { + t = tmpTime; + } + } + } else { + t = millis + millisInDay - computeZoneOffset(millis, millisInDay, status); + } + } + if (U_SUCCESS(status)) { + internalSetTime(t); + } +} + +/** + * Find the previous zone transition near the given time. + */ +UBool Calendar::getImmediatePreviousZoneTransition(UDate base, UDate *transitionTime, UErrorCode& status) const { + if (U_FAILURE(status)) { + return false; + } + BasicTimeZone *btz = getBasicTimeZone(); + if (btz) { + TimeZoneTransition trans; + UBool hasTransition = btz->getPreviousTransition(base, true, trans); + if (hasTransition) { + *transitionTime = trans.getTime(); + return true; + } else { + // Could not find any transitions. + // Note: This should never happen. + status = U_INTERNAL_PROGRAM_ERROR; + } + } else { + // If not BasicTimeZone, return unsupported error for now. + // TODO: We may support non-BasicTimeZone in future. + status = U_UNSUPPORTED_ERROR; + } + return false; +} + +/** +* Compute the milliseconds in the day from the fields. This is a +* value from 0 to 23:59:59.999 inclusive, unless fields are out of +* range, in which case it can be an arbitrary value. This value +* reflects local zone wall time. +* @stable ICU 2.0 +*/ +double Calendar::computeMillisInDay() { + // Do the time portion of the conversion. + + double millisInDay = 0; + + // Find the best set of fields specifying the time of day. There + // are only two possibilities here; the HOUR_OF_DAY or the + // AM_PM and the HOUR. + int32_t hourOfDayStamp = fStamp[UCAL_HOUR_OF_DAY]; + int32_t hourStamp = (fStamp[UCAL_HOUR] > fStamp[UCAL_AM_PM])?fStamp[UCAL_HOUR]:fStamp[UCAL_AM_PM]; + int32_t bestStamp = (hourStamp > hourOfDayStamp) ? hourStamp : hourOfDayStamp; + + // Hours + if (bestStamp != kUnset) { + if (bestStamp == hourOfDayStamp) { + // Don't normalize here; let overflow bump into the next period. + // This is consistent with how we handle other fields. + millisInDay += internalGet(UCAL_HOUR_OF_DAY); + } else { + // Don't normalize here; let overflow bump into the next period. + // This is consistent with how we handle other fields. + millisInDay += internalGet(UCAL_HOUR); + millisInDay += 12 * internalGet(UCAL_AM_PM); // Default works for unset AM_PM + } + } + + // We use the fact that unset == 0; we start with millisInDay + // == HOUR_OF_DAY. + millisInDay *= 60; + millisInDay += internalGet(UCAL_MINUTE); // now have minutes + millisInDay *= 60; + millisInDay += internalGet(UCAL_SECOND); // now have seconds + millisInDay *= 1000; + millisInDay += internalGet(UCAL_MILLISECOND); // now have millis + + return millisInDay; +} + +/** +* This method can assume EXTENDED_YEAR has been set. +* @param millis milliseconds of the date fields +* @param millisInDay milliseconds of the time fields; may be out +* or range. +* @stable ICU 2.0 +*/ +int32_t Calendar::computeZoneOffset(double millis, double millisInDay, UErrorCode &ec) { + if (U_FAILURE(ec)) { + return 0; + } + int32_t rawOffset, dstOffset; + UDate wall = millis + millisInDay; + BasicTimeZone* btz = getBasicTimeZone(); + if (btz) { + UTimeZoneLocalOption duplicatedTimeOpt = (fRepeatedWallTime == UCAL_WALLTIME_FIRST) ? UCAL_TZ_LOCAL_FORMER : UCAL_TZ_LOCAL_LATTER; + UTimeZoneLocalOption nonExistingTimeOpt = (fSkippedWallTime == UCAL_WALLTIME_FIRST) ? UCAL_TZ_LOCAL_LATTER : UCAL_TZ_LOCAL_FORMER; + btz->getOffsetFromLocal(wall, nonExistingTimeOpt, duplicatedTimeOpt, rawOffset, dstOffset, ec); + } else { + const TimeZone& tz = getTimeZone(); + // By default, TimeZone::getOffset behaves UCAL_WALLTIME_LAST for both. + tz.getOffset(wall, true, rawOffset, dstOffset, ec); + + UBool sawRecentNegativeShift = false; + if (fRepeatedWallTime == UCAL_WALLTIME_FIRST) { + // Check if the given wall time falls into repeated time range + UDate tgmt = wall - (rawOffset + dstOffset); + + // Any negative zone transition within last 6 hours? + // Note: The maximum historic negative zone transition is -3 hours in the tz database. + // 6 hour window would be sufficient for this purpose. + int32_t tmpRaw, tmpDst; + tz.getOffset(tgmt - 6*60*60*1000, false, tmpRaw, tmpDst, ec); + int32_t offsetDelta = (rawOffset + dstOffset) - (tmpRaw + tmpDst); + + U_ASSERT(offsetDelta < -6*60*60*1000); + if (offsetDelta < 0) { + sawRecentNegativeShift = true; + // Negative shift within last 6 hours. When UCAL_WALLTIME_FIRST is used and the given wall time falls + // into the repeated time range, use offsets before the transition. + // Note: If it does not fall into the repeated time range, offsets remain unchanged below. + tz.getOffset(wall + offsetDelta, true, rawOffset, dstOffset, ec); + } + } + if (!sawRecentNegativeShift && fSkippedWallTime == UCAL_WALLTIME_FIRST) { + // When skipped wall time option is WALLTIME_FIRST, + // recalculate offsets from the resolved time (non-wall). + // When the given wall time falls into skipped wall time, + // the offsets will be based on the zone offsets AFTER + // the transition (which means, earliest possible interpretation). + UDate tgmt = wall - (rawOffset + dstOffset); + tz.getOffset(tgmt, false, rawOffset, dstOffset, ec); + } + } + return rawOffset + dstOffset; +} + +int32_t Calendar::computeJulianDay() +{ + // We want to see if any of the date fields is newer than the + // JULIAN_DAY. If not, then we use JULIAN_DAY. If so, then we do + // the normal resolution. We only use JULIAN_DAY if it has been + // set by the user. This makes it possible for the caller to set + // the calendar to a time and call clear(MONTH) to reset the MONTH + // to January. This is legacy behavior. Without this, + // clear(MONTH) has no effect, since the internally set JULIAN_DAY + // is used. + if (fStamp[UCAL_JULIAN_DAY] >= (int32_t)kMinimumUserStamp) { + int32_t bestStamp = newestStamp(UCAL_ERA, UCAL_DAY_OF_WEEK_IN_MONTH, kUnset); + bestStamp = newestStamp(UCAL_YEAR_WOY, UCAL_EXTENDED_YEAR, bestStamp); + bestStamp = newestStamp(UCAL_ORDINAL_MONTH, UCAL_ORDINAL_MONTH, bestStamp); + if (bestStamp <= fStamp[UCAL_JULIAN_DAY]) { + return internalGet(UCAL_JULIAN_DAY); + } + } + + UCalendarDateFields bestField = resolveFields(getFieldResolutionTable()); + if (bestField == UCAL_FIELD_COUNT) { + bestField = UCAL_DAY_OF_MONTH; + } + + return handleComputeJulianDay(bestField); +} + +// ------------------------------------------- + +int32_t Calendar::handleComputeJulianDay(UCalendarDateFields bestField) { + UBool useMonth = (bestField == UCAL_DAY_OF_MONTH || + bestField == UCAL_WEEK_OF_MONTH || + bestField == UCAL_DAY_OF_WEEK_IN_MONTH); + int32_t year; + + if (bestField == UCAL_WEEK_OF_YEAR && newerField(UCAL_YEAR_WOY, UCAL_YEAR) == UCAL_YEAR_WOY) { + year = internalGet(UCAL_YEAR_WOY); + } else { + year = handleGetExtendedYear(); + } + + internalSet(UCAL_EXTENDED_YEAR, year); + +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: bestField= %s - y=%d\n", __FILE__, __LINE__, fldName(bestField), year); +#endif + + // Get the Julian day of the day BEFORE the start of this year. + // If useMonth is true, get the day before the start of the month. + + // give calendar subclass a chance to have a default 'first' month + int32_t month; + + if(isSet(UCAL_MONTH) || isSet(UCAL_ORDINAL_MONTH)) { + month = internalGetMonth(); + } else { + month = getDefaultMonthInYear(year); + } + + int32_t julianDay = handleComputeMonthStart(year, useMonth ? month : 0, useMonth); + + if (bestField == UCAL_DAY_OF_MONTH) { + + // give calendar subclass a chance to have a default 'first' dom + int32_t dayOfMonth; + if(isSet(UCAL_DAY_OF_MONTH)) { + dayOfMonth = internalGet(UCAL_DAY_OF_MONTH,1); + } else { + dayOfMonth = getDefaultDayInMonth(year, month); + } + return julianDay + dayOfMonth; + } + + if (bestField == UCAL_DAY_OF_YEAR) { + return julianDay + internalGet(UCAL_DAY_OF_YEAR); + } + + int32_t firstDayOfWeek = getFirstDayOfWeek(); // Localized fdw + + // At this point julianDay is the 0-based day BEFORE the first day of + // January 1, year 1 of the given calendar. If julianDay == 0, it + // specifies (Jan. 1, 1) - 1, in whatever calendar we are using (Julian + // or Gregorian). (or it is before the month we are in, if useMonth is True) + + // At this point we need to process the WEEK_OF_MONTH or + // WEEK_OF_YEAR, which are similar, or the DAY_OF_WEEK_IN_MONTH. + // First, perform initial shared computations. These locate the + // first week of the period. + + // Get the 0-based localized DOW of day one of the month or year. + // Valid range 0..6. + int32_t first = julianDayToDayOfWeek(julianDay + 1) - firstDayOfWeek; + if (first < 0) { + first += 7; + } + + int32_t dowLocal = getLocalDOW(); + + // Find the first target DOW (dowLocal) in the month or year. + // Actually, it may be just before the first of the month or year. + // It will be an integer from -5..7. + int32_t date = 1 - first + dowLocal; + + if (bestField == UCAL_DAY_OF_WEEK_IN_MONTH) { + // Adjust the target DOW to be in the month or year. + if (date < 1) { + date += 7; + } + + // The only trickiness occurs if the day-of-week-in-month is + // negative. + int32_t dim = internalGet(UCAL_DAY_OF_WEEK_IN_MONTH, 1); + if (dim >= 0) { + date += 7*(dim - 1); + + } else { + // Move date to the last of this day-of-week in this month, + // then back up as needed. If dim==-1, we don't back up at + // all. If dim==-2, we back up once, etc. Don't back up + // past the first of the given day-of-week in this month. + // Note that we handle -2, -3, etc. correctly, even though + // values < -1 are technically disallowed. + int32_t m = internalGetMonth(UCAL_JANUARY); + int32_t monthLength = handleGetMonthLength(year, m); + date += ((monthLength - date) / 7 + dim + 1) * 7; + } + } else { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - bf= %s\n", __FILE__, __LINE__, fldName(bestField)); +#endif + + if(bestField == UCAL_WEEK_OF_YEAR) { // ------------------------------------- WOY ------------- + if(!isSet(UCAL_YEAR_WOY) || // YWOY not set at all or + ( (resolveFields(kYearPrecedence) != UCAL_YEAR_WOY) // YWOY doesn't have precedence + && (fStamp[UCAL_YEAR_WOY]!=kInternallySet) ) ) // (excluding where all fields are internally set - then YWOY is used) + { + // need to be sure to stay in 'real' year. + int32_t woy = internalGet(bestField); + + int32_t nextJulianDay = handleComputeMonthStart(year+1, 0, false); // jd of day before jan 1 + int32_t nextFirst = julianDayToDayOfWeek(nextJulianDay + 1) - firstDayOfWeek; + + if (nextFirst < 0) { // 0..6 ldow of Jan 1 + nextFirst += 7; + } + + if(woy==1) { // FIRST WEEK --------------------------------- +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - woy=%d, yp=%d, nj(%d)=%d, nf=%d", __FILE__, __LINE__, + internalGet(bestField), resolveFields(kYearPrecedence), year+1, + nextJulianDay, nextFirst); + + fprintf(stderr, " next: %d DFW, min=%d \n", (7-nextFirst), getMinimalDaysInFirstWeek() ); +#endif + + // nextFirst is now the localized DOW of Jan 1 of y-woy+1 + if((nextFirst > 0) && // Jan 1 starts on FDOW + (7-nextFirst) >= getMinimalDaysInFirstWeek()) // or enough days in the week + { + // Jan 1 of (yearWoy+1) is in yearWoy+1 - recalculate JD to next year +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - was going to move JD from %d to %d [d%d]\n", __FILE__, __LINE__, + julianDay, nextJulianDay, (nextJulianDay-julianDay)); +#endif + julianDay = nextJulianDay; + + // recalculate 'first' [0-based local dow of jan 1] + first = julianDayToDayOfWeek(julianDay + 1) - firstDayOfWeek; + if (first < 0) { + first += 7; + } + // recalculate date. + date = 1 - first + dowLocal; + } + } else if(woy>=getLeastMaximum(bestField)) { + // could be in the last week- find out if this JD would overstep + int32_t testDate = date; + if ((7 - first) < getMinimalDaysInFirstWeek()) { + testDate += 7; + } + + // Now adjust for the week number. + testDate += 7 * (woy - 1); + +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - y=%d, y-1=%d doy%d, njd%d (C.F. %d)\n", + __FILE__, __LINE__, year, year-1, testDate, julianDay+testDate, nextJulianDay); +#endif + if(julianDay+testDate > nextJulianDay) { // is it past Dec 31? (nextJulianDay is day BEFORE year+1's Jan 1) + // Fire up the calculating engines.. retry YWOY = (year-1) + julianDay = handleComputeMonthStart(year-1, 0, false); // jd before Jan 1 of previous year + first = julianDayToDayOfWeek(julianDay + 1) - firstDayOfWeek; // 0 based local dow of first week + + if(first < 0) { // 0..6 + first += 7; + } + date = 1 - first + dowLocal; + +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - date now %d, jd%d, ywoy%d\n", + __FILE__, __LINE__, date, julianDay, year-1); +#endif + + + } /* correction needed */ + } /* leastmaximum */ + } /* resolvefields(year) != year_woy */ + } /* bestfield != week_of_year */ + + // assert(bestField == WEEK_OF_MONTH || bestField == WEEK_OF_YEAR) + // Adjust for minimal days in first week + if ((7 - first) < getMinimalDaysInFirstWeek()) { + date += 7; + } + + // Now adjust for the week number. + date += 7 * (internalGet(bestField) - 1); + } + + return julianDay + date; +} + +int32_t +Calendar::getDefaultMonthInYear(int32_t /*eyear*/) +{ + return 0; +} + +int32_t +Calendar::getDefaultDayInMonth(int32_t /*eyear*/, int32_t /*month*/) +{ + return 1; +} + + +int32_t Calendar::getLocalDOW() +{ + // Get zero-based localized DOW, valid range 0..6. This is the DOW + // we are looking for. + int32_t dowLocal = 0; + switch (resolveFields(kDOWPrecedence)) { + case UCAL_DAY_OF_WEEK: + dowLocal = internalGet(UCAL_DAY_OF_WEEK) - fFirstDayOfWeek; + break; + case UCAL_DOW_LOCAL: + dowLocal = internalGet(UCAL_DOW_LOCAL) - 1; + break; + default: + break; + } + dowLocal = dowLocal % 7; + if (dowLocal < 0) { + dowLocal += 7; + } + return dowLocal; +} + +int32_t Calendar::handleGetExtendedYearFromWeekFields(int32_t yearWoy, int32_t woy) +{ + // We have UCAL_YEAR_WOY and UCAL_WEEK_OF_YEAR - from those, determine + // what year we fall in, so that other code can set it properly. + // (code borrowed from computeWeekFields and handleComputeJulianDay) + //return yearWoy; + + // First, we need a reliable DOW. + UCalendarDateFields bestField = resolveFields(kDatePrecedence); // !! Note: if subclasses have a different table, they should override handleGetExtendedYearFromWeekFields + + // Now, a local DOW + int32_t dowLocal = getLocalDOW(); // 0..6 + int32_t firstDayOfWeek = getFirstDayOfWeek(); // Localized fdw + int32_t jan1Start = handleComputeMonthStart(yearWoy, 0, false); + int32_t nextJan1Start = handleComputeMonthStart(yearWoy+1, 0, false); // next year's Jan1 start + + // At this point julianDay is the 0-based day BEFORE the first day of + // January 1, year 1 of the given calendar. If julianDay == 0, it + // specifies (Jan. 1, 1) - 1, in whatever calendar we are using (Julian + // or Gregorian). (or it is before the month we are in, if useMonth is True) + + // At this point we need to process the WEEK_OF_MONTH or + // WEEK_OF_YEAR, which are similar, or the DAY_OF_WEEK_IN_MONTH. + // First, perform initial shared computations. These locate the + // first week of the period. + + // Get the 0-based localized DOW of day one of the month or year. + // Valid range 0..6. + int32_t first = julianDayToDayOfWeek(jan1Start + 1) - firstDayOfWeek; + if (first < 0) { + first += 7; + } + + //// (nextFirst was not used below) + // int32_t nextFirst = julianDayToDayOfWeek(nextJan1Start + 1) - firstDayOfWeek; + // if (nextFirst < 0) { + // nextFirst += 7; + //} + + int32_t minDays = getMinimalDaysInFirstWeek(); + UBool jan1InPrevYear = false; // January 1st in the year of WOY is the 1st week? (i.e. first week is < minimal ) + //UBool nextJan1InPrevYear = false; // January 1st of Year of WOY + 1 is in the first week? + + if((7 - first) < minDays) { + jan1InPrevYear = true; + } + + // if((7 - nextFirst) < minDays) { + // nextJan1InPrevYear = true; + // } + + switch(bestField) { + case UCAL_WEEK_OF_YEAR: + if(woy == 1) { + if(jan1InPrevYear) { + // the first week of January is in the previous year + // therefore WOY1 is always solidly within yearWoy + return yearWoy; + } else { + // First WOY is split between two years + if( dowLocal < first) { // we are prior to Jan 1 + return yearWoy-1; // previous year + } else { + return yearWoy; // in this year + } + } + } else if(woy >= getLeastMaximum(bestField)) { + // we _might_ be in the last week.. + int32_t jd = // Calculate JD of our target day: + jan1Start + // JD of Jan 1 + (7-first) + // days in the first week (Jan 1.. ) + (woy-1)*7 + // add the weeks of the year + dowLocal; // the local dow (0..6) of last week + if(jan1InPrevYear==false) { + jd -= 7; // woy already includes Jan 1's week. + } + + if( (jd+1) >= nextJan1Start ) { + // we are in week 52 or 53 etc. - actual year is yearWoy+1 + return yearWoy+1; + } else { + // still in yearWoy; + return yearWoy; + } + } else { + // we're not possibly in the last week -must be ywoy + return yearWoy; + } + + case UCAL_DATE: + { + int32_t m = internalGetMonth(); + if((m == 0) && + (woy >= getLeastMaximum(UCAL_WEEK_OF_YEAR))) { + return yearWoy+1; // month 0, late woy = in the next year + } else if(woy==1) { + //if(nextJan1InPrevYear) { + if(m == 0) { + return yearWoy; + } else { + return yearWoy-1; + } + //} + } + } + //(internalGet(UCAL_DATE) <= (7-first)) /* && in minDow */ ) { + //within 1st week and in this month.. + //return yearWoy+1; + return yearWoy; + + default: // assume the year is appropriate + return yearWoy; + } +} + +int32_t Calendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const +{ + return handleComputeMonthStart(extendedYear, month+1, true) - + handleComputeMonthStart(extendedYear, month, true); +} + +int32_t Calendar::handleGetYearLength(int32_t eyear) const { + return handleComputeMonthStart(eyear+1, 0, false) - + handleComputeMonthStart(eyear, 0, false); +} + +int32_t +Calendar::getActualMaximum(UCalendarDateFields field, UErrorCode& status) const +{ + if (U_FAILURE(status)) { + return 0; + } + int32_t result; + switch (field) { + case UCAL_DATE: + { + if(U_FAILURE(status)) return 0; + Calendar *cal = clone(); + if(!cal) { status = U_MEMORY_ALLOCATION_ERROR; return 0; } + cal->setLenient(true); + cal->prepareGetActual(field,false,status); + result = handleGetMonthLength(cal->get(UCAL_EXTENDED_YEAR, status), cal->get(UCAL_MONTH, status)); + delete cal; + } + break; + + case UCAL_DAY_OF_YEAR: + { + if(U_FAILURE(status)) return 0; + Calendar *cal = clone(); + if(!cal) { status = U_MEMORY_ALLOCATION_ERROR; return 0; } + cal->setLenient(true); + cal->prepareGetActual(field,false,status); + result = handleGetYearLength(cal->get(UCAL_EXTENDED_YEAR, status)); + delete cal; + } + break; + + case UCAL_DAY_OF_WEEK: + case UCAL_AM_PM: + case UCAL_HOUR: + case UCAL_HOUR_OF_DAY: + case UCAL_MINUTE: + case UCAL_SECOND: + case UCAL_MILLISECOND: + case UCAL_ZONE_OFFSET: + case UCAL_DST_OFFSET: + case UCAL_DOW_LOCAL: + case UCAL_JULIAN_DAY: + case UCAL_MILLISECONDS_IN_DAY: + // These fields all have fixed minima/maxima + result = getMaximum(field); + break; + + case UCAL_ORDINAL_MONTH: + result = inTemporalLeapYear(status) ? getMaximum(UCAL_ORDINAL_MONTH) : getLeastMaximum(UCAL_ORDINAL_MONTH); + break; + + default: + // For all other fields, do it the hard way.... + result = getActualHelper(field, getLeastMaximum(field), getMaximum(field),status); + break; + } + return result; +} + + +/** +* Prepare this calendar for computing the actual minimum or maximum. +* This method modifies this calendar's fields; it is called on a +* temporary calendar. +* +*

Rationale: The semantics of getActualXxx() is to return the +* maximum or minimum value that the given field can take, taking into +* account other relevant fields. In general these other fields are +* larger fields. For example, when computing the actual maximum +* DATE, the current value of DATE itself is ignored, +* as is the value of any field smaller. +* +*

The time fields all have fixed minima and maxima, so we don't +* need to worry about them. This also lets us set the +* MILLISECONDS_IN_DAY to zero to erase any effects the time fields +* might have when computing date fields. +* +*

DAY_OF_WEEK is adjusted specially for the WEEK_OF_MONTH and +* WEEK_OF_YEAR fields to ensure that they are computed correctly. +* @internal +*/ +void Calendar::prepareGetActual(UCalendarDateFields field, UBool isMinimum, UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + set(UCAL_MILLISECONDS_IN_DAY, 0); + + switch (field) { + case UCAL_YEAR: + case UCAL_EXTENDED_YEAR: + set(UCAL_DAY_OF_YEAR, getGreatestMinimum(UCAL_DAY_OF_YEAR)); + break; + + case UCAL_YEAR_WOY: + set(UCAL_WEEK_OF_YEAR, getGreatestMinimum(UCAL_WEEK_OF_YEAR)); + U_FALLTHROUGH; + case UCAL_MONTH: + set(UCAL_DATE, getGreatestMinimum(UCAL_DATE)); + break; + + case UCAL_DAY_OF_WEEK_IN_MONTH: + // For dowim, the maximum occurs for the DOW of the first of the + // month. + set(UCAL_DATE, 1); + set(UCAL_DAY_OF_WEEK, get(UCAL_DAY_OF_WEEK, status)); // Make this user set + break; + + case UCAL_WEEK_OF_MONTH: + case UCAL_WEEK_OF_YEAR: + // If we're counting weeks, set the day of the week to either the + // first or last localized DOW. We know the last week of a month + // or year will contain the first day of the week, and that the + // first week will contain the last DOW. + { + int32_t dow = fFirstDayOfWeek; + if (isMinimum) { + dow = (dow + 6) % 7; // set to last DOW + if (dow < UCAL_SUNDAY) { + dow += 7; + } + } +#if defined (U_DEBUG_CAL) + fprintf(stderr, "prepareGetActualHelper(WOM/WOY) - dow=%d\n", dow); +#endif + set(UCAL_DAY_OF_WEEK, dow); + } + break; + default: + break; + } + + // Do this last to give it the newest time stamp + set(field, getGreatestMinimum(field)); +} + +int32_t Calendar::getActualHelper(UCalendarDateFields field, int32_t startValue, int32_t endValue, UErrorCode &status) const +{ +#if defined (U_DEBUG_CAL) + fprintf(stderr, "getActualHelper(%d,%d .. %d, %s)\n", field, startValue, endValue, u_errorName(status)); +#endif + if (U_FAILURE(status)) { + return 0; + } + if (startValue == endValue) { + // if we know that the maximum value is always the same, just return it + return startValue; + } + + int32_t delta = (endValue > startValue) ? 1 : -1; + + // clone the calendar so we don't mess with the real one, and set it to + // accept anything for the field values + if(U_FAILURE(status)) return startValue; + Calendar *work = clone(); + if(!work) { status = U_MEMORY_ALLOCATION_ERROR; return startValue; } + + // need to resolve time here, otherwise, fields set for actual limit + // may cause conflict with fields previously set (but not yet resolved). + work->complete(status); + + work->setLenient(true); + work->prepareGetActual(field, delta < 0, status); + + // now try each value from the start to the end one by one until + // we get a value that normalizes to another value. The last value that + // normalizes to itself is the actual maximum for the current date + work->set(field, startValue); + + // prepareGetActual sets the first day of week in the same week with + // the first day of a month. Unlike WEEK_OF_YEAR, week number for the + // week which contains days from both previous and current month is + // not unique. For example, last several days in the previous month + // is week 5, and the rest of week is week 1. + int32_t result = startValue; + if ((work->get(field, status) != startValue + && field != UCAL_WEEK_OF_MONTH && delta > 0 ) || U_FAILURE(status)) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "getActualHelper(fld %d) - got %d (not %d) - %s\n", field, work->get(field,status), startValue, u_errorName(status)); +#endif + } else { + do { + startValue += delta; + work->add(field, delta, status); + if (work->get(field, status) != startValue || U_FAILURE(status)) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "getActualHelper(fld %d) - got %d (not %d), BREAK - %s\n", field, work->get(field,status), startValue, u_errorName(status)); +#endif + break; + } + result = startValue; + } while (startValue != endValue); + } + delete work; +#if defined (U_DEBUG_CAL) + fprintf(stderr, "getActualHelper(%d) = %d\n", field, result); +#endif + return result; +} + + + + +// ------------------------------------- + +void +Calendar::setWeekData(const Locale& desiredLocale, const char *type, UErrorCode& status) +{ + + if (U_FAILURE(status)) return; + + fFirstDayOfWeek = UCAL_SUNDAY; + fMinimalDaysInFirstWeek = 1; + fWeekendOnset = UCAL_SATURDAY; + fWeekendOnsetMillis = 0; + fWeekendCease = UCAL_SUNDAY; + fWeekendCeaseMillis = 86400000; // 24*60*60*1000 + + // Since week and weekend data is territory based instead of language based, + // we may need to tweak the locale that we are using to try to get the appropriate + // values, using the following logic: + // 1). If the locale has a language but no territory, use the territory as defined by + // the likely subtags. + // 2). If the locale has a script designation then we ignore it, + // then remove it ( i.e. "en_Latn_US" becomes "en_US" ) + + UErrorCode myStatus = U_ZERO_ERROR; + + Locale min(desiredLocale); + min.minimizeSubtags(myStatus); + Locale useLocale; + if ( uprv_strlen(desiredLocale.getCountry()) == 0 || + (uprv_strlen(desiredLocale.getScript()) > 0 && uprv_strlen(min.getScript()) == 0) ) { + myStatus = U_ZERO_ERROR; + Locale max(desiredLocale); + max.addLikelySubtags(myStatus); + useLocale = Locale(max.getLanguage(),max.getCountry()); + } else { + useLocale = desiredLocale; + } + + /* The code here is somewhat of a hack, since week data and weekend data aren't really tied to + a specific calendar, they aren't truly locale data. But this is the only place where valid and + actual locale can be set, so we take a shot at it here by loading a representative resource + from the calendar data. The code used to use the dateTimeElements resource to get first day + of week data, but this was moved to supplemental data under ticket 7755. (JCE) */ + + // Get the monthNames resource bundle for the calendar 'type'. Fallback to gregorian if the resource is not + // found. + LocalUResourceBundlePointer calData(ures_open(nullptr, useLocale.getBaseName(), &status)); + ures_getByKey(calData.getAlias(), gCalendar, calData.getAlias(), &status); + + LocalUResourceBundlePointer monthNames; + if (type != nullptr && *type != '\0' && uprv_strcmp(type, gGregorian) != 0) { + monthNames.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), type, nullptr, &status)); + ures_getByKeyWithFallback(monthNames.getAlias(), gMonthNames, + monthNames.getAlias(), &status); + } + + if (monthNames.isNull() || status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + monthNames.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), gGregorian, + monthNames.orphan(), &status)); + ures_getByKeyWithFallback(monthNames.getAlias(), gMonthNames, + monthNames.getAlias(), &status); + } + + if (U_SUCCESS(status)) { + U_LOCALE_BASED(locBased,*this); + locBased.setLocaleIDs(ures_getLocaleByType(monthNames.getAlias(), ULOC_VALID_LOCALE, &status), + ures_getLocaleByType(monthNames.getAlias(), ULOC_ACTUAL_LOCALE, &status)); + } else { + status = U_USING_FALLBACK_WARNING; + return; + } + + char region[ULOC_COUNTRY_CAPACITY]; + (void)ulocimp_getRegionForSupplementalData(desiredLocale.getName(), true, region, sizeof(region), &status); + + // Read week data values from supplementalData week data + UResourceBundle *rb = ures_openDirect(nullptr, "supplementalData", &status); + ures_getByKey(rb, "weekData", rb, &status); + UResourceBundle *weekData = ures_getByKey(rb, region, nullptr, &status); + if (status == U_MISSING_RESOURCE_ERROR && rb != nullptr) { + status = U_ZERO_ERROR; + weekData = ures_getByKey(rb, "001", nullptr, &status); + } + + if (U_FAILURE(status)) { + status = U_USING_FALLBACK_WARNING; + } else { + int32_t arrLen; + const int32_t *weekDataArr = ures_getIntVector(weekData,&arrLen,&status); + if( U_SUCCESS(status) && arrLen == 6 + && 1 <= weekDataArr[0] && weekDataArr[0] <= 7 + && 1 <= weekDataArr[1] && weekDataArr[1] <= 7 + && 1 <= weekDataArr[2] && weekDataArr[2] <= 7 + && 1 <= weekDataArr[4] && weekDataArr[4] <= 7) { + fFirstDayOfWeek = (UCalendarDaysOfWeek)weekDataArr[0]; + fMinimalDaysInFirstWeek = (uint8_t)weekDataArr[1]; + fWeekendOnset = (UCalendarDaysOfWeek)weekDataArr[2]; + fWeekendOnsetMillis = weekDataArr[3]; + fWeekendCease = (UCalendarDaysOfWeek)weekDataArr[4]; + fWeekendCeaseMillis = weekDataArr[5]; + } else { + status = U_INVALID_FORMAT_ERROR; + } + + // Check if the locale has a "fw" u extension and we honor it if present. + // And we don't change the overal status, as the presence / lack of "fw" is not an error. + UErrorCode fwStatus = U_ZERO_ERROR; + char fwExt[ULOC_FULLNAME_CAPACITY] = ""; + desiredLocale.getKeywordValue("fw", fwExt, ULOC_FULLNAME_CAPACITY, fwStatus); + if (U_SUCCESS(fwStatus)) { + if (uprv_strcmp(fwExt, "sun") == 0) { + fFirstDayOfWeek = UCAL_SUNDAY; + } else if (uprv_strcmp(fwExt, "mon") == 0) { + fFirstDayOfWeek = UCAL_MONDAY; + } else if (uprv_strcmp(fwExt, "tue") == 0) { + fFirstDayOfWeek = UCAL_TUESDAY; + } else if (uprv_strcmp(fwExt, "wed") == 0) { + fFirstDayOfWeek = UCAL_WEDNESDAY; + } else if (uprv_strcmp(fwExt, "thu") == 0) { + fFirstDayOfWeek = UCAL_THURSDAY; + } else if (uprv_strcmp(fwExt, "fri") == 0) { + fFirstDayOfWeek = UCAL_FRIDAY; + } else if (uprv_strcmp(fwExt, "sat") == 0) { + fFirstDayOfWeek = UCAL_SATURDAY; + } + } + } + ures_close(weekData); + ures_close(rb); +} + +/** +* Recompute the time and update the status fields isTimeSet +* and areFieldsSet. Callers should check isTimeSet and only +* call this method if isTimeSet is false. +*/ +void +Calendar::updateTime(UErrorCode& status) +{ + computeTime(status); + if(U_FAILURE(status)) + return; + + // If we are lenient, we need to recompute the fields to normalize + // the values. Also, if we haven't set all the fields yet (i.e., + // in a newly-created object), we need to fill in the fields. [LIU] + if (isLenient() || ! fAreAllFieldsSet) + fAreFieldsSet = false; + + fIsTimeSet = true; + fAreFieldsVirtuallySet = false; +} + +Locale +Calendar::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocale(type, status); +} + +const char * +Calendar::getLocaleID(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocaleID(type, status); +} + +void +Calendar::recalculateStamp() { + int32_t index; + int32_t currentValue; + int32_t j, i; + + fNextStamp = 1; + + for (j = 0; j < UCAL_FIELD_COUNT; j++) { + currentValue = STAMP_MAX; + index = -1; + for (i = 0; i < UCAL_FIELD_COUNT; i++) { + if (fStamp[i] > fNextStamp && fStamp[i] < currentValue) { + currentValue = fStamp[i]; + index = i; + } + } + + if (index >= 0) { + fStamp[index] = ++fNextStamp; + } else { + break; + } + } + fNextStamp++; +} + +// Deprecated function. This doesn't need to be inline. +void +Calendar::internalSet(EDateFields field, int32_t value) +{ + internalSet((UCalendarDateFields) field, value); +} + +int32_t Calendar::internalGetMonth() const { + if (resolveFields(kMonthPrecedence) == UCAL_MONTH) { + return internalGet(UCAL_MONTH); + } + return internalGet(UCAL_ORDINAL_MONTH); +} + +int32_t Calendar::internalGetMonth(int32_t defaultValue) const { + if (resolveFields(kMonthPrecedence) == UCAL_MONTH) { + return internalGet(UCAL_MONTH, defaultValue); + } + return internalGet(UCAL_ORDINAL_MONTH); +} + +BasicTimeZone* +Calendar::getBasicTimeZone() const { + if (dynamic_cast(fZone) != nullptr + || dynamic_cast(fZone) != nullptr + || dynamic_cast(fZone) != nullptr + || dynamic_cast(fZone) != nullptr) { + return (BasicTimeZone*)fZone; + } + return nullptr; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + + +//eof diff --git a/intl/icu/source/i18n/casetrn.cpp b/intl/icu/source/i18n/casetrn.cpp new file mode 100644 index 0000000000..2f9699ee9d --- /dev/null +++ b/intl/icu/source/i18n/casetrn.cpp @@ -0,0 +1,193 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2001-2011, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: casetrn.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep03 +* created by: Markus W. Scherer +* +* Implementation class for lower-/upper-/title-casing transliterators. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/utf.h" +#include "unicode/utf16.h" +#include "tolowtrn.h" +#include "ucase.h" +#include "cpputils.h" + +/* case context iterator using a Replaceable */ +U_CFUNC UChar32 U_CALLCONV +utrans_rep_caseContextIterator(void *context, int8_t dir) +{ + U_NAMESPACE_USE + + UCaseContext *csc=(UCaseContext *)context; + Replaceable *rep=(Replaceable *)csc->p; + UChar32 c; + + if(dir<0) { + /* reset for backward iteration */ + csc->index=csc->cpStart; + csc->dir=dir; + } else if(dir>0) { + /* reset for forward iteration */ + csc->index=csc->cpLimit; + csc->dir=dir; + } else { + /* continue current iteration direction */ + dir=csc->dir; + } + + // automatically adjust start and limit if the Replaceable disagrees + // with the original values + if(dir<0) { + if(csc->startindex) { + c=rep->char32At(csc->index-1); + if(c<0) { + csc->start=csc->index; + } else { + csc->index-=U16_LENGTH(c); + return c; + } + } + } else { + // detect, and store in csc->b1, if we hit the limit + if(csc->indexlimit) { + c=rep->char32At(csc->index); + if(c<0) { + csc->limit=csc->index; + csc->b1=true; + } else { + csc->index+=U16_LENGTH(c); + return c; + } + } else { + csc->b1=true; + } + } + return U_SENTINEL; +} + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(CaseMapTransliterator) + +/** + * Constructs a transliterator. + */ +CaseMapTransliterator::CaseMapTransliterator(const UnicodeString &id, UCaseMapFull *map) : + Transliterator(id, 0), + fMap(map) +{ + // TODO test incremental mode with context-sensitive text (e.g. greek sigma) + // TODO need to call setMaximumContextLength()?! +} + +/** + * Destructor. + */ +CaseMapTransliterator::~CaseMapTransliterator() { +} + +/** + * Copy constructor. + */ +CaseMapTransliterator::CaseMapTransliterator(const CaseMapTransliterator& o) : + Transliterator(o), + fMap(o.fMap) +{ +} + +/** + * Assignment operator. + */ +/*CaseMapTransliterator& CaseMapTransliterator::operator=(const CaseMapTransliterator& o) { + Transliterator::operator=(o); + fMap = o.fMap; + return *this; +}*/ + +/** + * Transliterator API. + */ +/*CaseMapTransliterator* CaseMapTransliterator::clone() const { + return new CaseMapTransliterator(*this); +}*/ + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void CaseMapTransliterator::handleTransliterate(Replaceable& text, + UTransPosition& offsets, + UBool isIncremental) const +{ + if (offsets.start >= offsets.limit) { + return; + } + + UCaseContext csc; + uprv_memset(&csc, 0, sizeof(csc)); + csc.p = &text; + csc.start = offsets.contextStart; + csc.limit = offsets.contextLimit; + + UnicodeString tmp; + const char16_t *s; + UChar32 c; + int32_t textPos, delta, result; + + for(textPos=offsets.start; textPos=0) { + // replace the current code point with its full case mapping result + // see UCASE_MAX_STRING_LENGTH + if(result<=UCASE_MAX_STRING_LENGTH) { + // string s[result] + tmp.setTo(false, s, result); + delta=result-U16_LENGTH(c); + } else { + // single code point + tmp.setTo(result); + delta=tmp.length()-U16_LENGTH(c); + } + text.handleReplaceBetween(csc.cpStart, textPos, tmp); + if(delta!=0) { + textPos+=delta; + csc.limit=offsets.contextLimit+=delta; + offsets.limit+=delta; + } + } + } + offsets.start=textPos; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/casetrn.h b/intl/icu/source/i18n/casetrn.h new file mode 100644 index 0000000000..a00480db60 --- /dev/null +++ b/intl/icu/source/i18n/casetrn.h @@ -0,0 +1,105 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2001-2008, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: casetrn.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004sep03 +* created by: Markus W. Scherer +* +* Implementation class for lower-/upper-/title-casing transliterators. +*/ + +#ifndef __CASETRN_H__ +#define __CASETRN_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" +#include "ucase.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that performs locale-sensitive + * case mapping. + */ +class CaseMapTransliterator : public Transliterator { +public: + /** + * Constructs a transliterator. + * @param loc the given locale. + * @param id the transliterator ID. + * @param map the full case mapping function (see ucase.h) + */ + CaseMapTransliterator(const UnicodeString &id, UCaseMapFull *map); + + /** + * Destructor. + */ + virtual ~CaseMapTransliterator(); + + /** + * Copy constructor. + */ + CaseMapTransliterator(const CaseMapTransliterator&); + + /** + * Transliterator API. + * @return a copy of the object. + */ + virtual CaseMapTransliterator* clone() const override = 0; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + //virtual UClassID getDynamicClassID() const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + +protected: + /** + * Implements {@link Transliterator#handleTransliterate}. + * @param text the buffer holding transliterated and + * untransliterated text + * @param offset the start and limit of the text, the position + * of the cursor, and the start and limit of transliteration. + * @param incremental if true, assume more text may be coming after + * pos.contextLimit. Otherwise, assume the text is complete. + */ + virtual void handleTransliterate(Replaceable& text, + UTransPosition& offsets, + UBool isIncremental) const override; + + UCaseMapFull *fMap; + +private: + /** + * Assignment operator. + */ + CaseMapTransliterator& operator=(const CaseMapTransliterator&); + +}; + +U_NAMESPACE_END + +/** case context iterator using a Replaceable. This must be a C function because it is a callback. */ +U_CFUNC UChar32 U_CALLCONV +utrans_rep_caseContextIterator(void *context, int8_t dir); + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/cecal.cpp b/intl/icu/source/i18n/cecal.cpp new file mode 100644 index 0000000000..456801ee37 --- /dev/null +++ b/intl/icu/source/i18n/cecal.cpp @@ -0,0 +1,158 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2009, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "cecal.h" +#include "gregoimp.h" //Math +#include "cstring.h" + +U_NAMESPACE_BEGIN + +static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 1, 1}, // ERA + { 1, 1, 5000000, 5000000}, // YEAR + { 0, 0, 12, 12}, // MONTH + { 1, 1, 52, 53}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 5, 30}, // DAY_OF_MONTH + { 1, 1, 365, 366}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 1, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 12, 12}, // ORDINAL_MONTH +}; + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +CECalendar::CECalendar(const Locale& aLocale, UErrorCode& success) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success) +{ + setTimeInMillis(getNow(), success); +} + +CECalendar::CECalendar (const CECalendar& other) +: Calendar(other) +{ +} + +CECalendar::~CECalendar() +{ +} + +CECalendar& +CECalendar::operator=(const CECalendar& right) +{ + Calendar::operator=(right); + return *this; +} + +//------------------------------------------------------------------------- +// Calendar framework +//------------------------------------------------------------------------- + +int32_t +CECalendar::handleComputeMonthStart(int32_t eyear,int32_t emonth, UBool /*useMonth*/) const +{ + return ceToJD(eyear, emonth, 0, getJDEpochOffset()); +} + +int32_t +CECalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const +{ + return LIMITS[field][limitType]; +} + +UBool +CECalendar::haveDefaultCentury() const +{ + return true; +} + +//------------------------------------------------------------------------- +// Calendar system Conversion methods... +//------------------------------------------------------------------------- +int32_t +CECalendar::ceToJD(int32_t year, int32_t month, int32_t date, int32_t jdEpochOffset) +{ + // handle month > 12, < 0 (e.g. from add/set) + if ( month >= 0 ) { + year += month/13; + month %= 13; + } else { + ++month; + year += month/13 - 1; + month = month%13 + 12; + } + return (int32_t) ( + jdEpochOffset // difference from Julian epoch to 1,1,1 + + 365 * year // number of days from years + + ClockMath::floorDivide(year, 4) // extra day of leap year + + 30 * month // number of days from months (months are 0-based) + + date - 1 // number of days for present month (1 based) + ); +} + +void +CECalendar::jdToCE(int32_t julianDay, int32_t jdEpochOffset, int32_t& year, int32_t& month, int32_t& day) +{ + int32_t c4; // number of 4 year cycle (1461 days) + int32_t r4; // remainder of 4 year cycle, always positive + + c4 = ClockMath::floorDivide(julianDay - jdEpochOffset, 1461, &r4); + + year = 4 * c4 + (r4/365 - r4/1460); // 4 * + + + int32_t doy = (r4 == 1460) ? 365 : (r4 % 365); // days in present year + + month = doy / 30; // 30 -> Coptic/Ethiopic month length up to 12th month + day = (doy % 30) + 1; // 1-based days in a month +} + +static const char* kMonthCode13 = "M13"; + +const char* CECalendar::getTemporalMonthCode(UErrorCode& status) const { + if (get(UCAL_MONTH, status) == 12) return kMonthCode13; + return Calendar::getTemporalMonthCode(status); +} + +void +CECalendar::setTemporalMonthCode(const char* code, UErrorCode& status) { + if (U_FAILURE(status)) return; + if (uprv_strcmp(code, kMonthCode13) == 0) { + set(UCAL_MONTH, 12); + set(UCAL_IS_LEAP_MONTH, 0); + return; + } + Calendar::setTemporalMonthCode(code, status); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +//eof diff --git a/intl/icu/source/i18n/cecal.h b/intl/icu/source/i18n/cecal.h new file mode 100644 index 0000000000..16f36a7976 --- /dev/null +++ b/intl/icu/source/i18n/cecal.h @@ -0,0 +1,155 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2008, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef CECAL_H +#define CECAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" + +U_NAMESPACE_BEGIN + +/** + * Base class for EthiopicCalendar and CopticCalendar. + * @internal + */ +class U_I18N_API CECalendar : public Calendar { + +public: + + /** + * Gets The Temporal monthCode value corresponding to the month for the date. + * The value is a string identifier that starts with the literal grapheme + * "M" followed by two graphemes representing the zero-padded month number + * of the current month in a normal (non-leap) year. For the short thirteen + * month in each year in the CECalendar, the value is "M13". + * + * @param status ICU Error Code + * @return One of 13 possible strings in {"M01".. "M12", "M13"}. + * @draft ICU 73 + */ + virtual const char* getTemporalMonthCode(UErrorCode& status) const override; + + /** + * Sets The Temporal monthCode which is a string identifier that starts + * with the literal grapheme "M" followed by two graphemes representing + * the zero-padded month number of the current month in a normal + * (non-leap) year. For CECalendar calendar, the values + * are "M01" .. "M13" while the "M13" is represent the short thirteen month + * in each year. + * + * @param temporalMonth The value to be set for temporal monthCode. + * @param status ICU Error Code + * + * @draft ICU 73 + */ + virtual void setTemporalMonthCode(const char* code, UErrorCode& status) override; + +protected: + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs a CECalendar based on the current time in the default time zone + * with the given locale with the Julian epoch offiset + * + * @param aLocale The given locale. + * @param success Indicates the status of CECalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + CECalendar(const Locale& aLocale, UErrorCode& success); + + /** + * Copy Constructor + * @internal + */ + CECalendar (const CECalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~CECalendar(); + + /** + * Default assignment operator + * @param right Calendar object to be copied + * @internal + */ + CECalendar& operator=(const CECalendar& right); + +protected: + //------------------------------------------------------------------------- + // Calendar framework + //------------------------------------------------------------------------- + + /** + * Return JD of start of given month/extended year + * @internal + */ + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const override; + + /** + * Calculate the limit for a specified type of limit and field + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Returns true because Coptic/Ethiopic Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + +protected: + /** + * The Coptic and Ethiopic calendars differ only in their epochs. + * This method must be implemented by CECalendar subclasses to + * return the date offset from Julian + * @internal + */ + virtual int32_t getJDEpochOffset() const = 0; + + /** + * Convert an Coptic/Ethiopic year, month, and day to a Julian day. + * + * @param year the extended year + * @param month the month + * @param day the day + * @param jdEpochOffset the epoch offset from Julian epoch + * @return Julian day + * @internal + */ + static int32_t ceToJD(int32_t year, int32_t month, int32_t date, + int32_t jdEpochOffset); + + /** + * Convert a Julian day to an Coptic/Ethiopic year, month and day + * + * @param julianDay the Julian day + * @param jdEpochOffset the epoch offset from Julian epoch + * @param year receives the extended year + * @param month receives the month + * @param date receives the day + * @internal + */ + static void jdToCE(int32_t julianDay, int32_t jdEpochOffset, + int32_t& year, int32_t& month, int32_t& day); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif +//eof diff --git a/intl/icu/source/i18n/chnsecal.cpp b/intl/icu/source/i18n/chnsecal.cpp new file mode 100644 index 0000000000..58685632e0 --- /dev/null +++ b/intl/icu/source/i18n/chnsecal.cpp @@ -0,0 +1,992 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ****************************************************************************** + * Copyright (C) 2007-2014, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File CHNSECAL.CPP + * + * Modification History: + * + * Date Name Description + * 9/18/2007 ajmacher ported from java ChineseCalendar + ***************************************************************************** + */ + +#include "chnsecal.h" + +#if !UCONFIG_NO_FORMATTING + +#include "umutex.h" +#include +#include "gregoimp.h" // Math +#include "astro.h" // CalendarAstronomer +#include "unicode/simpletz.h" +#include "uhash.h" +#include "ucln_in.h" +#include "cstring.h" + +// Debugging +#ifdef U_DEBUG_CHNSECAL +# include +# include +static void debug_chnsecal_loc(const char *f, int32_t l) +{ + fprintf(stderr, "%s:%d: ", f, l); +} + +static void debug_chnsecal_msg(const char *pat, ...) +{ + va_list ap; + va_start(ap, pat); + vfprintf(stderr, pat, ap); + fflush(stderr); +} +// must use double parens, i.e.: U_DEBUG_CHNSECAL_MSG(("four is: %d",4)); +#define U_DEBUG_CHNSECAL_MSG(x) {debug_chnsecal_loc(__FILE__,__LINE__);debug_chnsecal_msg x;} +#else +#define U_DEBUG_CHNSECAL_MSG(x) +#endif + + +// --- The cache -- +static icu::UMutex astroLock; +static icu::CalendarAstronomer *gChineseCalendarAstro = nullptr; + +// Lazy Creation & Access synchronized by class CalendarCache with a mutex. +static icu::CalendarCache *gChineseCalendarWinterSolsticeCache = nullptr; +static icu::CalendarCache *gChineseCalendarNewYearCache = nullptr; + +static icu::TimeZone *gChineseCalendarZoneAstroCalc = nullptr; +static icu::UInitOnce gChineseCalendarZoneAstroCalcInitOnce {}; + +/** + * The start year of the Chinese calendar, the 61st year of the reign + * of Huang Di. Some sources use the first year of his reign, + * resulting in EXTENDED_YEAR values 60 years greater and ERA (cycle) + * values one greater. + */ +static const int32_t CHINESE_EPOCH_YEAR = -2636; // Gregorian year + +/** + * The offset from GMT in milliseconds at which we perform astronomical + * computations. Some sources use a different historically accurate + * offset of GMT+7:45:40 for years before 1929; we do not do this. + */ +static const int32_t CHINA_OFFSET = 8 * kOneHour; + +/** + * Value to be added or subtracted from the local days of a new moon to + * get close to the next or prior new moon, but not cross it. Must be + * >= 1 and < CalendarAstronomer.SYNODIC_MONTH. + */ +static const int32_t SYNODIC_GAP = 25; + + +U_CDECL_BEGIN +static UBool calendar_chinese_cleanup() { + if (gChineseCalendarAstro) { + delete gChineseCalendarAstro; + gChineseCalendarAstro = nullptr; + } + if (gChineseCalendarWinterSolsticeCache) { + delete gChineseCalendarWinterSolsticeCache; + gChineseCalendarWinterSolsticeCache = nullptr; + } + if (gChineseCalendarNewYearCache) { + delete gChineseCalendarNewYearCache; + gChineseCalendarNewYearCache = nullptr; + } + if (gChineseCalendarZoneAstroCalc) { + delete gChineseCalendarZoneAstroCalc; + gChineseCalendarZoneAstroCalc = nullptr; + } + gChineseCalendarZoneAstroCalcInitOnce.reset(); + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + + +// Implementation of the ChineseCalendar class + + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + + +ChineseCalendar* ChineseCalendar::clone() const { + return new ChineseCalendar(*this); +} + +ChineseCalendar::ChineseCalendar(const Locale& aLocale, UErrorCode& success) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success), + hasLeapMonthBetweenWinterSolstices(false), + fEpochYear(CHINESE_EPOCH_YEAR), + fZoneAstroCalc(getChineseCalZoneAstroCalc()) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +ChineseCalendar::ChineseCalendar(const Locale& aLocale, int32_t epochYear, + const TimeZone* zoneAstroCalc, UErrorCode &success) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success), + hasLeapMonthBetweenWinterSolstices(false), + fEpochYear(epochYear), + fZoneAstroCalc(zoneAstroCalc) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +ChineseCalendar::ChineseCalendar(const ChineseCalendar& other) : Calendar(other) { + hasLeapMonthBetweenWinterSolstices = other.hasLeapMonthBetweenWinterSolstices; + fEpochYear = other.fEpochYear; + fZoneAstroCalc = other.fZoneAstroCalc; +} + +ChineseCalendar::~ChineseCalendar() +{ +} + +const char *ChineseCalendar::getType() const { + return "chinese"; +} + +static void U_CALLCONV initChineseCalZoneAstroCalc() { + gChineseCalendarZoneAstroCalc = new SimpleTimeZone(CHINA_OFFSET, UNICODE_STRING_SIMPLE("CHINA_ZONE") ); + ucln_i18n_registerCleanup(UCLN_I18N_CHINESE_CALENDAR, calendar_chinese_cleanup); +} + +const TimeZone* ChineseCalendar::getChineseCalZoneAstroCalc() const { + umtx_initOnce(gChineseCalendarZoneAstroCalcInitOnce, &initChineseCalZoneAstroCalc); + return gChineseCalendarZoneAstroCalc; +} + +//------------------------------------------------------------------------- +// Minimum / Maximum access functions +//------------------------------------------------------------------------- + + +static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 1, 1, 83333, 83333}, // ERA + { 1, 1, 60, 60}, // YEAR + { 0, 0, 11, 11}, // MONTH + { 1, 1, 50, 55}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 29, 30}, // DAY_OF_MONTH + { 1, 1, 353, 385}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 5, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + { 0, 0, 1, 1}, // IS_LEAP_MONTH + { 0, 0, 11, 12}, // ORDINAL_MONTH +}; + + +/** +* @draft ICU 2.4 +*/ +int32_t ChineseCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return LIMITS[field][limitType]; +} + + +//---------------------------------------------------------------------- +// Calendar framework +//---------------------------------------------------------------------- + +/** + * Implement abstract Calendar method to return the extended year + * defined by the current fields. This will use either the ERA and + * YEAR field as the cycle and year-of-cycle, or the EXTENDED_YEAR + * field as the continuous year count, depending on which is newer. + * @stable ICU 2.8 + */ +int32_t ChineseCalendar::handleGetExtendedYear() { + int32_t year; + if (newestStamp(UCAL_ERA, UCAL_YEAR, kUnset) <= fStamp[UCAL_EXTENDED_YEAR]) { + year = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + int32_t cycle = internalGet(UCAL_ERA, 1) - 1; // 0-based cycle + // adjust to the instance specific epoch + year = cycle * 60 + internalGet(UCAL_YEAR, 1) - (fEpochYear - CHINESE_EPOCH_YEAR); + } + return year; +} + +/** + * Override Calendar method to return the number of days in the given + * extended year and month. + * + *

Note: This method also reads the IS_LEAP_MONTH field to determine + * whether or not the given month is a leap month. + * @stable ICU 2.8 + */ +int32_t ChineseCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + int32_t thisStart = handleComputeMonthStart(extendedYear, month, true) - + kEpochStartAsJulianDay + 1; // Julian day -> local days + int32_t nextStart = newMoonNear(thisStart + SYNODIC_GAP, true); + return nextStart - thisStart; +} + +/** + * Override Calendar to compute several fields specific to the Chinese + * calendar system. These are: + * + *

  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * + *

Compute the ChineseCalendar-specific field IS_LEAP_MONTH. + * @stable ICU 2.8 + */ +void ChineseCalendar::handleComputeFields(int32_t julianDay, UErrorCode &/*status*/) { + + computeChineseFields(julianDay - kEpochStartAsJulianDay, // local days + getGregorianYear(), getGregorianMonth(), + true); // set all fields +} + +/** + * Field resolution table that incorporates IS_LEAP_MONTH. + */ +const UFieldResolutionTable ChineseCalendar::CHINESE_DATE_PRECEDENCE[] = +{ + { + { UCAL_DAY_OF_MONTH, kResolveSTOP }, + { UCAL_WEEK_OF_YEAR, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { UCAL_WEEK_OF_YEAR, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { UCAL_DAY_OF_YEAR, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_MONTH, UCAL_IS_LEAP_MONTH, kResolveSTOP }, + { kResolveSTOP } + }, + { + { UCAL_WEEK_OF_YEAR, kResolveSTOP }, + { UCAL_WEEK_OF_MONTH, kResolveSTOP }, + { UCAL_DAY_OF_WEEK_IN_MONTH, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DAY_OF_WEEK, kResolveSTOP }, + { kResolveRemap | UCAL_DAY_OF_WEEK_IN_MONTH, UCAL_DOW_LOCAL, kResolveSTOP }, + { kResolveSTOP } + }, + {{kResolveSTOP}} +}; + +/** + * Override Calendar to add IS_LEAP_MONTH to the field resolution + * table. + * @stable ICU 2.8 + */ +const UFieldResolutionTable* ChineseCalendar::getFieldResolutionTable() const { + return CHINESE_DATE_PRECEDENCE; +} + +/** + * Return the Julian day number of day before the first day of the + * given month in the given extended year. + * + *

Note: This method reads the IS_LEAP_MONTH field to determine + * whether the given month is a leap month. + * @param eyear the extended year + * @param month the zero-based month. The month is also determined + * by reading the IS_LEAP_MONTH field. + * @return the Julian day number of the day before the first + * day of the given month and year + * @stable ICU 2.8 + */ +int32_t ChineseCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const { + ChineseCalendar *nonConstThis = (ChineseCalendar*)this; // cast away const + + // If the month is out of range, adjust it into range, and + // modify the extended year value accordingly. + if (month < 0 || month > 11) { + double m = month; + eyear += (int32_t)ClockMath::floorDivide(m, 12.0, &m); + month = (int32_t)m; + } + + int32_t gyear = eyear + fEpochYear - 1; // Gregorian year + int32_t theNewYear = newYear(gyear); + int32_t newMoon = newMoonNear(theNewYear + month * 29, true); + + int32_t julianDay = newMoon + kEpochStartAsJulianDay; + + // Save fields for later restoration + int32_t saveMonth = internalGet(UCAL_MONTH); + int32_t saveOrdinalMonth = internalGet(UCAL_ORDINAL_MONTH); + int32_t saveIsLeapMonth = internalGet(UCAL_IS_LEAP_MONTH); + + // Ignore IS_LEAP_MONTH field if useMonth is false + int32_t isLeapMonth = useMonth ? saveIsLeapMonth : 0; + + UErrorCode status = U_ZERO_ERROR; + nonConstThis->computeGregorianFields(julianDay, status); + if (U_FAILURE(status)) + return 0; + + // This will modify the MONTH and IS_LEAP_MONTH fields (only) + nonConstThis->computeChineseFields(newMoon, getGregorianYear(), + getGregorianMonth(), false); + + if (month != internalGet(UCAL_MONTH) || + isLeapMonth != internalGet(UCAL_IS_LEAP_MONTH)) { + newMoon = newMoonNear(newMoon + SYNODIC_GAP, true); + julianDay = newMoon + kEpochStartAsJulianDay; + } + + nonConstThis->internalSet(UCAL_MONTH, saveMonth); + nonConstThis->internalSet(UCAL_ORDINAL_MONTH, saveOrdinalMonth); + nonConstThis->internalSet(UCAL_IS_LEAP_MONTH, saveIsLeapMonth); + return julianDay - 1; +} + + +/** + * Override Calendar to handle leap months properly. + * @stable ICU 2.8 + */ +void ChineseCalendar::add(UCalendarDateFields field, int32_t amount, UErrorCode& status) { + switch (field) { + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + if (amount != 0) { + int32_t dom = get(UCAL_DAY_OF_MONTH, status); + if (U_FAILURE(status)) break; + int32_t day = get(UCAL_JULIAN_DAY, status) - kEpochStartAsJulianDay; // Get local day + if (U_FAILURE(status)) break; + int32_t moon = day - dom + 1; // New moon + offsetMonth(moon, dom, amount); + } + break; + default: + Calendar::add(field, amount, status); + break; + } +} + +/** + * Override Calendar to handle leap months properly. + * @stable ICU 2.8 + */ +void ChineseCalendar::add(EDateFields field, int32_t amount, UErrorCode& status) { + add((UCalendarDateFields)field, amount, status); +} + +/** + * Override Calendar to handle leap months properly. + * @stable ICU 2.8 + */ +void ChineseCalendar::roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) { + switch (field) { + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + if (amount != 0) { + int32_t dom = get(UCAL_DAY_OF_MONTH, status); + if (U_FAILURE(status)) break; + int32_t day = get(UCAL_JULIAN_DAY, status) - kEpochStartAsJulianDay; // Get local day + if (U_FAILURE(status)) break; + int32_t moon = day - dom + 1; // New moon (start of this month) + + // Note throughout the following: Months 12 and 1 are never + // followed by a leap month (D&R p. 185). + + // Compute the adjusted month number m. This is zero-based + // value from 0..11 in a non-leap year, and from 0..12 in a + // leap year. + int32_t m = get(UCAL_MONTH, status); // 0-based month + if (U_FAILURE(status)) break; + if (hasLeapMonthBetweenWinterSolstices) { // (member variable) + if (get(UCAL_IS_LEAP_MONTH, status) == 1) { + ++m; + } else { + // Check for a prior leap month. (In the + // following, month 0 is the first month of the + // year.) Month 0 is never followed by a leap + // month, and we know month m is not a leap month. + // moon1 will be the start of month 0 if there is + // no leap month between month 0 and month m; + // otherwise it will be the start of month 1. + int moon1 = moon - + (int) (CalendarAstronomer::SYNODIC_MONTH * (m - 0.5)); + moon1 = newMoonNear(moon1, true); + if (isLeapMonthBetween(moon1, moon)) { + ++m; + } + } + if (U_FAILURE(status)) break; + } + + // Now do the standard roll computation on m, with the + // allowed range of 0..n-1, where n is 12 or 13. + int32_t n = hasLeapMonthBetweenWinterSolstices ? 13 : 12; // Months in this year + int32_t newM = (m + amount) % n; + if (newM < 0) { + newM += n; + } + + if (newM != m) { + offsetMonth(moon, dom, newM - m); + } + } + break; + default: + Calendar::roll(field, amount, status); + break; + } +} + +void ChineseCalendar::roll(EDateFields field, int32_t amount, UErrorCode& status) { + roll((UCalendarDateFields)field, amount, status); +} + + +//------------------------------------------------------------------ +// Support methods and constants +//------------------------------------------------------------------ + +/** + * Convert local days to UTC epoch milliseconds. + * This is not an accurate conversion in that getTimezoneOffset + * takes the milliseconds in GMT (not local time). In theory, more + * accurate algorithm can be implemented but practically we do not need + * to go through that complication as long as the historical timezone + * changes did not happen around the 'tricky' new moon (new moon around + * midnight). + * + * @param days days after January 1, 1970 0:00 in the astronomical base zone + * @return milliseconds after January 1, 1970 0:00 GMT + */ +double ChineseCalendar::daysToMillis(double days) const { + double millis = days * (double)kOneDay; + if (fZoneAstroCalc != nullptr) { + int32_t rawOffset, dstOffset; + UErrorCode status = U_ZERO_ERROR; + fZoneAstroCalc->getOffset(millis, false, rawOffset, dstOffset, status); + if (U_SUCCESS(status)) { + return millis - (double)(rawOffset + dstOffset); + } + } + return millis - (double)CHINA_OFFSET; +} + +/** + * Convert UTC epoch milliseconds to local days. + * @param millis milliseconds after January 1, 1970 0:00 GMT + * @return days after January 1, 1970 0:00 in the astronomical base zone + */ +double ChineseCalendar::millisToDays(double millis) const { + if (fZoneAstroCalc != nullptr) { + int32_t rawOffset, dstOffset; + UErrorCode status = U_ZERO_ERROR; + fZoneAstroCalc->getOffset(millis, false, rawOffset, dstOffset, status); + if (U_SUCCESS(status)) { + return ClockMath::floorDivide(millis + (double)(rawOffset + dstOffset), kOneDay); + } + } + return ClockMath::floorDivide(millis + (double)CHINA_OFFSET, kOneDay); +} + +//------------------------------------------------------------------ +// Astronomical computations +//------------------------------------------------------------------ + + +/** + * Return the major solar term on or after December 15 of the given + * Gregorian year, that is, the winter solstice of the given year. + * Computations are relative to Asia/Shanghai time zone. + * @param gyear a Gregorian year + * @return days after January 1, 1970 0:00 Asia/Shanghai of the + * winter solstice of the given year + */ +int32_t ChineseCalendar::winterSolstice(int32_t gyear) const { + + UErrorCode status = U_ZERO_ERROR; + int32_t cacheValue = CalendarCache::get(&gChineseCalendarWinterSolsticeCache, gyear, status); + + if (cacheValue == 0) { + // In books December 15 is used, but it fails for some years + // using our algorithms, e.g.: 1298 1391 1492 1553 1560. That + // is, winterSolstice(1298) starts search at Dec 14 08:00:00 + // PST 1298 with a final result of Dec 14 10:31:59 PST 1299. + double ms = daysToMillis(Grego::fieldsToDay(gyear, UCAL_DECEMBER, 1)); + + umtx_lock(&astroLock); + if(gChineseCalendarAstro == nullptr) { + gChineseCalendarAstro = new CalendarAstronomer(); + ucln_i18n_registerCleanup(UCLN_I18N_CHINESE_CALENDAR, calendar_chinese_cleanup); + } + gChineseCalendarAstro->setTime(ms); + UDate solarLong = gChineseCalendarAstro->getSunTime(CalendarAstronomer::WINTER_SOLSTICE(), true); + umtx_unlock(&astroLock); + + // Winter solstice is 270 degrees solar longitude aka Dongzhi + cacheValue = (int32_t)millisToDays(solarLong); + CalendarCache::put(&gChineseCalendarWinterSolsticeCache, gyear, cacheValue, status); + } + if(U_FAILURE(status)) { + cacheValue = 0; + } + return cacheValue; +} + +/** + * Return the closest new moon to the given date, searching either + * forward or backward in time. + * @param days days after January 1, 1970 0:00 Asia/Shanghai + * @param after if true, search for a new moon on or after the given + * date; otherwise, search for a new moon before it + * @return days after January 1, 1970 0:00 Asia/Shanghai of the nearest + * new moon after or before days + */ +int32_t ChineseCalendar::newMoonNear(double days, UBool after) const { + + umtx_lock(&astroLock); + if(gChineseCalendarAstro == nullptr) { + gChineseCalendarAstro = new CalendarAstronomer(); + ucln_i18n_registerCleanup(UCLN_I18N_CHINESE_CALENDAR, calendar_chinese_cleanup); + } + gChineseCalendarAstro->setTime(daysToMillis(days)); + UDate newMoon = gChineseCalendarAstro->getMoonTime(CalendarAstronomer::NEW_MOON(), after); + umtx_unlock(&astroLock); + + return (int32_t) millisToDays(newMoon); +} + +/** + * Return the nearest integer number of synodic months between + * two dates. + * @param day1 days after January 1, 1970 0:00 Asia/Shanghai + * @param day2 days after January 1, 1970 0:00 Asia/Shanghai + * @return the nearest integer number of months between day1 and day2 + */ +int32_t ChineseCalendar::synodicMonthsBetween(int32_t day1, int32_t day2) const { + double roundme = ((day2 - day1) / CalendarAstronomer::SYNODIC_MONTH); + return (int32_t) (roundme + (roundme >= 0 ? .5 : -.5)); +} + +/** + * Return the major solar term on or before a given date. This + * will be an integer from 1..12, with 1 corresponding to 330 degrees, + * 2 to 0 degrees, 3 to 30 degrees,..., and 12 to 300 degrees. + * @param days days after January 1, 1970 0:00 Asia/Shanghai + */ +int32_t ChineseCalendar::majorSolarTerm(int32_t days) const { + + umtx_lock(&astroLock); + if(gChineseCalendarAstro == nullptr) { + gChineseCalendarAstro = new CalendarAstronomer(); + ucln_i18n_registerCleanup(UCLN_I18N_CHINESE_CALENDAR, calendar_chinese_cleanup); + } + gChineseCalendarAstro->setTime(daysToMillis(days)); + UDate solarLongitude = gChineseCalendarAstro->getSunLongitude(); + umtx_unlock(&astroLock); + + // Compute (floor(solarLongitude / (pi/6)) + 2) % 12 + int32_t term = ( ((int32_t)(6 * solarLongitude / CalendarAstronomer::PI)) + 2 ) % 12; + if (term < 1) { + term += 12; + } + return term; +} + +/** + * Return true if the given month lacks a major solar term. + * @param newMoon days after January 1, 1970 0:00 Asia/Shanghai of a new + * moon + */ +UBool ChineseCalendar::hasNoMajorSolarTerm(int32_t newMoon) const { + return majorSolarTerm(newMoon) == + majorSolarTerm(newMoonNear(newMoon + SYNODIC_GAP, true)); +} + + +//------------------------------------------------------------------ +// Time to fields +//------------------------------------------------------------------ + +/** + * Return true if there is a leap month on or after month newMoon1 and + * at or before month newMoon2. + * @param newMoon1 days after January 1, 1970 0:00 astronomical base zone + * of a new moon + * @param newMoon2 days after January 1, 1970 0:00 astronomical base zone + * of a new moon + */ +UBool ChineseCalendar::isLeapMonthBetween(int32_t newMoon1, int32_t newMoon2) const { + +#ifdef U_DEBUG_CHNSECAL + // This is only needed to debug the timeOfAngle divergence bug. + // Remove this later. Liu 11/9/00 + if (synodicMonthsBetween(newMoon1, newMoon2) >= 50) { + U_DEBUG_CHNSECAL_MSG(( + "isLeapMonthBetween(%d, %d): Invalid parameters", newMoon1, newMoon2 + )); + } +#endif + + return (newMoon2 >= newMoon1) && + (isLeapMonthBetween(newMoon1, newMoonNear(newMoon2 - SYNODIC_GAP, false)) || + hasNoMajorSolarTerm(newMoon2)); +} + +/** + * Compute fields for the Chinese calendar system. This method can + * either set all relevant fields, as required by + * handleComputeFields(), or it can just set the MONTH and + * IS_LEAP_MONTH fields, as required by + * handleComputeMonthStart(). + * + *

As a side effect, this method sets {@link #hasLeapMonthBetweenWinterSolstices}. + * @param days days after January 1, 1970 0:00 astronomical base zone + * of the date to compute fields for + * @param gyear the Gregorian year of the given date + * @param gmonth the Gregorian month of the given date + * @param setAllFields if true, set the EXTENDED_YEAR, ERA, YEAR, + * DAY_OF_MONTH, and DAY_OF_YEAR fields. In either case set the MONTH + * and IS_LEAP_MONTH fields. + */ +void ChineseCalendar::computeChineseFields(int32_t days, int32_t gyear, int32_t gmonth, + UBool setAllFields) { + // Find the winter solstices before and after the target date. + // These define the boundaries of this Chinese year, specifically, + // the position of month 11, which always contains the solstice. + // We want solsticeBefore <= date < solsticeAfter. + int32_t solsticeBefore; + int32_t solsticeAfter = winterSolstice(gyear); + if (days < solsticeAfter) { + solsticeBefore = winterSolstice(gyear - 1); + } else { + solsticeBefore = solsticeAfter; + solsticeAfter = winterSolstice(gyear + 1); + } + + // Find the start of the month after month 11. This will be either + // the prior month 12 or leap month 11 (very rare). Also find the + // start of the following month 11. + int32_t firstMoon = newMoonNear(solsticeBefore + 1, true); + int32_t lastMoon = newMoonNear(solsticeAfter + 1, false); + int32_t thisMoon = newMoonNear(days + 1, false); // Start of this month + // Note: hasLeapMonthBetweenWinterSolstices is a member variable + hasLeapMonthBetweenWinterSolstices = synodicMonthsBetween(firstMoon, lastMoon) == 12; + + int32_t month = synodicMonthsBetween(firstMoon, thisMoon); + int32_t theNewYear = newYear(gyear); + if (days < theNewYear) { + theNewYear = newYear(gyear-1); + } + if (hasLeapMonthBetweenWinterSolstices && isLeapMonthBetween(firstMoon, thisMoon)) { + month--; + } + if (month < 1) { + month += 12; + } + int32_t ordinalMonth = synodicMonthsBetween(theNewYear, thisMoon); + if (ordinalMonth < 0) { + ordinalMonth += 12; + } + UBool isLeapMonth = hasLeapMonthBetweenWinterSolstices && + hasNoMajorSolarTerm(thisMoon) && + !isLeapMonthBetween(firstMoon, newMoonNear(thisMoon - SYNODIC_GAP, false)); + + internalSet(UCAL_MONTH, month-1); // Convert from 1-based to 0-based + internalSet(UCAL_ORDINAL_MONTH, ordinalMonth); // Convert from 1-based to 0-based + internalSet(UCAL_IS_LEAP_MONTH, isLeapMonth?1:0); + + + if (setAllFields) { + + // Extended year and cycle year is based on the epoch year + + int32_t extended_year = gyear - fEpochYear; + int cycle_year = gyear - CHINESE_EPOCH_YEAR; + if (month < 11 || + gmonth >= UCAL_JULY) { + extended_year++; + cycle_year++; + } + int32_t dayOfMonth = days - thisMoon + 1; + + internalSet(UCAL_EXTENDED_YEAR, extended_year); + + // 0->0,60 1->1,1 60->1,60 61->2,1 etc. + int32_t yearOfCycle; + int32_t cycle = ClockMath::floorDivide(cycle_year - 1, 60, &yearOfCycle); + internalSet(UCAL_ERA, cycle + 1); + internalSet(UCAL_YEAR, yearOfCycle + 1); + + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + + // Days will be before the first new year we compute if this + // date is in month 11, leap 11, 12. There is never a leap 12. + // New year computations are cached so this should be cheap in + // the long run. + int32_t theNewYear = newYear(gyear); + if (days < theNewYear) { + theNewYear = newYear(gyear-1); + } + internalSet(UCAL_DAY_OF_YEAR, days - theNewYear + 1); + } +} + + +//------------------------------------------------------------------ +// Fields to time +//------------------------------------------------------------------ + +/** + * Return the Chinese new year of the given Gregorian year. + * @param gyear a Gregorian year + * @return days after January 1, 1970 0:00 astronomical base zone of the + * Chinese new year of the given year (this will be a new moon) + */ +int32_t ChineseCalendar::newYear(int32_t gyear) const { + UErrorCode status = U_ZERO_ERROR; + int32_t cacheValue = CalendarCache::get(&gChineseCalendarNewYearCache, gyear, status); + + if (cacheValue == 0) { + + int32_t solsticeBefore= winterSolstice(gyear - 1); + int32_t solsticeAfter = winterSolstice(gyear); + int32_t newMoon1 = newMoonNear(solsticeBefore + 1, true); + int32_t newMoon2 = newMoonNear(newMoon1 + SYNODIC_GAP, true); + int32_t newMoon11 = newMoonNear(solsticeAfter + 1, false); + + if (synodicMonthsBetween(newMoon1, newMoon11) == 12 && + (hasNoMajorSolarTerm(newMoon1) || hasNoMajorSolarTerm(newMoon2))) { + cacheValue = newMoonNear(newMoon2 + SYNODIC_GAP, true); + } else { + cacheValue = newMoon2; + } + + CalendarCache::put(&gChineseCalendarNewYearCache, gyear, cacheValue, status); + } + if(U_FAILURE(status)) { + cacheValue = 0; + } + return cacheValue; +} + +/** + * Adjust this calendar to be delta months before or after a given + * start position, pinning the day of month if necessary. The start + * position is given as a local days number for the start of the month + * and a day-of-month. Used by add() and roll(). + * @param newMoon the local days of the first day of the month of the + * start position (days after January 1, 1970 0:00 Asia/Shanghai) + * @param dom the 1-based day-of-month of the start position + * @param delta the number of months to move forward or backward from + * the start position + */ +void ChineseCalendar::offsetMonth(int32_t newMoon, int32_t dom, int32_t delta) { + UErrorCode status = U_ZERO_ERROR; + + // Move to the middle of the month before our target month. + newMoon += (int32_t) (CalendarAstronomer::SYNODIC_MONTH * (delta - 0.5)); + + // Search forward to the target month's new moon + newMoon = newMoonNear(newMoon, true); + + // Find the target dom + int32_t jd = newMoon + kEpochStartAsJulianDay - 1 + dom; + + // Pin the dom. In this calendar all months are 29 or 30 days + // so pinning just means handling dom 30. + if (dom > 29) { + set(UCAL_JULIAN_DAY, jd-1); + // TODO Fix this. We really shouldn't ever have to + // explicitly call complete(). This is either a bug in + // this method, in ChineseCalendar, or in + // Calendar.getActualMaximum(). I suspect the last. + complete(status); + if (U_FAILURE(status)) return; + if (getActualMaximum(UCAL_DAY_OF_MONTH, status) >= dom) { + if (U_FAILURE(status)) return; + set(UCAL_JULIAN_DAY, jd); + } + } else { + set(UCAL_JULIAN_DAY, jd); + } +} + +constexpr uint32_t kChineseRelatedYearDiff = -2637; + +int32_t ChineseCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kChineseRelatedYearDiff; +} + +void ChineseCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kChineseRelatedYearDiff); +} + +// default century + +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInitOnce {}; + + +UBool ChineseCalendar::haveDefaultCentury() const +{ + return true; +} + +UDate ChineseCalendar::defaultCenturyStart() const +{ + return internalGetDefaultCenturyStart(); +} + +int32_t ChineseCalendar::defaultCenturyStartYear() const +{ + return internalGetDefaultCenturyStartYear(); +} + +static void U_CALLCONV initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + ChineseCalendar calendar(Locale("@calendar=chinese"),status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate +ChineseCalendar::internalGetDefaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInitOnce, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t +ChineseCalendar::internalGetDefaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInitOnce, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + +bool +ChineseCalendar::inTemporalLeapYear(UErrorCode &status) const +{ + int32_t days = getActualMaximum(UCAL_DAY_OF_YEAR, status); + if (U_FAILURE(status)) return false; + return days > 360; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ChineseCalendar) + + +static const char * const gTemporalLeapMonthCodes[] = { + "M01L", "M02L", "M03L", "M04L", "M05L", "M06L", + "M07L", "M08L", "M09L", "M10L", "M11L", "M12L", nullptr +}; + +const char* ChineseCalendar::getTemporalMonthCode(UErrorCode &status) const { + // We need to call get, not internalGet, to force the calculation + // from UCAL_ORDINAL_MONTH. + int32_t is_leap = get(UCAL_IS_LEAP_MONTH, status); + if (U_FAILURE(status)) return nullptr; + if (is_leap != 0) { + int32_t month = get(UCAL_MONTH, status); + if (U_FAILURE(status)) return nullptr; + return gTemporalLeapMonthCodes[month]; + } + return Calendar::getTemporalMonthCode(status); +} + +void +ChineseCalendar::setTemporalMonthCode(const char* code, UErrorCode& status ) +{ + if (U_FAILURE(status)) return; + int32_t len = static_cast(uprv_strlen(code)); + if (len != 4 || code[0] != 'M' || code[3] != 'L') { + set(UCAL_IS_LEAP_MONTH, 0); + return Calendar::setTemporalMonthCode(code, status); + } + for (int m = 0; gTemporalLeapMonthCodes[m] != nullptr; m++) { + if (uprv_strcmp(code, gTemporalLeapMonthCodes[m]) == 0) { + set(UCAL_MONTH, m); + set(UCAL_IS_LEAP_MONTH, 1); + return; + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; +} + +int32_t ChineseCalendar::internalGetMonth() const { + if (resolveFields(kMonthPrecedence) == UCAL_MONTH) { + return internalGet(UCAL_MONTH); + } + LocalPointer temp(this->clone()); + temp->set(UCAL_MONTH, 0); + temp->set(UCAL_IS_LEAP_MONTH, 0); + temp->set(UCAL_DATE, 1); + // Calculate the UCAL_MONTH and UCAL_IS_LEAP_MONTH by adding number of + // months. + UErrorCode status = U_ZERO_ERROR; + temp->roll(UCAL_MONTH, internalGet(UCAL_ORDINAL_MONTH), status); + + + ChineseCalendar *nonConstThis = (ChineseCalendar*)this; // cast away const + nonConstThis->internalSet(UCAL_IS_LEAP_MONTH, temp->get(UCAL_IS_LEAP_MONTH, status)); + int32_t month = temp->get(UCAL_MONTH, status); + U_ASSERT(U_SUCCESS(status)); + nonConstThis->internalSet(UCAL_MONTH, month); + return month; +} + +int32_t ChineseCalendar::internalGetMonth(int32_t defaultValue) const { + if (resolveFields(kMonthPrecedence) == UCAL_MONTH) { + return internalGet(UCAL_MONTH, defaultValue); + } + return internalGetMonth(); +} + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/chnsecal.h b/intl/icu/source/i18n/chnsecal.h new file mode 100644 index 0000000000..9b5a629fad --- /dev/null +++ b/intl/icu/source/i18n/chnsecal.h @@ -0,0 +1,337 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ***************************************************************************** + * Copyright (C) 2007-2013, International Business Machines Corporation + * and others. All Rights Reserved. + ***************************************************************************** + * + * File CHNSECAL.H + * + * Modification History: + * + * Date Name Description + * 9/18/2007 ajmacher ported from java ChineseCalendar + ***************************************************************************** + */ + +#ifndef CHNSECAL_H +#define CHNSECAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/timezone.h" + +U_NAMESPACE_BEGIN + +/** + * ChineseCalendar is a concrete subclass of {@link Calendar} + * that implements a traditional Chinese calendar. The traditional Chinese + * calendar is a lunisolar calendar: Each month starts on a new moon, and + * the months are numbered according to solar events, specifically, to + * guarantee that month 11 always contains the winter solstice. In order + * to accomplish this, leap months are inserted in certain years. Leap + * months are numbered the same as the month they follow. The decision of + * which month is a leap month depends on the relative movements of the sun + * and moon. + * + *

This class defines one addition field beyond those defined by + * Calendar: The IS_LEAP_MONTH field takes the + * value of 0 for normal months, or 1 for leap months. + * + *

All astronomical computations are performed with respect to a time + * zone of GMT+8:00 and a longitude of 120 degrees east. Although some + * calendars implement a historically more accurate convention of using + * Beijing's local longitude (116 degrees 25 minutes east) and time zone + * (GMT+7:45:40) for dates before 1929, we do not implement this here. + * + *

Years are counted in two different ways in the Chinese calendar. The + * first method is by sequential numbering from the 61st year of the reign + * of Huang Di, 2637 BCE, which is designated year 1 on the Chinese + * calendar. The second method uses 60-year cycles from the same starting + * point, which is designated year 1 of cycle 1. In this class, the + * EXTENDED_YEAR field contains the sequential year count. + * The ERA field contains the cycle number, and the + * YEAR field contains the year of the cycle, a value between + * 1 and 60. + * + *

There is some variation in what is considered the starting point of + * the calendar, with some sources starting in the first year of the reign + * of Huang Di, rather than the 61st. This gives continuous year numbers + * 60 years greater and cycle numbers one greater than what this class + * implements. + * + *

Because ChineseCalendar defines an additional field and + * redefines the way the ERA field is used, it requires a new + * format class, ChineseDateFormat. As always, use the + * methods DateFormat.getXxxInstance(Calendar cal,...) to + * obtain a formatter for this calendar. + * + *

References:

+ * + *

+ * This class should only be subclassed to implement variants of the Chinese lunar calendar.

+ *

+ * ChineseCalendar usually should be instantiated using + * {@link com.ibm.icu.util.Calendar#getInstance(ULocale)} passing in a ULocale + * with the tag "@calendar=chinese".

+ * + * @see com.ibm.icu.text.ChineseDateFormat + * @see com.ibm.icu.util.Calendar + * @author Alan Liu + * @internal + */ +class U_I18N_API ChineseCalendar : public Calendar { + public: + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs a ChineseCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of ChineseCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + ChineseCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Returns true if the date is in a leap year. + * + * @param status ICU Error Code + * @return True if the date in the fields is in a Temporal proposal + * defined leap year. False otherwise. + */ + virtual bool inTemporalLeapYear(UErrorCode &status) const override; + + /** + * Gets The Temporal monthCode value corresponding to the month for the date. + * The value is a string identifier that starts with the literal grapheme + * "M" followed by two graphemes representing the zero-padded month number + * of the current month in a normal (non-leap) year and suffixed by an + * optional literal grapheme "L" if this is a leap month in a lunisolar + * calendar. For Chinese calendars (including Dangi), the values are + * "M01" .. "M12" for non-leap year, and "M01" .. "M12" with one of + * "M01L" .. "M12L" for leap year. + * + * @param status ICU Error Code + * @return One of 24 possible strings in + * {"M01" .. "M12", "M01L" .. "M12L"}. + * @draft ICU 73 + */ + virtual const char* getTemporalMonthCode(UErrorCode &status) const override; + + /** + * Sets The Temporal monthCode which is a string identifier that starts + * with the literal grapheme "M" followed by two graphemes representing + * the zero-padded month number of the current month in a normal + * (non-leap) year and suffixed by an optional literal grapheme "L" if this + * is a leap month in a lunisolar calendar. For Chinese calendars, the values + * are "M01" .. "M12" for non-leap years, and "M01" .. "M12" plus one in + * "M01L" .. "M12L" for leap year. + * + * @param temporalMonth The value to be set for temporal monthCode. One of + * 24 possible strings in {"M01" .. "M12", "M01L" .. "M12L"}. + * @param status ICU Error Code + * + * @draft ICU 73 + */ + virtual void setTemporalMonthCode(const char* code, UErrorCode& status) override; + + protected: + + /** + * Constructs a ChineseCalendar based on the current time in the default time zone + * with the given locale, using the specified epoch year and time zone for + * astronomical calculations. + * + * @param aLocale The given locale. + * @param epochYear The epoch year to use for calculation. + * @param zoneAstroCalc The TimeZone to use for astronomical calculations. If null, + * will be set appropriately for Chinese calendar (UTC + 8:00). + * @param success Indicates the status of ChineseCalendar object construction; + * if successful, will not be changed to an error value. + * @internal + */ + ChineseCalendar(const Locale& aLocale, int32_t epochYear, const TimeZone* zoneAstroCalc, UErrorCode &success); + + public: + /** + * Copy Constructor + * @internal + */ + ChineseCalendar(const ChineseCalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~ChineseCalendar(); + + // clone + virtual ChineseCalendar* clone() const override; + + private: + + //------------------------------------------------------------------------- + // Internal data.... + //------------------------------------------------------------------------- + + // There is a leap month between the Winter Solstice before and after the + // current date.This is different from leap year because in some year, such as + // 1813 and 2033, the leap month is after the Winter Solstice of that year. So + // this value could be false for a date prior to the Winter Solstice of that + // year but that year still has a leap month and therefor is a leap year. + UBool hasLeapMonthBetweenWinterSolstices; + int32_t fEpochYear; // Start year of this Chinese calendar instance. + const TimeZone* fZoneAstroCalc; // Zone used for the astronomical calculation + // of this Chinese calendar instance. + + //---------------------------------------------------------------------- + // Calendar framework + //---------------------------------------------------------------------- + + protected: + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const override; + virtual int32_t handleGetExtendedYear() override; + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + virtual const UFieldResolutionTable* getFieldResolutionTable() const override; + + public: + virtual void add(UCalendarDateFields field, int32_t amount, UErrorCode &status) override; + virtual void add(EDateFields field, int32_t amount, UErrorCode &status) override; + virtual void roll(UCalendarDateFields field, int32_t amount, UErrorCode &status) override; + virtual void roll(EDateFields field, int32_t amount, UErrorCode &status) override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + //---------------------------------------------------------------------- + // Internal methods & astronomical calculations + //---------------------------------------------------------------------- + + private: + + static const UFieldResolutionTable CHINESE_DATE_PRECEDENCE[]; + + double daysToMillis(double days) const; + double millisToDays(double millis) const; + virtual int32_t winterSolstice(int32_t gyear) const; + virtual int32_t newMoonNear(double days, UBool after) const; + virtual int32_t synodicMonthsBetween(int32_t day1, int32_t day2) const; + virtual int32_t majorSolarTerm(int32_t days) const; + virtual UBool hasNoMajorSolarTerm(int32_t newMoon) const; + virtual UBool isLeapMonthBetween(int32_t newMoon1, int32_t newMoon2) const; + virtual void computeChineseFields(int32_t days, int32_t gyear, + int32_t gmonth, UBool setAllFields); + virtual int32_t newYear(int32_t gyear) const; + virtual void offsetMonth(int32_t newMoon, int32_t dom, int32_t delta); + const TimeZone* getChineseCalZoneAstroCalc() const; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "chinese". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + protected: + virtual int32_t internalGetMonth(int32_t defaultValue) const override; + + virtual int32_t internalGetMonth() const override; + + protected: + /** + * Returns true because the Islamic Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + + private: // default century stuff. + + /** + * Returns the beginning date of the 100-year window that dates + * with 2-digit years are considered to fall within. + */ + UDate internalGetDefaultCenturyStart() const; + + /** + * Returns the first year of the 100-year window that dates with + * 2-digit years are considered to fall within. + */ + int32_t internalGetDefaultCenturyStartYear() const; + + ChineseCalendar() = delete; // default constructor not implemented +}; + +U_NAMESPACE_END + +#endif +#endif diff --git a/intl/icu/source/i18n/choicfmt.cpp b/intl/icu/source/i18n/choicfmt.cpp new file mode 100644 index 0000000000..96e73fabcf --- /dev/null +++ b/intl/icu/source/i18n/choicfmt.cpp @@ -0,0 +1,577 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2013, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File CHOICFMT.CPP +* +* Modification History: +* +* Date Name Description +* 02/19/97 aliu Converted from java. +* 03/20/97 helena Finished first cut of implementation and got rid +* of nextDouble/previousDouble and replaced with +* boolean array. +* 4/10/97 aliu Clean up. Modified to work on AIX. +* 06/04/97 helena Fixed applyPattern(), toPattern() and not to include +* wchar.h. +* 07/09/97 helena Made ParsePosition into a class. +* 08/06/97 nos removed overloaded constructor, fixed 'format(array)' +* 07/22/98 stephen JDK 1.2 Sync - removed UBool array (doubleFlags) +* 02/22/99 stephen Removed character literals for EBCDIC safety +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/choicfmt.h" +#include "unicode/numfmt.h" +#include "unicode/locid.h" +#include "cpputils.h" +#include "cstring.h" +#include "messageimpl.h" +#include "putilimp.h" +#include "uassert.h" +#include +#include + +// ***************************************************************************** +// class ChoiceFormat +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ChoiceFormat) + +// Special characters used by ChoiceFormat. There are two characters +// used interchangeably to indicate <=. Either is parsed, but only +// LESS_EQUAL is generated by toPattern(). +#define SINGLE_QUOTE ((char16_t)0x0027) /*'*/ +#define LESS_THAN ((char16_t)0x003C) /*<*/ +#define LESS_EQUAL ((char16_t)0x0023) /*#*/ +#define LESS_EQUAL2 ((char16_t)0x2264) +#define VERTICAL_BAR ((char16_t)0x007C) /*|*/ +#define MINUS ((char16_t)0x002D) /*-*/ + +static const char16_t LEFT_CURLY_BRACE = 0x7B; /*{*/ +static const char16_t RIGHT_CURLY_BRACE = 0x7D; /*}*/ + +#ifdef INFINITY +#undef INFINITY +#endif +#define INFINITY ((char16_t)0x221E) + +//static const char16_t gPositiveInfinity[] = {INFINITY, 0}; +//static const char16_t gNegativeInfinity[] = {MINUS, INFINITY, 0}; +#define POSITIVE_INF_STRLEN 1 +#define NEGATIVE_INF_STRLEN 2 + +// ------------------------------------- +// Creates a ChoiceFormat instance based on the pattern. + +ChoiceFormat::ChoiceFormat(const UnicodeString& newPattern, + UErrorCode& status) +: constructorErrorCode(status), + msgPattern(status) +{ + applyPattern(newPattern, status); +} + +// ------------------------------------- +// Creates a ChoiceFormat instance with the limit array and +// format strings for each limit. + +ChoiceFormat::ChoiceFormat(const double* limits, + const UnicodeString* formats, + int32_t cnt ) +: constructorErrorCode(U_ZERO_ERROR), + msgPattern(constructorErrorCode) +{ + setChoices(limits, nullptr, formats, cnt, constructorErrorCode); +} + +// ------------------------------------- + +ChoiceFormat::ChoiceFormat(const double* limits, + const UBool* closures, + const UnicodeString* formats, + int32_t cnt ) +: constructorErrorCode(U_ZERO_ERROR), + msgPattern(constructorErrorCode) +{ + setChoices(limits, closures, formats, cnt, constructorErrorCode); +} + +// ------------------------------------- +// copy constructor + +ChoiceFormat::ChoiceFormat(const ChoiceFormat& that) +: NumberFormat(that), + constructorErrorCode(that.constructorErrorCode), + msgPattern(that.msgPattern) +{ +} + +// ------------------------------------- +// Private constructor that creates a +// ChoiceFormat instance based on the +// pattern and populates UParseError + +ChoiceFormat::ChoiceFormat(const UnicodeString& newPattern, + UParseError& parseError, + UErrorCode& status) +: constructorErrorCode(status), + msgPattern(status) +{ + applyPattern(newPattern,parseError, status); +} +// ------------------------------------- + +bool +ChoiceFormat::operator==(const Format& that) const +{ + if (this == &that) return true; + if (!NumberFormat::operator==(that)) return false; + const ChoiceFormat& thatAlias = static_cast(that); + return msgPattern == thatAlias.msgPattern; +} + +// ------------------------------------- +// copy constructor + +const ChoiceFormat& +ChoiceFormat::operator=(const ChoiceFormat& that) +{ + if (this != &that) { + NumberFormat::operator=(that); + constructorErrorCode = that.constructorErrorCode; + msgPattern = that.msgPattern; + } + return *this; +} + +// ------------------------------------- + +ChoiceFormat::~ChoiceFormat() +{ +} + +// ------------------------------------- + +/** + * Convert a double value to a string without the overhead of NumberFormat. + */ +UnicodeString& +ChoiceFormat::dtos(double value, + UnicodeString& string) +{ + /* Buffer to contain the digits and any extra formatting stuff. */ + char temp[DBL_DIG + 16]; + char *itrPtr = temp; + char *expPtr; + + snprintf(temp, sizeof(temp), "%.*g", DBL_DIG, value); + + /* Find and convert the decimal point. + Using setlocale on some machines will cause snprintf to use a comma for certain locales. + */ + while (*itrPtr && (*itrPtr == '-' || isdigit(*itrPtr))) { + itrPtr++; + } + if (*itrPtr != 0 && *itrPtr != 'e') { + /* We reached something that looks like a decimal point. + In case someone used setlocale(), which changes the decimal point. */ + *itrPtr = '.'; + itrPtr++; + } + /* Search for the exponent */ + while (*itrPtr && *itrPtr != 'e') { + itrPtr++; + } + if (*itrPtr == 'e') { + itrPtr++; + /* Verify the exponent sign */ + if (*itrPtr == '+' || *itrPtr == '-') { + itrPtr++; + } + /* Remove leading zeros. You will see this on Windows machines. */ + expPtr = itrPtr; + while (*itrPtr == '0') { + itrPtr++; + } + if (*itrPtr && expPtr != itrPtr) { + /* Shift the exponent without zeros. */ + while (*itrPtr) { + *(expPtr++) = *(itrPtr++); + } + // NUL terminate + *expPtr = 0; + } + } + + string = UnicodeString(temp, -1, US_INV); /* invariant codepage */ + return string; +} + +// ------------------------------------- +// calls the overloaded applyPattern method. + +void +ChoiceFormat::applyPattern(const UnicodeString& pattern, + UErrorCode& status) +{ + msgPattern.parseChoiceStyle(pattern, nullptr, status); + constructorErrorCode = status; +} + +// ------------------------------------- +// Applies the pattern to this ChoiceFormat instance. + +void +ChoiceFormat::applyPattern(const UnicodeString& pattern, + UParseError& parseError, + UErrorCode& status) +{ + msgPattern.parseChoiceStyle(pattern, &parseError, status); + constructorErrorCode = status; +} +// ------------------------------------- +// Returns the input pattern string. + +UnicodeString& +ChoiceFormat::toPattern(UnicodeString& result) const +{ + return result = msgPattern.getPatternString(); +} + +// ------------------------------------- +// Sets the limit and format arrays. +void +ChoiceFormat::setChoices( const double* limits, + const UnicodeString* formats, + int32_t cnt ) +{ + UErrorCode errorCode = U_ZERO_ERROR; + setChoices(limits, nullptr, formats, cnt, errorCode); +} + +// ------------------------------------- +// Sets the limit and format arrays. +void +ChoiceFormat::setChoices( const double* limits, + const UBool* closures, + const UnicodeString* formats, + int32_t cnt ) +{ + UErrorCode errorCode = U_ZERO_ERROR; + setChoices(limits, closures, formats, cnt, errorCode); +} + +void +ChoiceFormat::setChoices(const double* limits, + const UBool* closures, + const UnicodeString* formats, + int32_t count, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return; + } + if (limits == nullptr || formats == nullptr) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + // Reconstruct the original input pattern. + // Modified version of the pre-ICU 4.8 toPattern() implementation. + UnicodeString result; + for (int32_t i = 0; i < count; ++i) { + if (i != 0) { + result += VERTICAL_BAR; + } + UnicodeString buf; + if (uprv_isPositiveInfinity(limits[i])) { + result += INFINITY; + } else if (uprv_isNegativeInfinity(limits[i])) { + result += MINUS; + result += INFINITY; + } else { + result += dtos(limits[i], buf); + } + if (closures != nullptr && closures[i]) { + result += LESS_THAN; + } else { + result += LESS_EQUAL; + } + // Append formats[i], using quotes if there are special + // characters. Single quotes themselves must be escaped in + // either case. + const UnicodeString& text = formats[i]; + int32_t textLength = text.length(); + int32_t nestingLevel = 0; + for (int32_t j = 0; j < textLength; ++j) { + char16_t c = text[j]; + if (c == SINGLE_QUOTE && nestingLevel == 0) { + // Double each top-level apostrophe. + result.append(c); + } else if (c == VERTICAL_BAR && nestingLevel == 0) { + // Surround each pipe symbol with apostrophes for quoting. + // If the next character is an apostrophe, then that will be doubled, + // and although the parser will see the apostrophe pairs beginning + // and ending one character earlier than our doubling, the result + // is as desired. + // | -> '|' + // |' -> '|''' + // |'' -> '|''''' etc. + result.append(SINGLE_QUOTE).append(c).append(SINGLE_QUOTE); + continue; // Skip the append(c) at the end of the loop body. + } else if (c == LEFT_CURLY_BRACE) { + ++nestingLevel; + } else if (c == RIGHT_CURLY_BRACE && nestingLevel > 0) { + --nestingLevel; + } + result.append(c); + } + } + // Apply the reconstructed pattern. + applyPattern(result, errorCode); +} + +// ------------------------------------- +// Gets the limit array. + +const double* +ChoiceFormat::getLimits(int32_t& cnt) const +{ + cnt = 0; + return nullptr; +} + +// ------------------------------------- +// Gets the closures array. + +const UBool* +ChoiceFormat::getClosures(int32_t& cnt) const +{ + cnt = 0; + return nullptr; +} + +// ------------------------------------- +// Gets the format array. + +const UnicodeString* +ChoiceFormat::getFormats(int32_t& cnt) const +{ + cnt = 0; + return nullptr; +} + +// ------------------------------------- +// Formats an int64 number, it's actually formatted as +// a double. The returned format string may differ +// from the input number because of this. + +UnicodeString& +ChoiceFormat::format(int64_t number, + UnicodeString& appendTo, + FieldPosition& status) const +{ + return format((double) number, appendTo, status); +} + +// ------------------------------------- +// Formats an int32_t number, it's actually formatted as +// a double. + +UnicodeString& +ChoiceFormat::format(int32_t number, + UnicodeString& appendTo, + FieldPosition& status) const +{ + return format((double) number, appendTo, status); +} + +// ------------------------------------- +// Formats a double number. + +UnicodeString& +ChoiceFormat::format(double number, + UnicodeString& appendTo, + FieldPosition& /*pos*/) const +{ + if (msgPattern.countParts() == 0) { + // No pattern was applied, or it failed. + return appendTo; + } + // Get the appropriate sub-message. + int32_t msgStart = findSubMessage(msgPattern, 0, number); + if (!MessageImpl::jdkAposMode(msgPattern)) { + int32_t patternStart = msgPattern.getPart(msgStart).getLimit(); + int32_t msgLimit = msgPattern.getLimitPartIndex(msgStart); + appendTo.append(msgPattern.getPatternString(), + patternStart, + msgPattern.getPatternIndex(msgLimit) - patternStart); + return appendTo; + } + // JDK compatibility mode: Remove SKIP_SYNTAX. + return MessageImpl::appendSubMessageWithoutSkipSyntax(msgPattern, msgStart, appendTo); +} + +int32_t +ChoiceFormat::findSubMessage(const MessagePattern &pattern, int32_t partIndex, double number) { + int32_t count = pattern.countParts(); + int32_t msgStart; + // Iterate over (ARG_INT|DOUBLE, ARG_SELECTOR, message) tuples + // until ARG_LIMIT or end of choice-only pattern. + // Ignore the first number and selector and start the loop on the first message. + partIndex += 2; + for (;;) { + // Skip but remember the current sub-message. + msgStart = partIndex; + partIndex = pattern.getLimitPartIndex(partIndex); + if (++partIndex >= count) { + // Reached the end of the choice-only pattern. + // Return with the last sub-message. + break; + } + const MessagePattern::Part &part = pattern.getPart(partIndex++); + UMessagePatternPartType type = part.getType(); + if (type == UMSGPAT_PART_TYPE_ARG_LIMIT) { + // Reached the end of the ChoiceFormat style. + // Return with the last sub-message. + break; + } + // part is an ARG_INT or ARG_DOUBLE + U_ASSERT(MessagePattern::Part::hasNumericValue(type)); + double boundary = pattern.getNumericValue(part); + // Fetch the ARG_SELECTOR character. + int32_t selectorIndex = pattern.getPatternIndex(partIndex++); + char16_t boundaryChar = pattern.getPatternString().charAt(selectorIndex); + if (boundaryChar == LESS_THAN ? !(number > boundary) : !(number >= boundary)) { + // The number is in the interval between the previous boundary and the current one. + // Return with the sub-message between them. + // The !(a>b) and !(a>=b) comparisons are equivalent to + // (a<=b) and (a= 0) { + int32_t newIndex = start + len; + if (newIndex > furthest) { + furthest = newIndex; + bestNumber = tempNumber; + if (furthest == source.length()) { + break; + } + } + } + partIndex = msgLimit + 1; + } + if (furthest == start) { + pos.setErrorIndex(start); + } else { + pos.setIndex(furthest); + } + return bestNumber; +} + +int32_t +ChoiceFormat::matchStringUntilLimitPart( + const MessagePattern &pattern, int32_t partIndex, int32_t limitPartIndex, + const UnicodeString &source, int32_t sourceOffset) { + int32_t matchingSourceLength = 0; + const UnicodeString &msgString = pattern.getPatternString(); + int32_t prevIndex = pattern.getPart(partIndex).getLimit(); + for (;;) { + const MessagePattern::Part &part = pattern.getPart(++partIndex); + if (partIndex == limitPartIndex || part.getType() == UMSGPAT_PART_TYPE_SKIP_SYNTAX) { + int32_t index = part.getIndex(); + int32_t length = index - prevIndex; + if (length != 0 && 0 != source.compare(sourceOffset, length, msgString, prevIndex, length)) { + return -1; // mismatch + } + matchingSourceLength += length; + if (partIndex == limitPartIndex) { + return matchingSourceLength; + } + prevIndex = part.getLimit(); // SKIP_SYNTAX + } + } +} + +// ------------------------------------- + +ChoiceFormat* +ChoiceFormat::clone() const +{ + ChoiceFormat *aCopy = new ChoiceFormat(*this); + return aCopy; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/coleitr.cpp b/intl/icu/source/i18n/coleitr.cpp new file mode 100644 index 0000000000..be0a8e4690 --- /dev/null +++ b/intl/icu/source/i18n/coleitr.cpp @@ -0,0 +1,473 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1996-2014, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +/* +* File coleitr.cpp +* +* Created by: Helena Shih +* +* Modification History: +* +* Date Name Description +* +* 6/23/97 helena Adding comments to make code more readable. +* 08/03/98 erm Synched with 1.2 version of CollationElementIterator.java +* 12/10/99 aliu Ported Thai collation support from Java. +* 01/25/01 swquek Modified to a C++ wrapper calling C APIs (ucoliter.h) +* 02/19/01 swquek Removed CollationElementIterator() since it is +* private constructor and no calls are made to it +* 2012-2014 markus Rewritten in C++ again. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/chariter.h" +#include "unicode/coleitr.h" +#include "unicode/tblcoll.h" +#include "unicode/ustring.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationiterator.h" +#include "collationsets.h" +#include "collationtailoring.h" +#include "uassert.h" +#include "uhash.h" +#include "utf16collationiterator.h" +#include "uvectr32.h" + +/* Constants --------------------------------------------------------------- */ + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CollationElementIterator) + +/* CollationElementIterator public constructor/destructor ------------------ */ + +CollationElementIterator::CollationElementIterator( + const CollationElementIterator& other) + : UObject(other), iter_(nullptr), rbc_(nullptr), otherHalf_(0), dir_(0), offsets_(nullptr) { + *this = other; +} + +CollationElementIterator::~CollationElementIterator() +{ + delete iter_; + delete offsets_; +} + +/* CollationElementIterator public methods --------------------------------- */ + +namespace { + +uint32_t getFirstHalf(uint32_t p, uint32_t lower32) { + return (p & 0xffff0000) | ((lower32 >> 16) & 0xff00) | ((lower32 >> 8) & 0xff); +} +uint32_t getSecondHalf(uint32_t p, uint32_t lower32) { + return (p << 16) | ((lower32 >> 8) & 0xff00) | (lower32 & 0x3f); +} +UBool ceNeedsTwoParts(int64_t ce) { + return (ce & INT64_C(0xffff00ff003f)) != 0; +} + +} // namespace + +int32_t CollationElementIterator::getOffset() const +{ + if (dir_ < 0 && offsets_ != nullptr && !offsets_->isEmpty()) { + // CollationIterator::previousCE() decrements the CEs length + // while it pops CEs from its internal buffer. + int32_t i = iter_->getCEsLength(); + if (otherHalf_ != 0) { + // Return the trailing CE offset while we are in the middle of a 64-bit CE. + ++i; + } + U_ASSERT(i < offsets_->size()); + return offsets_->elementAti(i); + } + return iter_->getOffset(); +} + +/** +* Get the ordering priority of the next character in the string. +* @return the next character's ordering. Returns NULLORDER if an error has +* occurred or if the end of string has been reached +*/ +int32_t CollationElementIterator::next(UErrorCode& status) +{ + if (U_FAILURE(status)) { return NULLORDER; } + if (dir_ > 1) { + // Continue forward iteration. Test this first. + if (otherHalf_ != 0) { + uint32_t oh = otherHalf_; + otherHalf_ = 0; + return oh; + } + } else if (dir_ == 1) { + // next() after setOffset() + dir_ = 2; + } else if (dir_ == 0) { + // The iter_ is already reset to the start of the text. + dir_ = 2; + } else /* dir_ < 0 */ { + // illegal change of direction + status = U_INVALID_STATE_ERROR; + return NULLORDER; + } + // No need to keep all CEs in the buffer when we iterate. + iter_->clearCEsIfNoneRemaining(); + int64_t ce = iter_->nextCE(status); + if (ce == Collation::NO_CE) { return NULLORDER; } + // Turn the 64-bit CE into two old-style 32-bit CEs, without quaternary bits. + uint32_t p = (uint32_t)(ce >> 32); + uint32_t lower32 = (uint32_t)ce; + uint32_t firstHalf = getFirstHalf(p, lower32); + uint32_t secondHalf = getSecondHalf(p, lower32); + if (secondHalf != 0) { + otherHalf_ = secondHalf | 0xc0; // continuation CE + } + return firstHalf; +} + +bool CollationElementIterator::operator!=( + const CollationElementIterator& other) const +{ + return !(*this == other); +} + +bool CollationElementIterator::operator==( + const CollationElementIterator& that) const +{ + if (this == &that) { + return true; + } + + return + (rbc_ == that.rbc_ || *rbc_ == *that.rbc_) && + otherHalf_ == that.otherHalf_ && + normalizeDir() == that.normalizeDir() && + string_ == that.string_ && + *iter_ == *that.iter_; +} + +/** +* Get the ordering priority of the previous collation element in the string. +* @param status the error code status. +* @return the previous element's ordering. Returns NULLORDER if an error has +* occurred or if the start of string has been reached. +*/ +int32_t CollationElementIterator::previous(UErrorCode& status) +{ + if (U_FAILURE(status)) { return NULLORDER; } + if (dir_ < 0) { + // Continue backwards iteration. Test this first. + if (otherHalf_ != 0) { + uint32_t oh = otherHalf_; + otherHalf_ = 0; + return oh; + } + } else if (dir_ == 0) { + iter_->resetToOffset(string_.length()); + dir_ = -1; + } else if (dir_ == 1) { + // previous() after setOffset() + dir_ = -1; + } else /* dir_ > 1 */ { + // illegal change of direction + status = U_INVALID_STATE_ERROR; + return NULLORDER; + } + if (offsets_ == nullptr) { + offsets_ = new UVector32(status); + if (offsets_ == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULLORDER; + } + } + // If we already have expansion CEs, then we also have offsets. + // Otherwise remember the trailing offset in case we need to + // write offsets for an artificial expansion. + int32_t limitOffset = iter_->getCEsLength() == 0 ? iter_->getOffset() : 0; + int64_t ce = iter_->previousCE(*offsets_, status); + if (ce == Collation::NO_CE) { return NULLORDER; } + // Turn the 64-bit CE into two old-style 32-bit CEs, without quaternary bits. + uint32_t p = (uint32_t)(ce >> 32); + uint32_t lower32 = (uint32_t)ce; + uint32_t firstHalf = getFirstHalf(p, lower32); + uint32_t secondHalf = getSecondHalf(p, lower32); + if (secondHalf != 0) { + if (offsets_->isEmpty()) { + // When we convert a single 64-bit CE into two 32-bit CEs, + // we need to make this artificial expansion behave like a normal expansion. + // See CollationIterator::previousCE(). + offsets_->addElement(iter_->getOffset(), status); + offsets_->addElement(limitOffset, status); + } + otherHalf_ = firstHalf; + return secondHalf | 0xc0; // continuation CE + } + return firstHalf; +} + +/** +* Resets the cursor to the beginning of the string. +*/ +void CollationElementIterator::reset() +{ + iter_ ->resetToOffset(0); + otherHalf_ = 0; + dir_ = 0; +} + +void CollationElementIterator::setOffset(int32_t newOffset, + UErrorCode& status) +{ + if (U_FAILURE(status)) { return; } + if (0 < newOffset && newOffset < string_.length()) { + int32_t offset = newOffset; + do { + char16_t c = string_.charAt(offset); + if (!rbc_->isUnsafe(c) || + (U16_IS_LEAD(c) && !rbc_->isUnsafe(string_.char32At(offset)))) { + break; + } + // Back up to before this unsafe character. + --offset; + } while (offset > 0); + if (offset < newOffset) { + // We might have backed up more than necessary. + // For example, contractions "ch" and "cu" make both 'h' and 'u' unsafe, + // but for text "chu" setOffset(2) should remain at 2 + // although we initially back up to offset 0. + // Find the last safe offset no greater than newOffset by iterating forward. + int32_t lastSafeOffset = offset; + do { + iter_->resetToOffset(lastSafeOffset); + do { + iter_->nextCE(status); + if (U_FAILURE(status)) { return; } + } while ((offset = iter_->getOffset()) == lastSafeOffset); + if (offset <= newOffset) { + lastSafeOffset = offset; + } + } while (offset < newOffset); + newOffset = lastSafeOffset; + } + } + iter_->resetToOffset(newOffset); + otherHalf_ = 0; + dir_ = 1; +} + +/** +* Sets the source to the new source string. +*/ +void CollationElementIterator::setText(const UnicodeString& source, + UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + + string_ = source; + const char16_t *s = string_.getBuffer(); + CollationIterator *newIter; + UBool numeric = rbc_->settings->isNumeric(); + if (rbc_->settings->dontCheckFCD()) { + newIter = new UTF16CollationIterator(rbc_->data, numeric, s, s, s + string_.length()); + } else { + newIter = new FCDUTF16CollationIterator(rbc_->data, numeric, s, s, s + string_.length()); + } + if (newIter == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + delete iter_; + iter_ = newIter; + otherHalf_ = 0; + dir_ = 0; +} + +// Sets the source to the new character iterator. +void CollationElementIterator::setText(CharacterIterator& source, + UErrorCode& status) +{ + if (U_FAILURE(status)) + return; + + source.getText(string_); + setText(string_, status); +} + +int32_t CollationElementIterator::strengthOrder(int32_t order) const +{ + UColAttributeValue s = (UColAttributeValue)rbc_->settings->getStrength(); + // Mask off the unwanted differences. + if (s == UCOL_PRIMARY) { + order &= 0xffff0000; + } + else if (s == UCOL_SECONDARY) { + order &= 0xffffff00; + } + + return order; +} + +/* CollationElementIterator private constructors/destructors --------------- */ + +/** +* This is the "real" constructor for this class; it constructs an iterator +* over the source text using the specified collator +*/ +CollationElementIterator::CollationElementIterator( + const UnicodeString &source, + const RuleBasedCollator *coll, + UErrorCode &status) + : iter_(nullptr), rbc_(coll), otherHalf_(0), dir_(0), offsets_(nullptr) { + setText(source, status); +} + +/** +* This is the "real" constructor for this class; it constructs an iterator over +* the source text using the specified collator +*/ +CollationElementIterator::CollationElementIterator( + const CharacterIterator &source, + const RuleBasedCollator *coll, + UErrorCode &status) + : iter_(nullptr), rbc_(coll), otherHalf_(0), dir_(0), offsets_(nullptr) { + // We only call source.getText() which should be const anyway. + setText(const_cast(source), status); +} + +/* CollationElementIterator private methods -------------------------------- */ + +const CollationElementIterator& CollationElementIterator::operator=( + const CollationElementIterator& other) +{ + if (this == &other) { + return *this; + } + + CollationIterator *newIter; + const FCDUTF16CollationIterator *otherFCDIter = + dynamic_cast(other.iter_); + if(otherFCDIter != nullptr) { + newIter = new FCDUTF16CollationIterator(*otherFCDIter, string_.getBuffer()); + } else { + const UTF16CollationIterator *otherIter = + dynamic_cast(other.iter_); + if(otherIter != nullptr) { + newIter = new UTF16CollationIterator(*otherIter, string_.getBuffer()); + } else { + newIter = nullptr; + } + } + if(newIter != nullptr) { + delete iter_; + iter_ = newIter; + rbc_ = other.rbc_; + otherHalf_ = other.otherHalf_; + dir_ = other.dir_; + + string_ = other.string_; + } + if(other.dir_ < 0 && other.offsets_ != nullptr && !other.offsets_->isEmpty()) { + UErrorCode errorCode = U_ZERO_ERROR; + if(offsets_ == nullptr) { + offsets_ = new UVector32(other.offsets_->size(), errorCode); + } + if(offsets_ != nullptr) { + offsets_->assign(*other.offsets_, errorCode); + } + } + return *this; +} + +namespace { + +class MaxExpSink : public ContractionsAndExpansions::CESink { +public: + MaxExpSink(UHashtable *h, UErrorCode &ec) : maxExpansions(h), errorCode(ec) {} + virtual ~MaxExpSink(); + virtual void handleCE(int64_t /*ce*/) override {} + virtual void handleExpansion(const int64_t ces[], int32_t length) override { + if (length <= 1) { + // We do not need to add single CEs into the map. + return; + } + int32_t count = 0; // number of CE "halves" + for (int32_t i = 0; i < length; ++i) { + count += ceNeedsTwoParts(ces[i]) ? 2 : 1; + } + // last "half" of the last CE + int64_t ce = ces[length - 1]; + uint32_t p = (uint32_t)(ce >> 32); + uint32_t lower32 = (uint32_t)ce; + uint32_t lastHalf = getSecondHalf(p, lower32); + if (lastHalf == 0) { + lastHalf = getFirstHalf(p, lower32); + U_ASSERT(lastHalf != 0); + } else { + lastHalf |= 0xc0; // old-style continuation CE + } + if (count > uhash_igeti(maxExpansions, (int32_t)lastHalf)) { + uhash_iputi(maxExpansions, (int32_t)lastHalf, count, &errorCode); + } + } + +private: + UHashtable *maxExpansions; + UErrorCode &errorCode; +}; + +MaxExpSink::~MaxExpSink() {} + +} // namespace + +UHashtable * +CollationElementIterator::computeMaxExpansions(const CollationData *data, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return nullptr; } + UHashtable *maxExpansions = uhash_open(uhash_hashLong, uhash_compareLong, + uhash_compareLong, &errorCode); + if (U_FAILURE(errorCode)) { return nullptr; } + MaxExpSink sink(maxExpansions, errorCode); + ContractionsAndExpansions(nullptr, nullptr, &sink, true).forData(data, errorCode); + if (U_FAILURE(errorCode)) { + uhash_close(maxExpansions); + return nullptr; + } + return maxExpansions; +} + +int32_t +CollationElementIterator::getMaxExpansion(int32_t order) const { + return getMaxExpansion(rbc_->tailoring->maxExpansions, order); +} + +int32_t +CollationElementIterator::getMaxExpansion(const UHashtable *maxExpansions, int32_t order) { + if (order == 0) { return 1; } + int32_t max; + if(maxExpansions != nullptr && (max = uhash_igeti(maxExpansions, order)) != 0) { + return max; + } + if ((order & 0xc0) == 0xc0) { + // old-style continuation CE + return 2; + } else { + return 1; + } +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_COLLATION */ diff --git a/intl/icu/source/i18n/coll.cpp b/intl/icu/source/i18n/coll.cpp new file mode 100644 index 0000000000..ced55c8dbf --- /dev/null +++ b/intl/icu/source/i18n/coll.cpp @@ -0,0 +1,1021 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ****************************************************************************** + * Copyright (C) 1996-2014, International Business Machines Corporation and + * others. All Rights Reserved. + ****************************************************************************** + */ + +/** + * File coll.cpp + * + * Created by: Helena Shih + * + * Modification History: + * + * Date Name Description + * 2/5/97 aliu Modified createDefault to load collation data from + * binary files when possible. Added related methods + * createCollationFromFile, chopLocale, createPathName. + * 2/11/97 aliu Added methods addToCache, findInCache, which implement + * a Collation cache. Modified createDefault to look in + * cache first, and also to store newly created Collation + * objects in the cache. Modified to not use gLocPath. + * 2/12/97 aliu Modified to create objects from RuleBasedCollator cache. + * Moved cache out of Collation class. + * 2/13/97 aliu Moved several methods out of this class and into + * RuleBasedCollator, with modifications. Modified + * createDefault() to call new RuleBasedCollator(Locale&) + * constructor. General clean up and documentation. + * 2/20/97 helena Added clone, operator==, operator!=, operator=, and copy + * constructor. + * 05/06/97 helena Added memory allocation error detection. + * 05/08/97 helena Added createInstance(). + * 6/20/97 helena Java class name change. + * 04/23/99 stephen Removed EDecompositionMode, merged with + * Normalizer::EMode + * 11/23/9 srl Inlining of some critical functions + * 01/29/01 synwee Modified into a C++ wrapper calling C APIs (ucol.h) + * 2012-2014 markus Rewritten in C++ again. + */ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/coll.h" +#include "unicode/tblcoll.h" +#include "collationdata.h" +#include "collationroot.h" +#include "collationtailoring.h" +#include "ucol_imp.h" +#include "cstring.h" +#include "cmemory.h" +#include "umutex.h" +#include "servloc.h" +#include "uassert.h" +#include "ustrenum.h" +#include "uresimp.h" +#include "ucln_in.h" + +static icu::Locale* availableLocaleList = nullptr; +static int32_t availableLocaleListCount; +#if !UCONFIG_NO_SERVICE +static icu::ICULocaleService* gService = nullptr; +static icu::UInitOnce gServiceInitOnce {}; +#endif +static icu::UInitOnce gAvailableLocaleListInitOnce {}; + +/** + * Release all static memory held by collator. + */ +U_CDECL_BEGIN +static UBool U_CALLCONV collator_cleanup() { +#if !UCONFIG_NO_SERVICE + if (gService) { + delete gService; + gService = nullptr; + } + gServiceInitOnce.reset(); +#endif + if (availableLocaleList) { + delete []availableLocaleList; + availableLocaleList = nullptr; + } + availableLocaleListCount = 0; + gAvailableLocaleListInitOnce.reset(); + return true; +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +#if !UCONFIG_NO_SERVICE + +// ------------------------------------------ +// +// Registration +// + +//------------------------------------------- + +CollatorFactory::~CollatorFactory() {} + +//------------------------------------------- + +UBool +CollatorFactory::visible() const { + return true; +} + +//------------------------------------------- + +UnicodeString& +CollatorFactory::getDisplayName(const Locale& objectLocale, + const Locale& displayLocale, + UnicodeString& result) +{ + return objectLocale.getDisplayName(displayLocale, result); +} + +// ------------------------------------- + +class ICUCollatorFactory : public ICUResourceBundleFactory { + public: + ICUCollatorFactory() : ICUResourceBundleFactory(UnicodeString(U_ICUDATA_COLL, -1, US_INV)) { } + virtual ~ICUCollatorFactory(); + protected: + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const override; +}; + +ICUCollatorFactory::~ICUCollatorFactory() {} + +UObject* +ICUCollatorFactory::create(const ICUServiceKey& key, const ICUService* /* service */, UErrorCode& status) const { + if (handlesKey(key, status)) { + const LocaleKey& lkey = static_cast(key); + Locale loc; + // make sure the requested locale is correct + // default LocaleFactory uses currentLocale since that's the one vetted by handlesKey + // but for ICU rb resources we use the actual one since it will fallback again + lkey.canonicalLocale(loc); + + return Collator::makeInstance(loc, status); + } + return nullptr; +} + +// ------------------------------------- + +class ICUCollatorService : public ICULocaleService { +public: + ICUCollatorService() + : ICULocaleService(UNICODE_STRING_SIMPLE("Collator")) + { + UErrorCode status = U_ZERO_ERROR; + registerFactory(new ICUCollatorFactory(), status); + } + + virtual ~ICUCollatorService(); + + virtual UObject* cloneInstance(UObject* instance) const override { + return ((Collator*)instance)->clone(); + } + + virtual UObject* handleDefault(const ICUServiceKey& key, UnicodeString* actualID, UErrorCode& status) const override { + const LocaleKey* lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + if (actualID) { + // Ugly Hack Alert! We return an empty actualID to signal + // to callers that this is a default object, not a "real" + // service-created object. (TODO remove in 3.0) [aliu] + actualID->truncate(0); + } + Locale loc(""); + lkey->canonicalLocale(loc); + return Collator::makeInstance(loc, status); + } + + virtual UObject* getKey(ICUServiceKey& key, UnicodeString* actualReturn, UErrorCode& status) const override { + UnicodeString ar; + if (actualReturn == nullptr) { + actualReturn = &ar; + } + return (Collator*)ICULocaleService::getKey(key, actualReturn, status); + } + + virtual UBool isDefault() const override { + return countFactories() == 1; + } +}; + +ICUCollatorService::~ICUCollatorService() {} + +// ------------------------------------- + +static void U_CALLCONV initService() { + gService = new ICUCollatorService(); + ucln_i18n_registerCleanup(UCLN_I18N_COLLATOR, collator_cleanup); +} + + +static ICULocaleService* +getService() +{ + umtx_initOnce(gServiceInitOnce, &initService); + return gService; +} + +// ------------------------------------- + +static inline UBool +hasService() +{ + UBool retVal = !gServiceInitOnce.isReset() && (getService() != nullptr); + return retVal; +} + +#endif /* UCONFIG_NO_SERVICE */ + +static void U_CALLCONV +initAvailableLocaleList(UErrorCode &status) { + U_ASSERT(availableLocaleListCount == 0); + U_ASSERT(availableLocaleList == nullptr); + // for now, there is a hardcoded list, so just walk through that list and set it up. + UResourceBundle *index = nullptr; + StackUResourceBundle installed; + int32_t i = 0; + + index = ures_openDirect(U_ICUDATA_COLL, "res_index", &status); + ures_getByKey(index, "InstalledLocales", installed.getAlias(), &status); + + if(U_SUCCESS(status)) { + availableLocaleListCount = ures_getSize(installed.getAlias()); + availableLocaleList = new Locale[availableLocaleListCount]; + + if (availableLocaleList != nullptr) { + ures_resetIterator(installed.getAlias()); + while(ures_hasNext(installed.getAlias())) { + const char *tempKey = nullptr; + ures_getNextString(installed.getAlias(), nullptr, &tempKey, &status); + availableLocaleList[i++] = Locale(tempKey); + } + } + U_ASSERT(availableLocaleListCount == i); + } + ures_close(index); + ucln_i18n_registerCleanup(UCLN_I18N_COLLATOR, collator_cleanup); +} + +static UBool isAvailableLocaleListInitialized(UErrorCode &status) { + umtx_initOnce(gAvailableLocaleListInitOnce, &initAvailableLocaleList, status); + return U_SUCCESS(status); +} + + +// Collator public methods ----------------------------------------------- + +namespace { + +static const struct { + const char *name; + UColAttribute attr; +} collAttributes[] = { + { "colStrength", UCOL_STRENGTH }, + { "colBackwards", UCOL_FRENCH_COLLATION }, + { "colCaseLevel", UCOL_CASE_LEVEL }, + { "colCaseFirst", UCOL_CASE_FIRST }, + { "colAlternate", UCOL_ALTERNATE_HANDLING }, + { "colNormalization", UCOL_NORMALIZATION_MODE }, + { "colNumeric", UCOL_NUMERIC_COLLATION } +}; + +static const struct { + const char *name; + UColAttributeValue value; +} collAttributeValues[] = { + { "primary", UCOL_PRIMARY }, + { "secondary", UCOL_SECONDARY }, + { "tertiary", UCOL_TERTIARY }, + { "quaternary", UCOL_QUATERNARY }, + // Note: Not supporting typo "quarternary" because it was never supported in locale IDs. + { "identical", UCOL_IDENTICAL }, + { "no", UCOL_OFF }, + { "yes", UCOL_ON }, + { "shifted", UCOL_SHIFTED }, + { "non-ignorable", UCOL_NON_IGNORABLE }, + { "lower", UCOL_LOWER_FIRST }, + { "upper", UCOL_UPPER_FIRST } +}; + +static const char *collReorderCodes[UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST] = { + "space", "punct", "symbol", "currency", "digit" +}; + +int32_t getReorderCode(const char *s) { + for (int32_t i = 0; i < UPRV_LENGTHOF(collReorderCodes); ++i) { + if (uprv_stricmp(s, collReorderCodes[i]) == 0) { + return UCOL_REORDER_CODE_FIRST + i; + } + } + // Not supporting "others" = UCOL_REORDER_CODE_OTHERS + // as a synonym for Zzzz = USCRIPT_UNKNOWN for now: + // Avoid introducing synonyms/aliases. + return -1; +} + +/** + * Sets collation attributes according to locale keywords. See + * http://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Settings + * + * Using "alias" keywords and values where defined: + * http://www.unicode.org/reports/tr35/tr35.html#Old_Locale_Extension_Syntax + * http://unicode.org/repos/cldr/trunk/common/bcp47/collation.xml + */ +void setAttributesFromKeywords(const Locale &loc, Collator &coll, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return; + } + if (uprv_strcmp(loc.getName(), loc.getBaseName()) == 0) { + // No keywords. + return; + } + char value[1024]; // The reordering value could be long. + // Check for collation keywords that were already deprecated + // before any were supported in createInstance() (except for "collation"). + int32_t length = loc.getKeywordValue("colHiraganaQuaternary", value, UPRV_LENGTHOF(value), errorCode); + if (U_FAILURE(errorCode)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (length != 0) { + errorCode = U_UNSUPPORTED_ERROR; + return; + } + length = loc.getKeywordValue("variableTop", value, UPRV_LENGTHOF(value), errorCode); + if (U_FAILURE(errorCode)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (length != 0) { + errorCode = U_UNSUPPORTED_ERROR; + return; + } + // Parse known collation keywords, ignore others. + if (errorCode == U_STRING_NOT_TERMINATED_WARNING) { + errorCode = U_ZERO_ERROR; + } + for (int32_t i = 0; i < UPRV_LENGTHOF(collAttributes); ++i) { + length = loc.getKeywordValue(collAttributes[i].name, value, UPRV_LENGTHOF(value), errorCode); + if (U_FAILURE(errorCode) || errorCode == U_STRING_NOT_TERMINATED_WARNING) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (length == 0) { continue; } + for (int32_t j = 0;; ++j) { + if (j == UPRV_LENGTHOF(collAttributeValues)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (uprv_stricmp(value, collAttributeValues[j].name) == 0) { + coll.setAttribute(collAttributes[i].attr, collAttributeValues[j].value, errorCode); + break; + } + } + } + length = loc.getKeywordValue("colReorder", value, UPRV_LENGTHOF(value), errorCode); + if (U_FAILURE(errorCode) || errorCode == U_STRING_NOT_TERMINATED_WARNING) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (length != 0) { + int32_t codes[USCRIPT_CODE_LIMIT + (UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST)]; + int32_t codesLength = 0; + char *scriptName = value; + for (;;) { + if (codesLength == UPRV_LENGTHOF(codes)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + char *limit = scriptName; + char c; + while ((c = *limit) != 0 && c != '-') { ++limit; } + *limit = 0; + int32_t code; + if ((limit - scriptName) == 4) { + // Strict parsing, accept only 4-letter script codes, not long names. + code = u_getPropertyValueEnum(UCHAR_SCRIPT, scriptName); + } else { + code = getReorderCode(scriptName); + } + if (code < 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + codes[codesLength++] = code; + if (c == 0) { break; } + scriptName = limit + 1; + } + coll.setReorderCodes(codes, codesLength, errorCode); + } + length = loc.getKeywordValue("kv", value, UPRV_LENGTHOF(value), errorCode); + if (U_FAILURE(errorCode) || errorCode == U_STRING_NOT_TERMINATED_WARNING) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (length != 0) { + int32_t code = getReorderCode(value); + if (code < 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + coll.setMaxVariable((UColReorderCode)code, errorCode); + } + if (U_FAILURE(errorCode)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + } +} + +} // namespace + +Collator* U_EXPORT2 Collator::createInstance(UErrorCode& success) +{ + return createInstance(Locale::getDefault(), success); +} + +Collator* U_EXPORT2 Collator::createInstance(const Locale& desiredLocale, + UErrorCode& status) +{ + if (U_FAILURE(status)) + return 0; + if (desiredLocale.isBogus()) { + // Locale constructed from malformed locale ID or language tag. + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + Collator* coll; +#if !UCONFIG_NO_SERVICE + if (hasService()) { + Locale actualLoc; + coll = (Collator*)gService->get(desiredLocale, &actualLoc, status); + } else +#endif + { + coll = makeInstance(desiredLocale, status); + // Either returns nullptr with U_FAILURE(status), or non-nullptr with U_SUCCESS(status) + } + // The use of *coll in setAttributesFromKeywords can cause the nullptr check to be + // optimized out of the delete even though setAttributesFromKeywords returns + // immediately if U_FAILURE(status), so we add a check here. + if (U_FAILURE(status)) { + return nullptr; + } + setAttributesFromKeywords(desiredLocale, *coll, status); + if (U_FAILURE(status)) { + delete coll; + return nullptr; + } + return coll; +} + + +Collator* Collator::makeInstance(const Locale& desiredLocale, UErrorCode& status) { + const CollationCacheEntry *entry = CollationLoader::loadTailoring(desiredLocale, status); + if (U_SUCCESS(status)) { + Collator *result = new RuleBasedCollator(entry); + if (result != nullptr) { + // Both the unified cache's get() and the RBC constructor + // did addRef(). Undo one of them. + entry->removeRef(); + return result; + } + status = U_MEMORY_ALLOCATION_ERROR; + } + if (entry != nullptr) { + // Undo the addRef() from the cache.get(). + entry->removeRef(); + } + return nullptr; +} + +Collator * +Collator::safeClone() const { + return clone(); +} + +// implement deprecated, previously abstract method +Collator::EComparisonResult Collator::compare(const UnicodeString& source, + const UnicodeString& target) const +{ + UErrorCode ec = U_ZERO_ERROR; + return (EComparisonResult)compare(source, target, ec); +} + +// implement deprecated, previously abstract method +Collator::EComparisonResult Collator::compare(const UnicodeString& source, + const UnicodeString& target, + int32_t length) const +{ + UErrorCode ec = U_ZERO_ERROR; + return (EComparisonResult)compare(source, target, length, ec); +} + +// implement deprecated, previously abstract method +Collator::EComparisonResult Collator::compare(const char16_t* source, int32_t sourceLength, + const char16_t* target, int32_t targetLength) + const +{ + UErrorCode ec = U_ZERO_ERROR; + return (EComparisonResult)compare(source, sourceLength, target, targetLength, ec); +} + +UCollationResult Collator::compare(UCharIterator &/*sIter*/, + UCharIterator &/*tIter*/, + UErrorCode &status) const { + if(U_SUCCESS(status)) { + // Not implemented in the base class. + status = U_UNSUPPORTED_ERROR; + } + return UCOL_EQUAL; +} + +UCollationResult Collator::compareUTF8(const StringPiece &source, + const StringPiece &target, + UErrorCode &status) const { + if(U_FAILURE(status)) { + return UCOL_EQUAL; + } + UCharIterator sIter, tIter; + uiter_setUTF8(&sIter, source.data(), source.length()); + uiter_setUTF8(&tIter, target.data(), target.length()); + return compare(sIter, tIter, status); +} + +UBool Collator::equals(const UnicodeString& source, + const UnicodeString& target) const +{ + UErrorCode ec = U_ZERO_ERROR; + return (compare(source, target, ec) == UCOL_EQUAL); +} + +UBool Collator::greaterOrEqual(const UnicodeString& source, + const UnicodeString& target) const +{ + UErrorCode ec = U_ZERO_ERROR; + return (compare(source, target, ec) != UCOL_LESS); +} + +UBool Collator::greater(const UnicodeString& source, + const UnicodeString& target) const +{ + UErrorCode ec = U_ZERO_ERROR; + return (compare(source, target, ec) == UCOL_GREATER); +} + +// this API ignores registered collators, since it returns an +// array of indefinite lifetime +const Locale* U_EXPORT2 Collator::getAvailableLocales(int32_t& count) +{ + UErrorCode status = U_ZERO_ERROR; + Locale *result = nullptr; + count = 0; + if (isAvailableLocaleListInitialized(status)) + { + result = availableLocaleList; + count = availableLocaleListCount; + } + return result; +} + +UnicodeString& U_EXPORT2 Collator::getDisplayName(const Locale& objectLocale, + const Locale& displayLocale, + UnicodeString& name) +{ +#if !UCONFIG_NO_SERVICE + if (hasService()) { + UnicodeString locNameStr; + LocaleUtility::initNameFromLocale(objectLocale, locNameStr); + return gService->getDisplayName(locNameStr, name, displayLocale); + } +#endif + return objectLocale.getDisplayName(displayLocale, name); +} + +UnicodeString& U_EXPORT2 Collator::getDisplayName(const Locale& objectLocale, + UnicodeString& name) +{ + return getDisplayName(objectLocale, Locale::getDefault(), name); +} + +/* This is useless information */ +/*void Collator::getVersion(UVersionInfo versionInfo) const +{ + if (versionInfo!=nullptr) + uprv_memcpy(versionInfo, fVersion, U_MAX_VERSION_LENGTH); +} +*/ + +// UCollator protected constructor destructor ---------------------------- + +/** +* Default constructor. +* Constructor is different from the old default Collator constructor. +* The task for determining the default collation strength and normalization mode +* is left to the child class. +*/ +Collator::Collator() +: UObject() +{ +} + +/** +* Constructor. +* Empty constructor, does not handle the arguments. +* This constructor is done for backward compatibility with 1.7 and 1.8. +* The task for handling the argument collation strength and normalization +* mode is left to the child class. +* @param collationStrength collation strength +* @param decompositionMode +* @deprecated 2.4 use the default constructor instead +*/ +Collator::Collator(UCollationStrength, UNormalizationMode ) +: UObject() +{ +} + +Collator::~Collator() +{ +} + +Collator::Collator(const Collator &other) + : UObject(other) +{ +} + +bool Collator::operator==(const Collator& other) const +{ + // Subclasses: Call this method and then add more specific checks. + return typeid(*this) == typeid(other); +} + +bool Collator::operator!=(const Collator& other) const +{ + return !operator==(other); +} + +int32_t U_EXPORT2 Collator::getBound(const uint8_t *source, + int32_t sourceLength, + UColBoundMode boundType, + uint32_t noOfLevels, + uint8_t *result, + int32_t resultLength, + UErrorCode &status) +{ + return ucol_getBound(source, sourceLength, boundType, noOfLevels, result, resultLength, &status); +} + +void +Collator::setLocales(const Locale& /* requestedLocale */, const Locale& /* validLocale */, const Locale& /*actualLocale*/) { +} + +UnicodeSet *Collator::getTailoredSet(UErrorCode &status) const +{ + if(U_FAILURE(status)) { + return nullptr; + } + // everything can be changed + return new UnicodeSet(0, 0x10FFFF); +} + +// ------------------------------------- + +#if !UCONFIG_NO_SERVICE +URegistryKey U_EXPORT2 +Collator::registerInstance(Collator* toAdopt, const Locale& locale, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + // Set the collator locales while registering so that createInstance() + // need not guess whether the collator's locales are already set properly + // (as they are by the data loader). + toAdopt->setLocales(locale, locale, locale); + return getService()->registerInstance(toAdopt, locale, status); + } + return nullptr; +} + +// ------------------------------------- + +class CFactory : public LocaleKeyFactory { +private: + CollatorFactory* _delegate; + Hashtable* _ids; + +public: + CFactory(CollatorFactory* delegate, UErrorCode& status) + : LocaleKeyFactory(delegate->visible() ? VISIBLE : INVISIBLE) + , _delegate(delegate) + , _ids(nullptr) + { + if (U_SUCCESS(status)) { + int32_t count = 0; + _ids = new Hashtable(status); + if (_ids) { + const UnicodeString * idlist = _delegate->getSupportedIDs(count, status); + for (int i = 0; i < count; ++i) { + _ids->put(idlist[i], (void*)this, status); + if (U_FAILURE(status)) { + delete _ids; + _ids = nullptr; + return; + } + } + } else { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + } + + virtual ~CFactory(); + + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const override; + +protected: + virtual const Hashtable* getSupportedIDs(UErrorCode& status) const override + { + if (U_SUCCESS(status)) { + return _ids; + } + return nullptr; + } + + virtual UnicodeString& + getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const override; +}; + +CFactory::~CFactory() +{ + delete _delegate; + delete _ids; +} + +UObject* +CFactory::create(const ICUServiceKey& key, const ICUService* /* service */, UErrorCode& status) const +{ + if (handlesKey(key, status)) { + const LocaleKey* lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + Locale validLoc; + lkey->currentLocale(validLoc); + return _delegate->createCollator(validLoc); + } + return nullptr; +} + +UnicodeString& +CFactory::getDisplayName(const UnicodeString& id, const Locale& locale, UnicodeString& result) const +{ + if ((_coverage & 0x1) == 0) { + UErrorCode status = U_ZERO_ERROR; + const Hashtable* ids = getSupportedIDs(status); + if (ids && (ids->get(id) != nullptr)) { + Locale loc; + LocaleUtility::initLocaleFromName(id, loc); + return _delegate->getDisplayName(loc, locale, result); + } + } + result.setToBogus(); + return result; +} + +URegistryKey U_EXPORT2 +Collator::registerFactory(CollatorFactory* toAdopt, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + CFactory* f = new CFactory(toAdopt, status); + if (f) { + return getService()->registerFactory(f, status); + } + status = U_MEMORY_ALLOCATION_ERROR; + } + return nullptr; +} + +// ------------------------------------- + +UBool U_EXPORT2 +Collator::unregister(URegistryKey key, UErrorCode& status) +{ + if (U_SUCCESS(status)) { + if (hasService()) { + return gService->unregister(key, status); + } + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return false; +} +#endif /* UCONFIG_NO_SERVICE */ + +class CollationLocaleListEnumeration : public StringEnumeration { +private: + int32_t index; +public: + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +public: + CollationLocaleListEnumeration() + : index(0) + { + // The global variables should already be initialized. + //isAvailableLocaleListInitialized(status); + } + + virtual ~CollationLocaleListEnumeration(); + + virtual StringEnumeration * clone() const override + { + CollationLocaleListEnumeration *result = new CollationLocaleListEnumeration(); + if (result) { + result->index = index; + } + return result; + } + + virtual int32_t count(UErrorCode &/*status*/) const override { + return availableLocaleListCount; + } + + virtual const char* next(int32_t* resultLength, UErrorCode& /*status*/) override { + const char* result; + if(index < availableLocaleListCount) { + result = availableLocaleList[index++].getName(); + if(resultLength != nullptr) { + *resultLength = (int32_t)uprv_strlen(result); + } + } else { + if(resultLength != nullptr) { + *resultLength = 0; + } + result = nullptr; + } + return result; + } + + virtual const UnicodeString* snext(UErrorCode& status) override { + int32_t resultLength = 0; + const char *s = next(&resultLength, status); + return setChars(s, resultLength, status); + } + + virtual void reset(UErrorCode& /*status*/) override { + index = 0; + } +}; + +CollationLocaleListEnumeration::~CollationLocaleListEnumeration() {} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CollationLocaleListEnumeration) + + +// ------------------------------------- + +StringEnumeration* U_EXPORT2 +Collator::getAvailableLocales() +{ +#if !UCONFIG_NO_SERVICE + if (hasService()) { + return getService()->getAvailableLocales(); + } +#endif /* UCONFIG_NO_SERVICE */ + UErrorCode status = U_ZERO_ERROR; + if (isAvailableLocaleListInitialized(status)) { + return new CollationLocaleListEnumeration(); + } + return nullptr; +} + +StringEnumeration* U_EXPORT2 +Collator::getKeywords(UErrorCode& status) { + return UStringEnumeration::fromUEnumeration( + ucol_getKeywords(&status), status); +} + +StringEnumeration* U_EXPORT2 +Collator::getKeywordValues(const char *keyword, UErrorCode& status) { + return UStringEnumeration::fromUEnumeration( + ucol_getKeywordValues(keyword, &status), status); +} + +StringEnumeration* U_EXPORT2 +Collator::getKeywordValuesForLocale(const char* key, const Locale& locale, + UBool commonlyUsed, UErrorCode& status) { + return UStringEnumeration::fromUEnumeration( + ucol_getKeywordValuesForLocale( + key, locale.getName(), commonlyUsed, &status), + status); +} + +Locale U_EXPORT2 +Collator::getFunctionalEquivalent(const char* keyword, const Locale& locale, + UBool& isAvailable, UErrorCode& status) { + // This is a wrapper over ucol_getFunctionalEquivalent + char loc[ULOC_FULLNAME_CAPACITY]; + /*int32_t len =*/ ucol_getFunctionalEquivalent(loc, sizeof(loc), + keyword, locale.getName(), &isAvailable, &status); + if (U_FAILURE(status)) { + *loc = 0; // root + } + return Locale::createFromName(loc); +} + +Collator::ECollationStrength +Collator::getStrength() const { + UErrorCode intStatus = U_ZERO_ERROR; + return (ECollationStrength)getAttribute(UCOL_STRENGTH, intStatus); +} + +void +Collator::setStrength(ECollationStrength newStrength) { + UErrorCode intStatus = U_ZERO_ERROR; + setAttribute(UCOL_STRENGTH, (UColAttributeValue)newStrength, intStatus); +} + +Collator & +Collator::setMaxVariable(UColReorderCode /*group*/, UErrorCode &errorCode) { + if (U_SUCCESS(errorCode)) { + errorCode = U_UNSUPPORTED_ERROR; + } + return *this; +} + +UColReorderCode +Collator::getMaxVariable() const { + return UCOL_REORDER_CODE_PUNCTUATION; +} + +int32_t +Collator::getReorderCodes(int32_t* /* dest*/, + int32_t /* destCapacity*/, + UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } + return 0; +} + +void +Collator::setReorderCodes(const int32_t* /* reorderCodes */, + int32_t /* reorderCodesLength */, + UErrorCode& status) +{ + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } +} + +int32_t +Collator::getEquivalentReorderCodes(int32_t reorderCode, + int32_t *dest, int32_t capacity, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + if(capacity < 0 || (dest == nullptr && capacity > 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + const CollationData *baseData = CollationRoot::getData(errorCode); + if(U_FAILURE(errorCode)) { return 0; } + return baseData->getEquivalentScripts(reorderCode, dest, capacity, errorCode); +} + +int32_t +Collator::internalGetShortDefinitionString(const char * /*locale*/, + char * /*buffer*/, + int32_t /*capacity*/, + UErrorCode &status) const { + if(U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; /* Shouldn't happen, internal function */ + } + return 0; +} + +UCollationResult +Collator::internalCompareUTF8(const char *left, int32_t leftLength, + const char *right, int32_t rightLength, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + if((left == nullptr && leftLength != 0) || (right == nullptr && rightLength != 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_EQUAL; + } + return compareUTF8( + StringPiece(left, (leftLength < 0) ? static_cast(uprv_strlen(left)) : leftLength), + StringPiece(right, (rightLength < 0) ? static_cast(uprv_strlen(right)) : rightLength), + errorCode); +} + +int32_t +Collator::internalNextSortKeyPart(UCharIterator * /*iter*/, uint32_t /*state*/[2], + uint8_t * /*dest*/, int32_t /*count*/, UErrorCode &errorCode) const { + if (U_SUCCESS(errorCode)) { + errorCode = U_UNSUPPORTED_ERROR; + } + return 0; +} + +// UCollator private data members ---------------------------------------- + +/* This is useless information */ +/*const UVersionInfo Collator::fVersion = {1, 1, 0, 0};*/ + +// ------------------------------------- + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_COLLATION */ + +/* eof */ diff --git a/intl/icu/source/i18n/collation.cpp b/intl/icu/source/i18n/collation.cpp new file mode 100644 index 0000000000..705ee12e23 --- /dev/null +++ b/intl/icu/source/i18n/collation.cpp @@ -0,0 +1,141 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collation.cpp +* +* created on: 2010oct27 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "collation.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +uint32_t +Collation::incTwoBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, int32_t offset) { + // Extract the second byte, minus the minimum byte value, + // plus the offset, modulo the number of usable byte values, plus the minimum. + // Reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary. + uint32_t primary; + if(isCompressible) { + offset += ((int32_t)(basePrimary >> 16) & 0xff) - 4; + primary = (uint32_t)((offset % 251) + 4) << 16; + offset /= 251; + } else { + offset += ((int32_t)(basePrimary >> 16) & 0xff) - 2; + primary = (uint32_t)((offset % 254) + 2) << 16; + offset /= 254; + } + // First byte, assume no further overflow. + return primary | ((basePrimary & 0xff000000) + (uint32_t)(offset << 24)); +} + +uint32_t +Collation::incThreeBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, int32_t offset) { + // Extract the third byte, minus the minimum byte value, + // plus the offset, modulo the number of usable byte values, plus the minimum. + offset += ((int32_t)(basePrimary >> 8) & 0xff) - 2; + uint32_t primary = (uint32_t)((offset % 254) + 2) << 8; + offset /= 254; + // Same with the second byte, + // but reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary. + if(isCompressible) { + offset += ((int32_t)(basePrimary >> 16) & 0xff) - 4; + primary |= (uint32_t)((offset % 251) + 4) << 16; + offset /= 251; + } else { + offset += ((int32_t)(basePrimary >> 16) & 0xff) - 2; + primary |= (uint32_t)((offset % 254) + 2) << 16; + offset /= 254; + } + // First byte, assume no further overflow. + return primary | ((basePrimary & 0xff000000) + (uint32_t)(offset << 24)); +} + +uint32_t +Collation::decTwoBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step) { + // Extract the second byte, minus the minimum byte value, + // minus the step, modulo the number of usable byte values, plus the minimum. + // Reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary. + // Assume no further underflow for the first byte. + U_ASSERT(0 < step && step <= 0x7f); + int32_t byte2 = ((int32_t)(basePrimary >> 16) & 0xff) - step; + if(isCompressible) { + if(byte2 < 4) { + byte2 += 251; + basePrimary -= 0x1000000; + } + } else { + if(byte2 < 2) { + byte2 += 254; + basePrimary -= 0x1000000; + } + } + return (basePrimary & 0xff000000) | ((uint32_t)byte2 << 16); +} + +uint32_t +Collation::decThreeBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step) { + // Extract the third byte, minus the minimum byte value, + // minus the step, modulo the number of usable byte values, plus the minimum. + U_ASSERT(0 < step && step <= 0x7f); + int32_t byte3 = ((int32_t)(basePrimary >> 8) & 0xff) - step; + if(byte3 >= 2) { + return (basePrimary & 0xffff0000) | ((uint32_t)byte3 << 8); + } + byte3 += 254; + // Same with the second byte, + // but reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary. + int32_t byte2 = ((int32_t)(basePrimary >> 16) & 0xff) - 1; + if(isCompressible) { + if(byte2 < 4) { + byte2 = 0xfe; + basePrimary -= 0x1000000; + } + } else { + if(byte2 < 2) { + byte2 = 0xff; + basePrimary -= 0x1000000; + } + } + // First byte, assume no further underflow. + return (basePrimary & 0xff000000) | ((uint32_t)byte2 << 16) | ((uint32_t)byte3 << 8); +} + +uint32_t +Collation::getThreeBytePrimaryForOffsetData(UChar32 c, int64_t dataCE) { + uint32_t p = (uint32_t)(dataCE >> 32); // three-byte primary pppppp00 + int32_t lower32 = (int32_t)dataCE; // base code point b & step s: bbbbbbss (bit 7: isCompressible) + int32_t offset = (c - (lower32 >> 8)) * (lower32 & 0x7f); // delta * increment + UBool isCompressible = (lower32 & 0x80) != 0; + return Collation::incThreeBytePrimaryByOffset(p, isCompressible, offset); +} + +uint32_t +Collation::unassignedPrimaryFromCodePoint(UChar32 c) { + // Create a gap before U+0000. Use c=-1 for [first unassigned]. + ++c; + // Fourth byte: 18 values, every 14th byte value (gap of 13). + uint32_t primary = 2 + (c % 18) * 14; + c /= 18; + // Third byte: 254 values. + primary |= (2 + (c % 254)) << 8; + c /= 254; + // Second byte: 251 values 04..FE excluding the primary compression bytes. + primary |= (4 + (c % 251)) << 16; + // One lead byte covers all code points (c < 0x1182B4 = 1*251*254*18). + return primary | (UNASSIGNED_IMPLICIT_BYTE << 24); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collation.h b/intl/icu/source/i18n/collation.h new file mode 100644 index 0000000000..2062ef2946 --- /dev/null +++ b/intl/icu/source/i18n/collation.h @@ -0,0 +1,503 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collation.h +* +* created on: 2010oct27 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATION_H__ +#define __COLLATION_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +U_NAMESPACE_BEGIN + +/** + * Collation v2 basic definitions and static helper functions. + * + * Data structures except for expansion tables store 32-bit CEs which are + * either specials (see tags below) or are compact forms of 64-bit CEs. + */ +class U_I18N_API Collation { +public: + // Special sort key bytes for all levels. + static const uint8_t TERMINATOR_BYTE = 0; + static const uint8_t LEVEL_SEPARATOR_BYTE = 1; + + /** The secondary/tertiary lower limit for tailoring before any root elements. */ + static const uint32_t BEFORE_WEIGHT16 = 0x0100; + + /** + * Merge-sort-key separator. + * Same as the unique primary and identical-level weights of U+FFFE. + * Must not be used as primary compression low terminator. + * Otherwise usable. + */ + static const uint8_t MERGE_SEPARATOR_BYTE = 2; + static const uint32_t MERGE_SEPARATOR_PRIMARY = 0x02000000; // U+FFFE + static const uint32_t MERGE_SEPARATOR_CE32 = 0x02000505; // U+FFFE + + /** + * Primary compression low terminator, must be greater than MERGE_SEPARATOR_BYTE. + * Reserved value in primary second byte if the lead byte is compressible. + * Otherwise usable in all CE weight bytes. + */ + static const uint8_t PRIMARY_COMPRESSION_LOW_BYTE = 3; + /** + * Primary compression high terminator. + * Reserved value in primary second byte if the lead byte is compressible. + * Otherwise usable in all CE weight bytes. + */ + static const uint8_t PRIMARY_COMPRESSION_HIGH_BYTE = 0xff; + + /** Default secondary/tertiary weight lead byte. */ + static const uint8_t COMMON_BYTE = 5; + static const uint32_t COMMON_WEIGHT16 = 0x0500; + /** Middle 16 bits of a CE with a common secondary weight. */ + static const uint32_t COMMON_SECONDARY_CE = 0x05000000; + /** Lower 16 bits of a CE with a common tertiary weight. */ + static const uint32_t COMMON_TERTIARY_CE = 0x0500; + /** Lower 32 bits of a CE with common secondary and tertiary weights. */ + static const uint32_t COMMON_SEC_AND_TER_CE = 0x05000500; + + static const uint32_t SECONDARY_MASK = 0xffff0000; + static const uint32_t CASE_MASK = 0xc000; + static const uint32_t SECONDARY_AND_CASE_MASK = SECONDARY_MASK | CASE_MASK; + /** Only the 2*6 bits for the pure tertiary weight. */ + static const uint32_t ONLY_TERTIARY_MASK = 0x3f3f; + /** Only the secondary & tertiary bits; no case, no quaternary. */ + static const uint32_t ONLY_SEC_TER_MASK = SECONDARY_MASK | ONLY_TERTIARY_MASK; + /** Case bits and tertiary bits. */ + static const uint32_t CASE_AND_TERTIARY_MASK = CASE_MASK | ONLY_TERTIARY_MASK; + static const uint32_t QUATERNARY_MASK = 0xc0; + /** Case bits and quaternary bits. */ + static const uint32_t CASE_AND_QUATERNARY_MASK = CASE_MASK | QUATERNARY_MASK; + + static const uint8_t UNASSIGNED_IMPLICIT_BYTE = 0xfe; // compressible + /** + * First unassigned: AlphabeticIndex overflow boundary. + * We want a 3-byte primary so that it fits into the root elements table. + * + * This 3-byte primary will not collide with + * any unassigned-implicit 4-byte primaries because + * the first few hundred Unicode code points all have real mappings. + */ + static const uint32_t FIRST_UNASSIGNED_PRIMARY = 0xfe040200; + + static const uint8_t TRAIL_WEIGHT_BYTE = 0xff; // not compressible + static const uint32_t FIRST_TRAILING_PRIMARY = 0xff020200; // [first trailing] + static const uint32_t MAX_PRIMARY = 0xffff0000; // U+FFFF + static const uint32_t MAX_REGULAR_CE32 = 0xffff0505; // U+FFFF + + // CE32 value for U+FFFD as well as illegal UTF-8 byte sequences (which behave like U+FFFD). + // We use the third-highest primary weight for U+FFFD (as in UCA 6.3+). + static const uint32_t FFFD_PRIMARY = MAX_PRIMARY - 0x20000; + static const uint32_t FFFD_CE32 = MAX_REGULAR_CE32 - 0x20000; + + /** + * A CE32 is special if its low byte is this or greater. + * Impossible case bits 11 mark special CE32s. + * This value itself is used to indicate a fallback to the base collator. + */ + static const uint8_t SPECIAL_CE32_LOW_BYTE = 0xc0; + static const uint32_t FALLBACK_CE32 = SPECIAL_CE32_LOW_BYTE; + /** + * Low byte of a long-primary special CE32. + */ + static const uint8_t LONG_PRIMARY_CE32_LOW_BYTE = 0xc1; // SPECIAL_CE32_LOW_BYTE | LONG_PRIMARY_TAG + + static const uint32_t UNASSIGNED_CE32 = 0xffffffff; // Compute an unassigned-implicit CE. + + static const uint32_t NO_CE32 = 1; + + /** No CE: End of input. Only used in runtime code, not stored in data. */ + static const uint32_t NO_CE_PRIMARY = 1; // not a left-adjusted weight + static const uint32_t NO_CE_WEIGHT16 = 0x0100; // weight of LEVEL_SEPARATOR_BYTE + static const int64_t NO_CE = INT64_C(0x101000100); // NO_CE_PRIMARY, NO_CE_WEIGHT16, NO_CE_WEIGHT16 + + /** Sort key levels. */ + enum Level { + /** Unspecified level. */ + NO_LEVEL, + PRIMARY_LEVEL, + SECONDARY_LEVEL, + CASE_LEVEL, + TERTIARY_LEVEL, + QUATERNARY_LEVEL, + IDENTICAL_LEVEL, + /** Beyond sort key bytes. */ + ZERO_LEVEL + }; + + /** + * Sort key level flags: xx_FLAG = 1 << xx_LEVEL. + * In Java, use enum Level with flag() getters, or use EnumSet rather than hand-made bit sets. + */ + static const uint32_t NO_LEVEL_FLAG = 1; + static const uint32_t PRIMARY_LEVEL_FLAG = 2; + static const uint32_t SECONDARY_LEVEL_FLAG = 4; + static const uint32_t CASE_LEVEL_FLAG = 8; + static const uint32_t TERTIARY_LEVEL_FLAG = 0x10; + static const uint32_t QUATERNARY_LEVEL_FLAG = 0x20; + static const uint32_t IDENTICAL_LEVEL_FLAG = 0x40; + static const uint32_t ZERO_LEVEL_FLAG = 0x80; + + /** + * Special-CE32 tags, from bits 3..0 of a special 32-bit CE. + * Bits 31..8 are available for tag-specific data. + * Bits 5..4: Reserved. May be used in the future to indicate lccc!=0 and tccc!=0. + */ + enum { + /** + * Fall back to the base collator. + * This is the tag value in SPECIAL_CE32_LOW_BYTE and FALLBACK_CE32. + * Bits 31..8: Unused, 0. + */ + FALLBACK_TAG = 0, + /** + * Long-primary CE with COMMON_SEC_AND_TER_CE. + * Bits 31..8: Three-byte primary. + */ + LONG_PRIMARY_TAG = 1, + /** + * Long-secondary CE with zero primary. + * Bits 31..16: Secondary weight. + * Bits 15.. 8: Tertiary weight. + */ + LONG_SECONDARY_TAG = 2, + /** + * Unused. + * May be used in the future for single-byte secondary CEs (SHORT_SECONDARY_TAG), + * storing the secondary in bits 31..24, the ccc in bits 23..16, + * and the tertiary in bits 15..8. + */ + RESERVED_TAG_3 = 3, + /** + * Latin mini expansions of two simple CEs [pp, 05, tt] [00, ss, 05]. + * Bits 31..24: Single-byte primary weight pp of the first CE. + * Bits 23..16: Tertiary weight tt of the first CE. + * Bits 15.. 8: Secondary weight ss of the second CE. + */ + LATIN_EXPANSION_TAG = 4, + /** + * Points to one or more simple/long-primary/long-secondary 32-bit CE32s. + * Bits 31..13: Index into uint32_t table. + * Bits 12.. 8: Length=1..31. + */ + EXPANSION32_TAG = 5, + /** + * Points to one or more 64-bit CEs. + * Bits 31..13: Index into CE table. + * Bits 12.. 8: Length=1..31. + */ + EXPANSION_TAG = 6, + /** + * Builder data, used only in the CollationDataBuilder, not in runtime data. + * + * If bit 8 is 0: Builder context, points to a list of context-sensitive mappings. + * Bits 31..13: Index to the builder's list of ConditionalCE32 for this character. + * Bits 12.. 9: Unused, 0. + * + * If bit 8 is 1 (IS_BUILDER_JAMO_CE32): Builder-only jamoCE32 value. + * The builder fetches the Jamo CE32 from the trie. + * Bits 31..13: Jamo code point. + * Bits 12.. 9: Unused, 0. + */ + BUILDER_DATA_TAG = 7, + /** + * Points to prefix trie. + * Bits 31..13: Index into prefix/contraction data. + * Bits 12.. 8: Unused, 0. + */ + PREFIX_TAG = 8, + /** + * Points to contraction data. + * Bits 31..13: Index into prefix/contraction data. + * Bit 12: Unused, 0. + * Bit 11: CONTRACT_HAS_STARTER flag. (Used by ICU4X only.) + * Bit 10: CONTRACT_TRAILING_CCC flag. + * Bit 9: CONTRACT_NEXT_CCC flag. + * Bit 8: CONTRACT_SINGLE_CP_NO_MATCH flag. + */ + CONTRACTION_TAG = 9, + /** + * Decimal digit. + * Bits 31..13: Index into uint32_t table for non-numeric-collation CE32. + * Bit 12: Unused, 0. + * Bits 11.. 8: Digit value 0..9. + */ + DIGIT_TAG = 10, + /** + * Tag for U+0000, for moving the NUL-termination handling + * from the regular fastpath into specials-handling code. + * Bits 31..8: Unused, 0. + */ + U0000_TAG = 11, + /** + * Tag for a Hangul syllable. + * Bits 31..9: Unused, 0. + * Bit 8: HANGUL_NO_SPECIAL_JAMO flag. + */ + HANGUL_TAG = 12, + /** + * Tag for a lead surrogate code unit. + * Optional optimization for UTF-16 string processing. + * Bits 31..10: Unused, 0. + * 9.. 8: =0: All associated supplementary code points are unassigned-implicit. + * =1: All associated supplementary code points fall back to the base data. + * else: (Normally 2) Look up the data for the supplementary code point. + */ + LEAD_SURROGATE_TAG = 13, + /** + * Tag for CEs with primary weights in code point order. + * Bits 31..13: Index into CE table, for one data "CE". + * Bits 12.. 8: Unused, 0. + * + * This data "CE" has the following bit fields: + * Bits 63..32: Three-byte primary pppppp00. + * 31.. 8: Start/base code point of the in-order range. + * 7: Flag isCompressible primary. + * 6.. 0: Per-code point primary-weight increment. + */ + OFFSET_TAG = 14, + /** + * Implicit CE tag. Compute an unassigned-implicit CE. + * All bits are set (UNASSIGNED_CE32=0xffffffff). + */ + IMPLICIT_TAG = 15 + }; + + static UBool isAssignedCE32(uint32_t ce32) { + return ce32 != FALLBACK_CE32 && ce32 != UNASSIGNED_CE32; + } + + /** + * We limit the number of CEs in an expansion + * so that we can use a small number of length bits in the data structure, + * and so that an implementation can copy CEs at runtime without growing a destination buffer. + */ + static const int32_t MAX_EXPANSION_LENGTH = 31; + static const int32_t MAX_INDEX = 0x7ffff; + + /** + * Set if there is no match for the single (no-suffix) character itself. + * This is only possible if there is a prefix. + * In this case, discontiguous contraction matching cannot add combining marks + * starting from an empty suffix. + * The default CE32 is used anyway if there is no suffix match. + */ + static const uint32_t CONTRACT_SINGLE_CP_NO_MATCH = 0x100; + /** Set if the first character of every contraction suffix has lccc!=0. */ + static const uint32_t CONTRACT_NEXT_CCC = 0x200; + /** Set if any contraction suffix ends with lccc!=0. */ + static const uint32_t CONTRACT_TRAILING_CCC = 0x400; + /** Set if any contraction suffix contains a starter. (Used by ICU4X only.) */ + static const uint32_t CONTRACT_HAS_STARTER = 0x800; + + /** For HANGUL_TAG: None of its Jamo CE32s isSpecialCE32(). */ + static const uint32_t HANGUL_NO_SPECIAL_JAMO = 0x100; + + static const uint32_t LEAD_ALL_UNASSIGNED = 0; + static const uint32_t LEAD_ALL_FALLBACK = 0x100; + static const uint32_t LEAD_MIXED = 0x200; + static const uint32_t LEAD_TYPE_MASK = 0x300; + + static uint32_t makeLongPrimaryCE32(uint32_t p) { return p | LONG_PRIMARY_CE32_LOW_BYTE; } + + /** Turns the long-primary CE32 into a primary weight pppppp00. */ + static inline uint32_t primaryFromLongPrimaryCE32(uint32_t ce32) { + return ce32 & 0xffffff00; + } + static inline int64_t ceFromLongPrimaryCE32(uint32_t ce32) { + return ((int64_t)(ce32 & 0xffffff00) << 32) | COMMON_SEC_AND_TER_CE; + } + + static uint32_t makeLongSecondaryCE32(uint32_t lower32) { + return lower32 | SPECIAL_CE32_LOW_BYTE | LONG_SECONDARY_TAG; + } + static inline int64_t ceFromLongSecondaryCE32(uint32_t ce32) { + return ce32 & 0xffffff00; + } + + /** Makes a special CE32 with tag, index and length. */ + static uint32_t makeCE32FromTagIndexAndLength(int32_t tag, int32_t index, int32_t length) { + return (index << 13) | (length << 8) | SPECIAL_CE32_LOW_BYTE | tag; + } + /** Makes a special CE32 with only tag and index. */ + static uint32_t makeCE32FromTagAndIndex(int32_t tag, int32_t index) { + return (index << 13) | SPECIAL_CE32_LOW_BYTE | tag; + } + + static inline UBool isSpecialCE32(uint32_t ce32) { + return (ce32 & 0xff) >= SPECIAL_CE32_LOW_BYTE; + } + + static inline int32_t tagFromCE32(uint32_t ce32) { + return (int32_t)(ce32 & 0xf); + } + + static inline UBool hasCE32Tag(uint32_t ce32, int32_t tag) { + return isSpecialCE32(ce32) && tagFromCE32(ce32) == tag; + } + + static inline UBool isLongPrimaryCE32(uint32_t ce32) { + return hasCE32Tag(ce32, LONG_PRIMARY_TAG); + } + + static UBool isSimpleOrLongCE32(uint32_t ce32) { + return !isSpecialCE32(ce32) || + tagFromCE32(ce32) == LONG_PRIMARY_TAG || + tagFromCE32(ce32) == LONG_SECONDARY_TAG; + } + + /** + * @return true if the ce32 yields one or more CEs without further data lookups + */ + static UBool isSelfContainedCE32(uint32_t ce32) { + return !isSpecialCE32(ce32) || + tagFromCE32(ce32) == LONG_PRIMARY_TAG || + tagFromCE32(ce32) == LONG_SECONDARY_TAG || + tagFromCE32(ce32) == LATIN_EXPANSION_TAG; + } + + static inline UBool isPrefixCE32(uint32_t ce32) { + return hasCE32Tag(ce32, PREFIX_TAG); + } + + static inline UBool isContractionCE32(uint32_t ce32) { + return hasCE32Tag(ce32, CONTRACTION_TAG); + } + + static inline UBool ce32HasContext(uint32_t ce32) { + return isSpecialCE32(ce32) && + (tagFromCE32(ce32) == PREFIX_TAG || + tagFromCE32(ce32) == CONTRACTION_TAG); + } + + /** + * Get the first of the two Latin-expansion CEs encoded in ce32. + * @see LATIN_EXPANSION_TAG + */ + static inline int64_t latinCE0FromCE32(uint32_t ce32) { + return ((int64_t)(ce32 & 0xff000000) << 32) | COMMON_SECONDARY_CE | ((ce32 & 0xff0000) >> 8); + } + + /** + * Get the second of the two Latin-expansion CEs encoded in ce32. + * @see LATIN_EXPANSION_TAG + */ + static inline int64_t latinCE1FromCE32(uint32_t ce32) { + return ((ce32 & 0xff00) << 16) | COMMON_TERTIARY_CE; + } + + /** + * Returns the data index from a special CE32. + */ + static inline int32_t indexFromCE32(uint32_t ce32) { + return (int32_t)(ce32 >> 13); + } + + /** + * Returns the data length from a ce32. + */ + static inline int32_t lengthFromCE32(uint32_t ce32) { + return (ce32 >> 8) & 31; + } + + /** + * Returns the digit value from a DIGIT_TAG ce32. + */ + static inline char digitFromCE32(uint32_t ce32) { + return (char)((ce32 >> 8) & 0xf); + } + + /** Returns a 64-bit CE from a simple CE32 (not special). */ + static inline int64_t ceFromSimpleCE32(uint32_t ce32) { + // normal form ppppsstt -> pppp0000ss00tt00 + // assert (ce32 & 0xff) < SPECIAL_CE32_LOW_BYTE + return ((int64_t)(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | ((ce32 & 0xff) << 8); + } + + /** Returns a 64-bit CE from a simple/long-primary/long-secondary CE32. */ + static inline int64_t ceFromCE32(uint32_t ce32) { + uint32_t tertiary = ce32 & 0xff; + if(tertiary < SPECIAL_CE32_LOW_BYTE) { + // normal form ppppsstt -> pppp0000ss00tt00 + return ((int64_t)(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (tertiary << 8); + } else { + ce32 -= tertiary; + if((tertiary & 0xf) == LONG_PRIMARY_TAG) { + // long-primary form ppppppC1 -> pppppp00050000500 + return ((int64_t)ce32 << 32) | COMMON_SEC_AND_TER_CE; + } else { + // long-secondary form ssssttC2 -> 00000000sssstt00 + // assert (tertiary & 0xf) == LONG_SECONDARY_TAG + return ce32; + } + } + } + + /** Creates a CE from a primary weight. */ + static inline int64_t makeCE(uint32_t p) { + return ((int64_t)p << 32) | COMMON_SEC_AND_TER_CE; + } + /** + * Creates a CE from a primary weight, + * 16-bit secondary/tertiary weights, and a 2-bit quaternary. + */ + static inline int64_t makeCE(uint32_t p, uint32_t s, uint32_t t, uint32_t q) { + return ((int64_t)p << 32) | (s << 16) | t | (q << 6); + } + + /** + * Increments a 2-byte primary by a code point offset. + */ + static uint32_t incTwoBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, + int32_t offset); + + /** + * Increments a 3-byte primary by a code point offset. + */ + static uint32_t incThreeBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, + int32_t offset); + + /** + * Decrements a 2-byte primary by one range step (1..0x7f). + */ + static uint32_t decTwoBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step); + + /** + * Decrements a 3-byte primary by one range step (1..0x7f). + */ + static uint32_t decThreeBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step); + + /** + * Computes a 3-byte primary for c's OFFSET_TAG data "CE". + */ + static uint32_t getThreeBytePrimaryForOffsetData(UChar32 c, int64_t dataCE); + + /** + * Returns the unassigned-character implicit primary weight for any valid code point c. + */ + static uint32_t unassignedPrimaryFromCodePoint(UChar32 c); + + static inline int64_t unassignedCEFromCodePoint(UChar32 c) { + return makeCE(unassignedPrimaryFromCodePoint(c)); + } + +private: + Collation() = delete; // No instantiation. +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATION_H__ diff --git a/intl/icu/source/i18n/collationbuilder.cpp b/intl/icu/source/i18n/collationbuilder.cpp new file mode 100644 index 0000000000..d8fb89738c --- /dev/null +++ b/intl/icu/source/i18n/collationbuilder.cpp @@ -0,0 +1,1723 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationbuilder.cpp +* +* (replaced the former ucol_bld.cpp) +* +* created on: 2013may06 +* created by: Markus W. Scherer +*/ + +#ifdef DEBUG_COLLATION_BUILDER +#include +#endif + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/caniter.h" +#include "unicode/normalizer2.h" +#include "unicode/tblcoll.h" +#include "unicode/parseerr.h" +#include "unicode/uchar.h" +#include "unicode/ucol.h" +#include "unicode/unistr.h" +#include "unicode/usetiter.h" +#include "unicode/utf16.h" +#include "unicode/uversion.h" +#include "cmemory.h" +#include "collation.h" +#include "collationbuilder.h" +#include "collationdata.h" +#include "collationdatabuilder.h" +#include "collationfastlatin.h" +#include "collationroot.h" +#include "collationrootelements.h" +#include "collationruleparser.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "collationweights.h" +#include "normalizer2impl.h" +#include "uassert.h" +#include "ucol_imp.h" +#include "utf16collationiterator.h" + +U_NAMESPACE_BEGIN + +namespace { + +class BundleImporter : public CollationRuleParser::Importer { +public: + BundleImporter() {} + virtual ~BundleImporter(); + virtual void getRules( + const char *localeID, const char *collationType, + UnicodeString &rules, + const char *&errorReason, UErrorCode &errorCode) override; +}; + +BundleImporter::~BundleImporter() {} + +void +BundleImporter::getRules( + const char *localeID, const char *collationType, + UnicodeString &rules, + const char *& /*errorReason*/, UErrorCode &errorCode) { + CollationLoader::loadRules(localeID, collationType, rules, errorCode); +} + +} // namespace + +// RuleBasedCollator implementation ---------------------------------------- *** + +// These methods are here, rather than in rulebasedcollator.cpp, +// for modularization: +// Most code using Collator does not need to build a Collator from rules. +// By moving these constructors and helper methods to a separate file, +// most code will not have a static dependency on the builder code. + +RuleBasedCollator::RuleBasedCollator() + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { +} + +RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + internalBuildTailoring(rules, UCOL_DEFAULT, UCOL_DEFAULT, nullptr, nullptr, errorCode); +} + +RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, ECollationStrength strength, + UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + internalBuildTailoring(rules, strength, UCOL_DEFAULT, nullptr, nullptr, errorCode); +} + +RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, + UColAttributeValue decompositionMode, + UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + internalBuildTailoring(rules, UCOL_DEFAULT, decompositionMode, nullptr, nullptr, errorCode); +} + +RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, + ECollationStrength strength, + UColAttributeValue decompositionMode, + UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + internalBuildTailoring(rules, strength, decompositionMode, nullptr, nullptr, errorCode); +} + +RuleBasedCollator::RuleBasedCollator(const UnicodeString &rules, + UParseError &parseError, UnicodeString &reason, + UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + internalBuildTailoring(rules, UCOL_DEFAULT, UCOL_DEFAULT, &parseError, &reason, errorCode); +} + +void +RuleBasedCollator::internalBuildTailoring(const UnicodeString &rules, + int32_t strength, + UColAttributeValue decompositionMode, + UParseError *outParseError, UnicodeString *outReason, + UErrorCode &errorCode) { + const CollationTailoring *base = CollationRoot::getRoot(errorCode); + if(U_FAILURE(errorCode)) { return; } + if(outReason != nullptr) { outReason->remove(); } + CollationBuilder builder(base, errorCode); + UVersionInfo noVersion = { 0, 0, 0, 0 }; + BundleImporter importer; + LocalPointer t(builder.parseAndBuild(rules, noVersion, + &importer, + outParseError, errorCode)); + if(U_FAILURE(errorCode)) { + const char *reason = builder.getErrorReason(); + if(reason != nullptr && outReason != nullptr) { + *outReason = UnicodeString(reason, -1, US_INV); + } + return; + } + t->actualLocale.setToBogus(); + adoptTailoring(t.orphan(), errorCode); + // Set attributes after building the collator, + // to keep the default settings consistent with the rule string. + if(strength != UCOL_DEFAULT) { + setAttribute(UCOL_STRENGTH, (UColAttributeValue)strength, errorCode); + } + if(decompositionMode != UCOL_DEFAULT) { + setAttribute(UCOL_NORMALIZATION_MODE, decompositionMode, errorCode); + } +} + +// CollationBuilder implementation ----------------------------------------- *** + +CollationBuilder::CollationBuilder(const CollationTailoring *b, UBool icu4xMode, UErrorCode &errorCode) + : nfd(*Normalizer2::getNFDInstance(errorCode)), + fcd(*Normalizer2Factory::getFCDInstance(errorCode)), + nfcImpl(*Normalizer2Factory::getNFCImpl(errorCode)), + base(b), + baseData(b->data), + rootElements(b->data->rootElements, b->data->rootElementsLength), + variableTop(0), + dataBuilder(new CollationDataBuilder(icu4xMode, errorCode)), fastLatinEnabled(true), + icu4xMode(icu4xMode), + errorReason(nullptr), + cesLength(0), + rootPrimaryIndexes(errorCode), nodes(errorCode) { + nfcImpl.ensureCanonIterData(errorCode); + if(U_FAILURE(errorCode)) { + errorReason = "CollationBuilder fields initialization failed"; + return; + } + if(dataBuilder == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + dataBuilder->initForTailoring(baseData, errorCode); + if(U_FAILURE(errorCode)) { + errorReason = "CollationBuilder initialization failed"; + } +} + +CollationBuilder::CollationBuilder(const CollationTailoring *b, UErrorCode &errorCode) + : CollationBuilder(b, false, errorCode) +{} + +CollationBuilder::~CollationBuilder() { + delete dataBuilder; +} + +CollationTailoring * +CollationBuilder::parseAndBuild(const UnicodeString &ruleString, + const UVersionInfo rulesVersion, + CollationRuleParser::Importer *importer, + UParseError *outParseError, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return nullptr; } + if(baseData->rootElements == nullptr) { + errorCode = U_MISSING_RESOURCE_ERROR; + errorReason = "missing root elements data, tailoring not supported"; + return nullptr; + } + LocalPointer tailoring(new CollationTailoring(base->settings)); + if(tailoring.isNull() || tailoring->isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + CollationRuleParser parser(baseData, errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + // Note: This always bases &[last variable] and &[first regular] + // on the root collator's maxVariable/variableTop. + // If we wanted this to change after [maxVariable x], then we would keep + // the tailoring.settings pointer here and read its variableTop when we need it. + // See http://unicode.org/cldr/trac/ticket/6070 + variableTop = base->settings->variableTop; + parser.setSink(this); + parser.setImporter(importer); + CollationSettings &ownedSettings = *SharedObject::copyOnWrite(tailoring->settings); + parser.parse(ruleString, ownedSettings, outParseError, errorCode); + errorReason = parser.getErrorReason(); + if(U_FAILURE(errorCode)) { return nullptr; } + if(dataBuilder->hasMappings()) { + makeTailoredCEs(errorCode); + if (!icu4xMode) { + closeOverComposites(errorCode); + } + finalizeCEs(errorCode); + if (!icu4xMode) { + // Copy all of ASCII, and Latin-1 letters, into each tailoring. + optimizeSet.add(0, 0x7f); + optimizeSet.add(0xc0, 0xff); + // Hangul is decomposed on the fly during collation, + // and the tailoring data is always built with HANGUL_TAG specials. + optimizeSet.remove(Hangul::HANGUL_BASE, Hangul::HANGUL_END); + dataBuilder->optimize(optimizeSet, errorCode); + } + tailoring->ensureOwnedData(errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + if(fastLatinEnabled) { dataBuilder->enableFastLatin(); } + dataBuilder->build(*tailoring->ownedData, errorCode); + tailoring->builder = dataBuilder; + dataBuilder = nullptr; + } else { + tailoring->data = baseData; + } + if(U_FAILURE(errorCode)) { return nullptr; } + ownedSettings.fastLatinOptions = CollationFastLatin::getOptions( + tailoring->data, ownedSettings, + ownedSettings.fastLatinPrimaries, UPRV_LENGTHOF(ownedSettings.fastLatinPrimaries)); + tailoring->rules = ruleString; + tailoring->rules.getTerminatedBuffer(); // ensure NUL-termination + tailoring->setVersion(base->version, rulesVersion); + return tailoring.orphan(); +} + +void +CollationBuilder::addReset(int32_t strength, const UnicodeString &str, + const char *&parserErrorReason, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + U_ASSERT(!str.isEmpty()); + if(str.charAt(0) == CollationRuleParser::POS_LEAD) { + ces[0] = getSpecialResetPosition(str, parserErrorReason, errorCode); + cesLength = 1; + if(U_FAILURE(errorCode)) { return; } + U_ASSERT((ces[0] & Collation::CASE_AND_QUATERNARY_MASK) == 0); + } else { + // normal reset to a character or string + UnicodeString nfdString = nfd.normalize(str, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "normalizing the reset position"; + return; + } + cesLength = dataBuilder->getCEs(nfdString, ces, 0); + if(cesLength > Collation::MAX_EXPANSION_LENGTH) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + parserErrorReason = "reset position maps to too many collation elements (more than 31)"; + return; + } + } + if(strength == UCOL_IDENTICAL) { return; } // simple reset-at-position + + // &[before strength]position + U_ASSERT(UCOL_PRIMARY <= strength && strength <= UCOL_TERTIARY); + int32_t index = findOrInsertNodeForCEs(strength, parserErrorReason, errorCode); + if(U_FAILURE(errorCode)) { return; } + + int64_t node = nodes.elementAti(index); + // If the index is for a "weaker" node, + // then skip backwards over this and further "weaker" nodes. + while(strengthFromNode(node) > strength) { + index = previousIndexFromNode(node); + node = nodes.elementAti(index); + } + + // Find or insert a node whose index we will put into a temporary CE. + if(strengthFromNode(node) == strength && isTailoredNode(node)) { + // Reset to just before this same-strength tailored node. + index = previousIndexFromNode(node); + } else if(strength == UCOL_PRIMARY) { + // root primary node (has no previous index) + uint32_t p = weight32FromNode(node); + if(p == 0) { + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "reset primary-before ignorable not possible"; + return; + } + if(p <= rootElements.getFirstPrimary()) { + // There is no primary gap between ignorables and the space-first-primary. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "reset primary-before first non-ignorable not supported"; + return; + } + if(p == Collation::FIRST_TRAILING_PRIMARY) { + // We do not support tailoring to an unassigned-implicit CE. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "reset primary-before [first trailing] not supported"; + return; + } + p = rootElements.getPrimaryBefore(p, baseData->isCompressiblePrimary(p)); + index = findOrInsertNodeForPrimary(p, errorCode); + // Go to the last node in this list: + // Tailor after the last node between adjacent root nodes. + for(;;) { + node = nodes.elementAti(index); + int32_t nextIndex = nextIndexFromNode(node); + if(nextIndex == 0) { break; } + index = nextIndex; + } + } else { + // &[before 2] or &[before 3] + index = findCommonNode(index, UCOL_SECONDARY); + if(strength >= UCOL_TERTIARY) { + index = findCommonNode(index, UCOL_TERTIARY); + } + // findCommonNode() stayed on the stronger node or moved to + // an explicit common-weight node of the reset-before strength. + node = nodes.elementAti(index); + if(strengthFromNode(node) == strength) { + // Found a same-strength node with an explicit weight. + uint32_t weight16 = weight16FromNode(node); + if(weight16 == 0) { + errorCode = U_UNSUPPORTED_ERROR; + if(strength == UCOL_SECONDARY) { + parserErrorReason = "reset secondary-before secondary ignorable not possible"; + } else { + parserErrorReason = "reset tertiary-before completely ignorable not possible"; + } + return; + } + U_ASSERT(weight16 > Collation::BEFORE_WEIGHT16); + // Reset to just before this node. + // Insert the preceding same-level explicit weight if it is not there already. + // Which explicit weight immediately precedes this one? + weight16 = getWeight16Before(index, node, strength); + // Does this preceding weight have a node? + uint32_t previousWeight16; + int32_t previousIndex = previousIndexFromNode(node); + for(int32_t i = previousIndex;; i = previousIndexFromNode(node)) { + node = nodes.elementAti(i); + int32_t previousStrength = strengthFromNode(node); + if(previousStrength < strength) { + U_ASSERT(weight16 >= Collation::COMMON_WEIGHT16 || i == previousIndex); + // Either the reset element has an above-common weight and + // the parent node provides the implied common weight, + // or the reset element has a weight<=common in the node + // right after the parent, and we need to insert the preceding weight. + previousWeight16 = Collation::COMMON_WEIGHT16; + break; + } else if(previousStrength == strength && !isTailoredNode(node)) { + previousWeight16 = weight16FromNode(node); + break; + } + // Skip weaker nodes and same-level tailored nodes. + } + if(previousWeight16 == weight16) { + // The preceding weight has a node, + // maybe with following weaker or tailored nodes. + // Reset to the last of them. + index = previousIndex; + } else { + // Insert a node with the preceding weight, reset to that. + node = nodeFromWeight16(weight16) | nodeFromStrength(strength); + index = insertNodeBetween(previousIndex, index, node, errorCode); + } + } else { + // Found a stronger node with implied strength-common weight. + uint32_t weight16 = getWeight16Before(index, node, strength); + index = findOrInsertWeakNode(index, weight16, strength, errorCode); + } + // Strength of the temporary CE = strength of its reset position. + // Code above raises an error if the before-strength is stronger. + strength = ceStrength(ces[cesLength - 1]); + } + if(U_FAILURE(errorCode)) { + parserErrorReason = "inserting reset position for &[before n]"; + return; + } + ces[cesLength - 1] = tempCEFromIndexAndStrength(index, strength); +} + +uint32_t +CollationBuilder::getWeight16Before(int32_t index, int64_t node, int32_t level) { + U_ASSERT(strengthFromNode(node) < level || !isTailoredNode(node)); + // Collect the root CE weights if this node is for a root CE. + // If it is not, then return the low non-primary boundary for a tailored CE. + uint32_t t; + if(strengthFromNode(node) == UCOL_TERTIARY) { + t = weight16FromNode(node); + } else { + t = Collation::COMMON_WEIGHT16; // Stronger node with implied common weight. + } + while(strengthFromNode(node) > UCOL_SECONDARY) { + index = previousIndexFromNode(node); + node = nodes.elementAti(index); + } + if(isTailoredNode(node)) { + return Collation::BEFORE_WEIGHT16; + } + uint32_t s; + if(strengthFromNode(node) == UCOL_SECONDARY) { + s = weight16FromNode(node); + } else { + s = Collation::COMMON_WEIGHT16; // Stronger node with implied common weight. + } + while(strengthFromNode(node) > UCOL_PRIMARY) { + index = previousIndexFromNode(node); + node = nodes.elementAti(index); + } + if(isTailoredNode(node)) { + return Collation::BEFORE_WEIGHT16; + } + // [p, s, t] is a root CE. Return the preceding weight for the requested level. + uint32_t p = weight32FromNode(node); + uint32_t weight16; + if(level == UCOL_SECONDARY) { + weight16 = rootElements.getSecondaryBefore(p, s); + } else { + weight16 = rootElements.getTertiaryBefore(p, s, t); + U_ASSERT((weight16 & ~Collation::ONLY_TERTIARY_MASK) == 0); + } + return weight16; +} + +int64_t +CollationBuilder::getSpecialResetPosition(const UnicodeString &str, + const char *&parserErrorReason, UErrorCode &errorCode) { + U_ASSERT(str.length() == 2); + int64_t ce; + int32_t strength = UCOL_PRIMARY; + UBool isBoundary = false; + UChar32 pos = str.charAt(1) - CollationRuleParser::POS_BASE; + U_ASSERT(0 <= pos && pos <= CollationRuleParser::LAST_TRAILING); + switch(pos) { + case CollationRuleParser::FIRST_TERTIARY_IGNORABLE: + // Quaternary CEs are not supported. + // Non-zero quaternary weights are possible only on tertiary or stronger CEs. + return 0; + case CollationRuleParser::LAST_TERTIARY_IGNORABLE: + return 0; + case CollationRuleParser::FIRST_SECONDARY_IGNORABLE: { + // Look for a tailored tertiary node after [0, 0, 0]. + int32_t index = findOrInsertNodeForRootCE(0, UCOL_TERTIARY, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + int64_t node = nodes.elementAti(index); + if((index = nextIndexFromNode(node)) != 0) { + node = nodes.elementAti(index); + U_ASSERT(strengthFromNode(node) <= UCOL_TERTIARY); + if(isTailoredNode(node) && strengthFromNode(node) == UCOL_TERTIARY) { + return tempCEFromIndexAndStrength(index, UCOL_TERTIARY); + } + } + return rootElements.getFirstTertiaryCE(); + // No need to look for nodeHasAnyBefore() on a tertiary node. + } + case CollationRuleParser::LAST_SECONDARY_IGNORABLE: + ce = rootElements.getLastTertiaryCE(); + strength = UCOL_TERTIARY; + break; + case CollationRuleParser::FIRST_PRIMARY_IGNORABLE: { + // Look for a tailored secondary node after [0, 0, *]. + int32_t index = findOrInsertNodeForRootCE(0, UCOL_SECONDARY, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + int64_t node = nodes.elementAti(index); + while((index = nextIndexFromNode(node)) != 0) { + node = nodes.elementAti(index); + strength = strengthFromNode(node); + if(strength < UCOL_SECONDARY) { break; } + if(strength == UCOL_SECONDARY) { + if(isTailoredNode(node)) { + if(nodeHasBefore3(node)) { + index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); + U_ASSERT(isTailoredNode(nodes.elementAti(index))); + } + return tempCEFromIndexAndStrength(index, UCOL_SECONDARY); + } else { + break; + } + } + } + ce = rootElements.getFirstSecondaryCE(); + strength = UCOL_SECONDARY; + break; + } + case CollationRuleParser::LAST_PRIMARY_IGNORABLE: + ce = rootElements.getLastSecondaryCE(); + strength = UCOL_SECONDARY; + break; + case CollationRuleParser::FIRST_VARIABLE: + ce = rootElements.getFirstPrimaryCE(); + isBoundary = true; // FractionalUCA.txt: FDD1 00A0, SPACE first primary + break; + case CollationRuleParser::LAST_VARIABLE: + ce = rootElements.lastCEWithPrimaryBefore(variableTop + 1); + break; + case CollationRuleParser::FIRST_REGULAR: + ce = rootElements.firstCEWithPrimaryAtLeast(variableTop + 1); + isBoundary = true; // FractionalUCA.txt: FDD1 263A, SYMBOL first primary + break; + case CollationRuleParser::LAST_REGULAR: + // Use the Hani-first-primary rather than the actual last "regular" CE before it, + // for backward compatibility with behavior before the introduction of + // script-first-primary CEs in the root collator. + ce = rootElements.firstCEWithPrimaryAtLeast( + baseData->getFirstPrimaryForGroup(USCRIPT_HAN)); + break; + case CollationRuleParser::FIRST_IMPLICIT: + ce = baseData->getSingleCE(0x4e00, errorCode); + break; + case CollationRuleParser::LAST_IMPLICIT: + // We do not support tailoring to an unassigned-implicit CE. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "reset to [last implicit] not supported"; + return 0; + case CollationRuleParser::FIRST_TRAILING: + ce = Collation::makeCE(Collation::FIRST_TRAILING_PRIMARY); + isBoundary = true; // trailing first primary (there is no mapping for it) + break; + case CollationRuleParser::LAST_TRAILING: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + parserErrorReason = "LDML forbids tailoring to U+FFFF"; + return 0; + default: + UPRV_UNREACHABLE_EXIT; + } + + int32_t index = findOrInsertNodeForRootCE(ce, strength, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + int64_t node = nodes.elementAti(index); + if((pos & 1) == 0) { + // even pos = [first xyz] + if(!nodeHasAnyBefore(node) && isBoundary) { + // A first primary boundary is artificially added to FractionalUCA.txt. + // It is reachable via its special contraction, but is not normally used. + // Find the first character tailored after the boundary CE, + // or the first real root CE after it. + if((index = nextIndexFromNode(node)) != 0) { + // If there is a following node, then it must be tailored + // because there are no root CEs with a boundary primary + // and non-common secondary/tertiary weights. + node = nodes.elementAti(index); + U_ASSERT(isTailoredNode(node)); + ce = tempCEFromIndexAndStrength(index, strength); + } else { + U_ASSERT(strength == UCOL_PRIMARY); + uint32_t p = (uint32_t)(ce >> 32); + int32_t pIndex = rootElements.findPrimary(p); + UBool isCompressible = baseData->isCompressiblePrimary(p); + p = rootElements.getPrimaryAfter(p, pIndex, isCompressible); + ce = Collation::makeCE(p); + index = findOrInsertNodeForRootCE(ce, UCOL_PRIMARY, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + node = nodes.elementAti(index); + } + } + if(nodeHasAnyBefore(node)) { + // Get the first node that was tailored before this one at a weaker strength. + if(nodeHasBefore2(node)) { + index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); + node = nodes.elementAti(index); + } + if(nodeHasBefore3(node)) { + index = nextIndexFromNode(nodes.elementAti(nextIndexFromNode(node))); + } + U_ASSERT(isTailoredNode(nodes.elementAti(index))); + ce = tempCEFromIndexAndStrength(index, strength); + } + } else { + // odd pos = [last xyz] + // Find the last node that was tailored after the [last xyz] + // at a strength no greater than the position's strength. + for(;;) { + int32_t nextIndex = nextIndexFromNode(node); + if(nextIndex == 0) { break; } + int64_t nextNode = nodes.elementAti(nextIndex); + if(strengthFromNode(nextNode) < strength) { break; } + index = nextIndex; + node = nextNode; + } + // Do not make a temporary CE for a root node. + // This last node might be the node for the root CE itself, + // or a node with a common secondary or tertiary weight. + if(isTailoredNode(node)) { + ce = tempCEFromIndexAndStrength(index, strength); + } + } + return ce; +} + +void +CollationBuilder::addRelation(int32_t strength, const UnicodeString &prefix, + const UnicodeString &str, const UnicodeString &extension, + const char *&parserErrorReason, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + UnicodeString nfdPrefix; + if(!prefix.isEmpty()) { + nfd.normalize(prefix, nfdPrefix, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "normalizing the relation prefix"; + return; + } + } + UnicodeString nfdString = nfd.normalize(str, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "normalizing the relation string"; + return; + } + + // The runtime code decomposes Hangul syllables on the fly, + // with recursive processing but without making the Jamo pieces visible for matching. + // It does not work with certain types of contextual mappings. + int32_t nfdLength = nfdString.length(); + if(nfdLength >= 2) { + char16_t c = nfdString.charAt(0); + if(Hangul::isJamoL(c) || Hangul::isJamoV(c)) { + // While handling a Hangul syllable, contractions starting with Jamo L or V + // would not see the following Jamo of that syllable. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "contractions starting with conjoining Jamo L or V not supported"; + return; + } + c = nfdString.charAt(nfdLength - 1); + if(Hangul::isJamoL(c) || + (Hangul::isJamoV(c) && Hangul::isJamoL(nfdString.charAt(nfdLength - 2)))) { + // A contraction ending with Jamo L or L+V would require + // generating Hangul syllables in addTailComposites() (588 for a Jamo L), + // or decomposing a following Hangul syllable on the fly, during contraction matching. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "contractions ending with conjoining Jamo L or L+V not supported"; + return; + } + // A Hangul syllable completely inside a contraction is ok. + } + // Note: If there is a prefix, then the parser checked that + // both the prefix and the string begin with NFC boundaries (not Jamo V or T). + // Therefore: prefix.isEmpty() || !isJamoVOrT(nfdString.charAt(0)) + // (While handling a Hangul syllable, prefixes on Jamo V or T + // would not see the previous Jamo of that syllable.) + + if(strength != UCOL_IDENTICAL) { + // Find the node index after which we insert the new tailored node. + int32_t index = findOrInsertNodeForCEs(strength, parserErrorReason, errorCode); + U_ASSERT(cesLength > 0); + int64_t ce = ces[cesLength - 1]; + if(strength == UCOL_PRIMARY && !isTempCE(ce) && (uint32_t)(ce >> 32) == 0) { + // There is no primary gap between ignorables and the space-first-primary. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "tailoring primary after ignorables not supported"; + return; + } + if(strength == UCOL_QUATERNARY && ce == 0) { + // The CE data structure does not support non-zero quaternary weights + // on tertiary ignorables. + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "tailoring quaternary after tertiary ignorables not supported"; + return; + } + // Insert the new tailored node. + index = insertTailoredNodeAfter(index, strength, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "modifying collation elements"; + return; + } + // Strength of the temporary CE: + // The new relation may yield a stronger CE but not a weaker one. + int32_t tempStrength = ceStrength(ce); + if(strength < tempStrength) { tempStrength = strength; } + ces[cesLength - 1] = tempCEFromIndexAndStrength(index, tempStrength); + } + + setCaseBits(nfdString, parserErrorReason, errorCode); + if(U_FAILURE(errorCode)) { return; } + + int32_t cesLengthBeforeExtension = cesLength; + if(!extension.isEmpty()) { + UnicodeString nfdExtension = nfd.normalize(extension, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "normalizing the relation extension"; + return; + } + cesLength = dataBuilder->getCEs(nfdExtension, ces, cesLength); + if(cesLength > Collation::MAX_EXPANSION_LENGTH) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + parserErrorReason = + "extension string adds too many collation elements (more than 31 total)"; + return; + } + } + uint32_t ce32 = Collation::UNASSIGNED_CE32; + if(!icu4xMode && (prefix != nfdPrefix || str != nfdString) && + !ignorePrefix(prefix, errorCode) && !ignoreString(str, errorCode)) { + // Map from the original input to the CEs. + // We do this in case the canonical closure is incomplete, + // so that it is possible to explicitly provide the missing mappings. + ce32 = addIfDifferent(prefix, str, ces, cesLength, ce32, errorCode); + } + if (!icu4xMode) { + addWithClosure(nfdPrefix, nfdString, ces, cesLength, ce32, errorCode); + } else { + addIfDifferent(nfdPrefix, nfdString, ces, cesLength, ce32, errorCode); + } + if(U_FAILURE(errorCode)) { + parserErrorReason = "writing collation elements"; + return; + } + cesLength = cesLengthBeforeExtension; +} + +int32_t +CollationBuilder::findOrInsertNodeForCEs(int32_t strength, const char *&parserErrorReason, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + U_ASSERT(UCOL_PRIMARY <= strength && strength <= UCOL_QUATERNARY); + + // Find the last CE that is at least as "strong" as the requested difference. + // Note: Stronger is smaller (UCOL_PRIMARY=0). + int64_t ce; + for(;; --cesLength) { + if(cesLength == 0) { + ce = ces[0] = 0; + cesLength = 1; + break; + } else { + ce = ces[cesLength - 1]; + } + if(ceStrength(ce) <= strength) { break; } + } + + if(isTempCE(ce)) { + // No need to findCommonNode() here for lower levels + // because insertTailoredNodeAfter() will do that anyway. + return indexFromTempCE(ce); + } + + // root CE + if((uint8_t)(ce >> 56) == Collation::UNASSIGNED_IMPLICIT_BYTE) { + errorCode = U_UNSUPPORTED_ERROR; + parserErrorReason = "tailoring relative to an unassigned code point not supported"; + return 0; + } + return findOrInsertNodeForRootCE(ce, strength, errorCode); +} + +int32_t +CollationBuilder::findOrInsertNodeForRootCE(int64_t ce, int32_t strength, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + U_ASSERT((uint8_t)(ce >> 56) != Collation::UNASSIGNED_IMPLICIT_BYTE); + + // Find or insert the node for each of the root CE's weights, + // down to the requested level/strength. + // Root CEs must have common=zero quaternary weights (for which we never insert any nodes). + U_ASSERT((ce & 0xc0) == 0); + int32_t index = findOrInsertNodeForPrimary((uint32_t)(ce >> 32), errorCode); + if(strength >= UCOL_SECONDARY) { + uint32_t lower32 = (uint32_t)ce; + index = findOrInsertWeakNode(index, lower32 >> 16, UCOL_SECONDARY, errorCode); + if(strength >= UCOL_TERTIARY) { + index = findOrInsertWeakNode(index, lower32 & Collation::ONLY_TERTIARY_MASK, + UCOL_TERTIARY, errorCode); + } + } + return index; +} + +namespace { + +/** + * Like Java Collections.binarySearch(List, key, Comparator). + * + * @return the index>=0 where the item was found, + * or the index<0 for inserting the string at ~index in sorted order + * (index into rootPrimaryIndexes) + */ +int32_t +binarySearchForRootPrimaryNode(const int32_t *rootPrimaryIndexes, int32_t length, + const int64_t *nodes, uint32_t p) { + if(length == 0) { return ~0; } + int32_t start = 0; + int32_t limit = length; + for (;;) { + int32_t i = (start + limit) / 2; + int64_t node = nodes[rootPrimaryIndexes[i]]; + uint32_t nodePrimary = (uint32_t)(node >> 32); // weight32FromNode(node) + if (p == nodePrimary) { + return i; + } else if (p < nodePrimary) { + if (i == start) { + return ~start; // insert s before i + } + limit = i; + } else { + if (i == start) { + return ~(start + 1); // insert s after i + } + start = i; + } + } +} + +} // namespace + +int32_t +CollationBuilder::findOrInsertNodeForPrimary(uint32_t p, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + + int32_t rootIndex = binarySearchForRootPrimaryNode( + rootPrimaryIndexes.getBuffer(), rootPrimaryIndexes.size(), nodes.getBuffer(), p); + if(rootIndex >= 0) { + return rootPrimaryIndexes.elementAti(rootIndex); + } else { + // Start a new list of nodes with this primary. + int32_t index = nodes.size(); + nodes.addElement(nodeFromWeight32(p), errorCode); + rootPrimaryIndexes.insertElementAt(index, ~rootIndex, errorCode); + return index; + } +} + +int32_t +CollationBuilder::findOrInsertWeakNode(int32_t index, uint32_t weight16, int32_t level, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + U_ASSERT(0 <= index && index < nodes.size()); + U_ASSERT(UCOL_SECONDARY <= level && level <= UCOL_TERTIARY); + + if(weight16 == Collation::COMMON_WEIGHT16) { + return findCommonNode(index, level); + } + + // If this will be the first below-common weight for the parent node, + // then we will also need to insert a common weight after it. + int64_t node = nodes.elementAti(index); + U_ASSERT(strengthFromNode(node) < level); // parent node is stronger + if(weight16 != 0 && weight16 < Collation::COMMON_WEIGHT16) { + int32_t hasThisLevelBefore = level == UCOL_SECONDARY ? HAS_BEFORE2 : HAS_BEFORE3; + if((node & hasThisLevelBefore) == 0) { + // The parent node has an implied level-common weight. + int64_t commonNode = + nodeFromWeight16(Collation::COMMON_WEIGHT16) | nodeFromStrength(level); + if(level == UCOL_SECONDARY) { + // Move the HAS_BEFORE3 flag from the parent node + // to the new secondary common node. + commonNode |= node & HAS_BEFORE3; + node &= ~(int64_t)HAS_BEFORE3; + } + nodes.setElementAt(node | hasThisLevelBefore, index); + // Insert below-common-weight node. + int32_t nextIndex = nextIndexFromNode(node); + node = nodeFromWeight16(weight16) | nodeFromStrength(level); + index = insertNodeBetween(index, nextIndex, node, errorCode); + // Insert common-weight node. + insertNodeBetween(index, nextIndex, commonNode, errorCode); + // Return index of below-common-weight node. + return index; + } + } + + // Find the root CE's weight for this level. + // Postpone insertion if not found: + // Insert the new root node before the next stronger node, + // or before the next root node with the same strength and a larger weight. + int32_t nextIndex; + while((nextIndex = nextIndexFromNode(node)) != 0) { + node = nodes.elementAti(nextIndex); + int32_t nextStrength = strengthFromNode(node); + if(nextStrength <= level) { + // Insert before a stronger node. + if(nextStrength < level) { break; } + // nextStrength == level + if(!isTailoredNode(node)) { + uint32_t nextWeight16 = weight16FromNode(node); + if(nextWeight16 == weight16) { + // Found the node for the root CE up to this level. + return nextIndex; + } + // Insert before a node with a larger same-strength weight. + if(nextWeight16 > weight16) { break; } + } + } + // Skip the next node. + index = nextIndex; + } + node = nodeFromWeight16(weight16) | nodeFromStrength(level); + return insertNodeBetween(index, nextIndex, node, errorCode); +} + +int32_t +CollationBuilder::insertTailoredNodeAfter(int32_t index, int32_t strength, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + U_ASSERT(0 <= index && index < nodes.size()); + if(strength >= UCOL_SECONDARY) { + index = findCommonNode(index, UCOL_SECONDARY); + if(strength >= UCOL_TERTIARY) { + index = findCommonNode(index, UCOL_TERTIARY); + } + } + // Postpone insertion: + // Insert the new node before the next one with a strength at least as strong. + int64_t node = nodes.elementAti(index); + int32_t nextIndex; + while((nextIndex = nextIndexFromNode(node)) != 0) { + node = nodes.elementAti(nextIndex); + if(strengthFromNode(node) <= strength) { break; } + // Skip the next node which has a weaker (larger) strength than the new one. + index = nextIndex; + } + node = IS_TAILORED | nodeFromStrength(strength); + return insertNodeBetween(index, nextIndex, node, errorCode); +} + +int32_t +CollationBuilder::insertNodeBetween(int32_t index, int32_t nextIndex, int64_t node, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + U_ASSERT(previousIndexFromNode(node) == 0); + U_ASSERT(nextIndexFromNode(node) == 0); + U_ASSERT(nextIndexFromNode(nodes.elementAti(index)) == nextIndex); + // Append the new node and link it to the existing nodes. + int32_t newIndex = nodes.size(); + node |= nodeFromPreviousIndex(index) | nodeFromNextIndex(nextIndex); + nodes.addElement(node, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + // nodes[index].nextIndex = newIndex + node = nodes.elementAti(index); + nodes.setElementAt(changeNodeNextIndex(node, newIndex), index); + // nodes[nextIndex].previousIndex = newIndex + if(nextIndex != 0) { + node = nodes.elementAti(nextIndex); + nodes.setElementAt(changeNodePreviousIndex(node, newIndex), nextIndex); + } + return newIndex; +} + +int32_t +CollationBuilder::findCommonNode(int32_t index, int32_t strength) const { + U_ASSERT(UCOL_SECONDARY <= strength && strength <= UCOL_TERTIARY); + int64_t node = nodes.elementAti(index); + if(strengthFromNode(node) >= strength) { + // The current node is no stronger. + return index; + } + if(strength == UCOL_SECONDARY ? !nodeHasBefore2(node) : !nodeHasBefore3(node)) { + // The current node implies the strength-common weight. + return index; + } + index = nextIndexFromNode(node); + node = nodes.elementAti(index); + U_ASSERT(!isTailoredNode(node) && strengthFromNode(node) == strength && + weight16FromNode(node) < Collation::COMMON_WEIGHT16); + // Skip to the explicit common node. + do { + index = nextIndexFromNode(node); + node = nodes.elementAti(index); + U_ASSERT(strengthFromNode(node) >= strength); + } while(isTailoredNode(node) || strengthFromNode(node) > strength || + weight16FromNode(node) < Collation::COMMON_WEIGHT16); + U_ASSERT(weight16FromNode(node) == Collation::COMMON_WEIGHT16); + return index; +} + +void +CollationBuilder::setCaseBits(const UnicodeString &nfdString, + const char *&parserErrorReason, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t numTailoredPrimaries = 0; + for(int32_t i = 0; i < cesLength; ++i) { + if(ceStrength(ces[i]) == UCOL_PRIMARY) { ++numTailoredPrimaries; } + } + // We should not be able to get too many case bits because + // cesLength<=31==MAX_EXPANSION_LENGTH. + // 31 pairs of case bits fit into an int64_t without setting its sign bit. + U_ASSERT(numTailoredPrimaries <= 31); + + int64_t cases = 0; + if(numTailoredPrimaries > 0) { + const char16_t *s = nfdString.getBuffer(); + UTF16CollationIterator baseCEs(baseData, false, s, s, s + nfdString.length()); + int32_t baseCEsLength = baseCEs.fetchCEs(errorCode) - 1; + if(U_FAILURE(errorCode)) { + parserErrorReason = "fetching root CEs for tailored string"; + return; + } + U_ASSERT(baseCEsLength >= 0 && baseCEs.getCE(baseCEsLength) == Collation::NO_CE); + + uint32_t lastCase = 0; + int32_t numBasePrimaries = 0; + for(int32_t i = 0; i < baseCEsLength; ++i) { + int64_t ce = baseCEs.getCE(i); + if((ce >> 32) != 0) { + ++numBasePrimaries; + uint32_t c = ((uint32_t)ce >> 14) & 3; + U_ASSERT(c == 0 || c == 2); // lowercase or uppercase, no mixed case in any base CE + if(numBasePrimaries < numTailoredPrimaries) { + cases |= (int64_t)c << ((numBasePrimaries - 1) * 2); + } else if(numBasePrimaries == numTailoredPrimaries) { + lastCase = c; + } else if(c != lastCase) { + // There are more base primary CEs than tailored primaries. + // Set mixed case if the case bits of the remainder differ. + lastCase = 1; + // Nothing more can change. + break; + } + } + } + if(numBasePrimaries >= numTailoredPrimaries) { + cases |= (int64_t)lastCase << ((numTailoredPrimaries - 1) * 2); + } + } + + for(int32_t i = 0; i < cesLength; ++i) { + int64_t ce = ces[i] & INT64_C(0xffffffffffff3fff); // clear old case bits + int32_t strength = ceStrength(ce); + if(strength == UCOL_PRIMARY) { + ce |= (cases & 3) << 14; + cases >>= 2; + } else if(strength == UCOL_TERTIARY) { + // Tertiary CEs must have uppercase bits. + // See the LDML spec, and comments in class CollationCompare. + ce |= 0x8000; + } + // Tertiary ignorable CEs must have 0 case bits. + // We set 0 case bits for secondary CEs too + // since currently only U+0345 is cased and maps to a secondary CE, + // and it is lowercase. Other secondaries are uncased. + // See [[:Cased:]&[:uca1=:]] where uca1 queries the root primary weight. + ces[i] = ce; + } +} + +void +CollationBuilder::suppressContractions(const UnicodeSet &set, const char *&parserErrorReason, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + dataBuilder->suppressContractions(set, errorCode); + if(U_FAILURE(errorCode)) { + parserErrorReason = "application of [suppressContractions [set]] failed"; + } +} + +void +CollationBuilder::optimize(const UnicodeSet &set, const char *& /* parserErrorReason */, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + optimizeSet.addAll(set); +} + +uint32_t +CollationBuilder::addWithClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode) { + // Map from the NFD input to the CEs. + ce32 = addIfDifferent(nfdPrefix, nfdString, newCEs, newCEsLength, ce32, errorCode); + ce32 = addOnlyClosure(nfdPrefix, nfdString, newCEs, newCEsLength, ce32, errorCode); + addTailComposites(nfdPrefix, nfdString, errorCode); + return ce32; +} + +uint32_t +CollationBuilder::addOnlyClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return ce32; } + + // Map from canonically equivalent input to the CEs. (But not from the all-NFD input.) + if(nfdPrefix.isEmpty()) { + CanonicalIterator stringIter(nfdString, errorCode); + if(U_FAILURE(errorCode)) { return ce32; } + UnicodeString prefix; + for(;;) { + UnicodeString str = stringIter.next(); + if(str.isBogus()) { break; } + if(ignoreString(str, errorCode) || str == nfdString) { continue; } + ce32 = addIfDifferent(prefix, str, newCEs, newCEsLength, ce32, errorCode); + if(U_FAILURE(errorCode)) { return ce32; } + } + } else { + CanonicalIterator prefixIter(nfdPrefix, errorCode); + CanonicalIterator stringIter(nfdString, errorCode); + if(U_FAILURE(errorCode)) { return ce32; } + for(;;) { + UnicodeString prefix = prefixIter.next(); + if(prefix.isBogus()) { break; } + if(ignorePrefix(prefix, errorCode)) { continue; } + UBool samePrefix = prefix == nfdPrefix; + for(;;) { + UnicodeString str = stringIter.next(); + if(str.isBogus()) { break; } + if(ignoreString(str, errorCode) || (samePrefix && str == nfdString)) { continue; } + ce32 = addIfDifferent(prefix, str, newCEs, newCEsLength, ce32, errorCode); + if(U_FAILURE(errorCode)) { return ce32; } + } + stringIter.reset(); + } + } + return ce32; +} + +void +CollationBuilder::addTailComposites(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + + // Look for the last starter in the NFD string. + UChar32 lastStarter; + int32_t indexAfterLastStarter = nfdString.length(); + for(;;) { + if(indexAfterLastStarter == 0) { return; } // no starter at all + lastStarter = nfdString.char32At(indexAfterLastStarter - 1); + if(nfd.getCombiningClass(lastStarter) == 0) { break; } + indexAfterLastStarter -= U16_LENGTH(lastStarter); + } + // No closure to Hangul syllables since we decompose them on the fly. + if(Hangul::isJamoL(lastStarter)) { return; } + + // Are there any composites whose decomposition starts with the lastStarter? + // Note: Normalizer2Impl does not currently return start sets for NFC_QC=Maybe characters. + // We might find some more equivalent mappings here if it did. + UnicodeSet composites; + if(!nfcImpl.getCanonStartSet(lastStarter, composites)) { return; } + + UnicodeString decomp; + UnicodeString newNFDString, newString; + int64_t newCEs[Collation::MAX_EXPANSION_LENGTH]; + UnicodeSetIterator iter(composites); + while(iter.next()) { + U_ASSERT(!iter.isString()); + UChar32 composite = iter.getCodepoint(); + nfd.getDecomposition(composite, decomp); + if(!mergeCompositeIntoString(nfdString, indexAfterLastStarter, composite, decomp, + newNFDString, newString, errorCode)) { + continue; + } + int32_t newCEsLength = dataBuilder->getCEs(nfdPrefix, newNFDString, newCEs, 0); + if(newCEsLength > Collation::MAX_EXPANSION_LENGTH) { + // Ignore mappings that we cannot store. + continue; + } + // Note: It is possible that the newCEs do not make use of the mapping + // for which we are adding the tail composites, in which case we might be adding + // unnecessary mappings. + // For example, when we add tail composites for ae^ (^=combining circumflex), + // UCA discontiguous-contraction matching does not find any matches + // for ae_^ (_=any combining diacritic below) *unless* there is also + // a contraction mapping for ae. + // Thus, if there is no ae contraction, then the ae^ mapping is ignored + // while fetching the newCEs for ae_^. + // TODO: Try to detect this effectively. + // (Alternatively, print a warning when prefix contractions are missing.) + + // We do not need an explicit mapping for the NFD strings. + // It is fine if the NFD input collates like this via a sequence of mappings. + // It also saves a little bit of space, and may reduce the set of characters with contractions. + uint32_t ce32 = addIfDifferent(nfdPrefix, newString, + newCEs, newCEsLength, Collation::UNASSIGNED_CE32, errorCode); + if(ce32 != Collation::UNASSIGNED_CE32) { + // was different, was added + addOnlyClosure(nfdPrefix, newNFDString, newCEs, newCEsLength, ce32, errorCode); + } + } +} + +UBool +CollationBuilder::mergeCompositeIntoString(const UnicodeString &nfdString, + int32_t indexAfterLastStarter, + UChar32 composite, const UnicodeString &decomp, + UnicodeString &newNFDString, UnicodeString &newString, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return false; } + U_ASSERT(nfdString.char32At(indexAfterLastStarter - 1) == decomp.char32At(0)); + int32_t lastStarterLength = decomp.moveIndex32(0, 1); + if(lastStarterLength == decomp.length()) { + // Singleton decompositions should be found by addWithClosure() + // and the CanonicalIterator, so we can ignore them here. + return false; + } + if(nfdString.compare(indexAfterLastStarter, 0x7fffffff, + decomp, lastStarterLength, 0x7fffffff) == 0) { + // same strings, nothing new to be found here + return false; + } + + // Make new FCD strings that combine a composite, or its decomposition, + // into the nfdString's last starter and the combining marks following it. + // Make an NFD version, and a version with the composite. + newNFDString.setTo(nfdString, 0, indexAfterLastStarter); + newString.setTo(nfdString, 0, indexAfterLastStarter - lastStarterLength).append(composite); + + // The following is related to discontiguous contraction matching, + // but builds only FCD strings (or else returns false). + int32_t sourceIndex = indexAfterLastStarter; + int32_t decompIndex = lastStarterLength; + // Small optimization: We keep the source character across loop iterations + // because we do not always consume it, + // and then need not fetch it again nor look up its combining class again. + UChar32 sourceChar = U_SENTINEL; + // The cc variables need to be declared before the loop so that at the end + // they are set to the last combining classes seen. + uint8_t sourceCC = 0; + uint8_t decompCC = 0; + for(;;) { + if(sourceChar < 0) { + if(sourceIndex >= nfdString.length()) { break; } + sourceChar = nfdString.char32At(sourceIndex); + sourceCC = nfd.getCombiningClass(sourceChar); + U_ASSERT(sourceCC != 0); + } + // We consume a decomposition character in each iteration. + if(decompIndex >= decomp.length()) { break; } + UChar32 decompChar = decomp.char32At(decompIndex); + decompCC = nfd.getCombiningClass(decompChar); + // Compare the two characters and their combining classes. + if(decompCC == 0) { + // Unable to merge because the source contains a non-zero combining mark + // but the composite's decomposition contains another starter. + // The strings would not be equivalent. + return false; + } else if(sourceCC < decompCC) { + // Composite + sourceChar would not be FCD. + return false; + } else if(decompCC < sourceCC) { + newNFDString.append(decompChar); + decompIndex += U16_LENGTH(decompChar); + } else if(decompChar != sourceChar) { + // Blocked because same combining class. + return false; + } else { // match: decompChar == sourceChar + newNFDString.append(decompChar); + decompIndex += U16_LENGTH(decompChar); + sourceIndex += U16_LENGTH(decompChar); + sourceChar = U_SENTINEL; + } + } + // We are at the end of at least one of the two inputs. + if(sourceChar >= 0) { // more characters from nfdString but not from decomp + if(sourceCC < decompCC) { + // Appending the next source character to the composite would not be FCD. + return false; + } + newNFDString.append(nfdString, sourceIndex, 0x7fffffff); + newString.append(nfdString, sourceIndex, 0x7fffffff); + } else if(decompIndex < decomp.length()) { // more characters from decomp, not from nfdString + newNFDString.append(decomp, decompIndex, 0x7fffffff); + } + U_ASSERT(nfd.isNormalized(newNFDString, errorCode)); + U_ASSERT(fcd.isNormalized(newString, errorCode)); + U_ASSERT(nfd.normalize(newString, errorCode) == newNFDString); // canonically equivalent + return true; +} + +UBool +CollationBuilder::ignorePrefix(const UnicodeString &s, UErrorCode &errorCode) const { + // Do not map non-FCD prefixes. + return !isFCD(s, errorCode); +} + +UBool +CollationBuilder::ignoreString(const UnicodeString &s, UErrorCode &errorCode) const { + // Do not map non-FCD strings. + // Do not map strings that start with Hangul syllables: We decompose those on the fly. + return !isFCD(s, errorCode) || Hangul::isHangul(s.charAt(0)); +} + +UBool +CollationBuilder::isFCD(const UnicodeString &s, UErrorCode &errorCode) const { + return U_SUCCESS(errorCode) && fcd.isNormalized(s, errorCode); +} + +void +CollationBuilder::closeOverComposites(UErrorCode &errorCode) { + UnicodeSet composites(UNICODE_STRING_SIMPLE("[:NFD_QC=N:]"), errorCode); // Java: static final + if(U_FAILURE(errorCode)) { return; } + // Hangul is decomposed on the fly during collation. + composites.remove(Hangul::HANGUL_BASE, Hangul::HANGUL_END); + UnicodeString prefix; // empty + UnicodeString nfdString; + UnicodeSetIterator iter(composites); + while(iter.next()) { + U_ASSERT(!iter.isString()); + nfd.getDecomposition(iter.getCodepoint(), nfdString); + cesLength = dataBuilder->getCEs(nfdString, ces, 0); + if(cesLength > Collation::MAX_EXPANSION_LENGTH) { + // Too many CEs from the decomposition (unusual), ignore this composite. + // We could add a capacity parameter to getCEs() and reallocate if necessary. + // However, this can only really happen in contrived cases. + continue; + } + const UnicodeString &composite(iter.getString()); + addIfDifferent(prefix, composite, ces, cesLength, Collation::UNASSIGNED_CE32, errorCode); + } +} + +uint32_t +CollationBuilder::addIfDifferent(const UnicodeString &prefix, const UnicodeString &str, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return ce32; } + int64_t oldCEs[Collation::MAX_EXPANSION_LENGTH]; + int32_t oldCEsLength = dataBuilder->getCEs(prefix, str, oldCEs, 0); + if(!sameCEs(newCEs, newCEsLength, oldCEs, oldCEsLength)) { + if(ce32 == Collation::UNASSIGNED_CE32) { + ce32 = dataBuilder->encodeCEs(newCEs, newCEsLength, errorCode); + } + dataBuilder->addCE32(prefix, str, ce32, errorCode); + } + return ce32; +} + +UBool +CollationBuilder::sameCEs(const int64_t ces1[], int32_t ces1Length, + const int64_t ces2[], int32_t ces2Length) { + if(ces1Length != ces2Length) { + return false; + } + U_ASSERT(ces1Length <= Collation::MAX_EXPANSION_LENGTH); + for(int32_t i = 0; i < ces1Length; ++i) { + if(ces1[i] != ces2[i]) { return false; } + } + return true; +} + +#ifdef DEBUG_COLLATION_BUILDER + +uint32_t +alignWeightRight(uint32_t w) { + if(w != 0) { + while((w & 0xff) == 0) { w >>= 8; } + } + return w; +} + +#endif + +void +CollationBuilder::makeTailoredCEs(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + + CollationWeights primaries, secondaries, tertiaries; + int64_t *nodesArray = nodes.getBuffer(); +#ifdef DEBUG_COLLATION_BUILDER + puts("\nCollationBuilder::makeTailoredCEs()"); +#endif + + for(int32_t rpi = 0; rpi < rootPrimaryIndexes.size(); ++rpi) { + int32_t i = rootPrimaryIndexes.elementAti(rpi); + int64_t node = nodesArray[i]; + uint32_t p = weight32FromNode(node); + uint32_t s = p == 0 ? 0 : Collation::COMMON_WEIGHT16; + uint32_t t = s; + uint32_t q = 0; + UBool pIsTailored = false; + UBool sIsTailored = false; + UBool tIsTailored = false; +#ifdef DEBUG_COLLATION_BUILDER + printf("\nprimary %lx\n", (long)alignWeightRight(p)); +#endif + int32_t pIndex = p == 0 ? 0 : rootElements.findPrimary(p); + int32_t nextIndex = nextIndexFromNode(node); + while(nextIndex != 0) { + i = nextIndex; + node = nodesArray[i]; + nextIndex = nextIndexFromNode(node); + int32_t strength = strengthFromNode(node); + if(strength == UCOL_QUATERNARY) { + U_ASSERT(isTailoredNode(node)); +#ifdef DEBUG_COLLATION_BUILDER + printf(" quat+ "); +#endif + if(q == 3) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + errorReason = "quaternary tailoring gap too small"; + return; + } + ++q; + } else { + if(strength == UCOL_TERTIARY) { + if(isTailoredNode(node)) { +#ifdef DEBUG_COLLATION_BUILDER + printf(" ter+ "); +#endif + if(!tIsTailored) { + // First tailored tertiary node for [p, s]. + int32_t tCount = countTailoredNodes(nodesArray, nextIndex, + UCOL_TERTIARY) + 1; + uint32_t tLimit; + if(t == 0) { + // Gap at the beginning of the tertiary CE range. + t = rootElements.getTertiaryBoundary() - 0x100; + tLimit = rootElements.getFirstTertiaryCE() & Collation::ONLY_TERTIARY_MASK; + } else if(!pIsTailored && !sIsTailored) { + // p and s are root weights. + tLimit = rootElements.getTertiaryAfter(pIndex, s, t); + } else if(t == Collation::BEFORE_WEIGHT16) { + tLimit = Collation::COMMON_WEIGHT16; + } else { + // [p, s] is tailored. + U_ASSERT(t == Collation::COMMON_WEIGHT16); + tLimit = rootElements.getTertiaryBoundary(); + } + U_ASSERT(tLimit == 0x4000 || (tLimit & ~Collation::ONLY_TERTIARY_MASK) == 0); + tertiaries.initForTertiary(); + if(!tertiaries.allocWeights(t, tLimit, tCount)) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + errorReason = "tertiary tailoring gap too small"; + return; + } + tIsTailored = true; + } + t = tertiaries.nextWeight(); + U_ASSERT(t != 0xffffffff); + } else { + t = weight16FromNode(node); + tIsTailored = false; +#ifdef DEBUG_COLLATION_BUILDER + printf(" ter %lx\n", (long)alignWeightRight(t)); +#endif + } + } else { + if(strength == UCOL_SECONDARY) { + if(isTailoredNode(node)) { +#ifdef DEBUG_COLLATION_BUILDER + printf(" sec+ "); +#endif + if(!sIsTailored) { + // First tailored secondary node for p. + int32_t sCount = countTailoredNodes(nodesArray, nextIndex, + UCOL_SECONDARY) + 1; + uint32_t sLimit; + if(s == 0) { + // Gap at the beginning of the secondary CE range. + s = rootElements.getSecondaryBoundary() - 0x100; + sLimit = rootElements.getFirstSecondaryCE() >> 16; + } else if(!pIsTailored) { + // p is a root primary. + sLimit = rootElements.getSecondaryAfter(pIndex, s); + } else if(s == Collation::BEFORE_WEIGHT16) { + sLimit = Collation::COMMON_WEIGHT16; + } else { + // p is a tailored primary. + U_ASSERT(s == Collation::COMMON_WEIGHT16); + sLimit = rootElements.getSecondaryBoundary(); + } + if(s == Collation::COMMON_WEIGHT16) { + // Do not tailor into the getSortKey() range of + // compressed common secondaries. + s = rootElements.getLastCommonSecondary(); + } + secondaries.initForSecondary(); + if(!secondaries.allocWeights(s, sLimit, sCount)) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + errorReason = "secondary tailoring gap too small"; +#ifdef DEBUG_COLLATION_BUILDER + printf("!secondaries.allocWeights(%lx, %lx, sCount=%ld)\n", + (long)alignWeightRight(s), (long)alignWeightRight(sLimit), + (long)alignWeightRight(sCount)); +#endif + return; + } + sIsTailored = true; + } + s = secondaries.nextWeight(); + U_ASSERT(s != 0xffffffff); + } else { + s = weight16FromNode(node); + sIsTailored = false; +#ifdef DEBUG_COLLATION_BUILDER + printf(" sec %lx\n", (long)alignWeightRight(s)); +#endif + } + } else /* UCOL_PRIMARY */ { + U_ASSERT(isTailoredNode(node)); +#ifdef DEBUG_COLLATION_BUILDER + printf("pri+ "); +#endif + if(!pIsTailored) { + // First tailored primary node in this list. + int32_t pCount = countTailoredNodes(nodesArray, nextIndex, + UCOL_PRIMARY) + 1; + UBool isCompressible = baseData->isCompressiblePrimary(p); + uint32_t pLimit = + rootElements.getPrimaryAfter(p, pIndex, isCompressible); + primaries.initForPrimary(isCompressible); + if(!primaries.allocWeights(p, pLimit, pCount)) { + errorCode = U_BUFFER_OVERFLOW_ERROR; // TODO: introduce a more specific UErrorCode? + errorReason = "primary tailoring gap too small"; + return; + } + pIsTailored = true; + } + p = primaries.nextWeight(); + U_ASSERT(p != 0xffffffff); + s = Collation::COMMON_WEIGHT16; + sIsTailored = false; + } + t = s == 0 ? 0 : Collation::COMMON_WEIGHT16; + tIsTailored = false; + } + q = 0; + } + if(isTailoredNode(node)) { + nodesArray[i] = Collation::makeCE(p, s, t, q); +#ifdef DEBUG_COLLATION_BUILDER + printf("%016llx\n", (long long)nodesArray[i]); +#endif + } + } + } +} + +int32_t +CollationBuilder::countTailoredNodes(const int64_t *nodesArray, int32_t i, int32_t strength) { + int32_t count = 0; + for(;;) { + if(i == 0) { break; } + int64_t node = nodesArray[i]; + if(strengthFromNode(node) < strength) { break; } + if(strengthFromNode(node) == strength) { + if(isTailoredNode(node)) { + ++count; + } else { + break; + } + } + i = nextIndexFromNode(node); + } + return count; +} + +class CEFinalizer : public CollationDataBuilder::CEModifier { +public: + CEFinalizer(const int64_t *ces) : finalCEs(ces) {} + virtual ~CEFinalizer(); + virtual int64_t modifyCE32(uint32_t ce32) const override { + U_ASSERT(!Collation::isSpecialCE32(ce32)); + if(CollationBuilder::isTempCE32(ce32)) { + // retain case bits + return finalCEs[CollationBuilder::indexFromTempCE32(ce32)] | ((ce32 & 0xc0) << 8); + } else { + return Collation::NO_CE; + } + } + virtual int64_t modifyCE(int64_t ce) const override { + if(CollationBuilder::isTempCE(ce)) { + // retain case bits + return finalCEs[CollationBuilder::indexFromTempCE(ce)] | (ce & 0xc000); + } else { + return Collation::NO_CE; + } + } + +private: + const int64_t *finalCEs; +}; + +CEFinalizer::~CEFinalizer() {} + +void +CollationBuilder::finalizeCEs(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + LocalPointer newBuilder(new CollationDataBuilder(icu4xMode, errorCode), errorCode); + if(U_FAILURE(errorCode)) { + return; + } + newBuilder->initForTailoring(baseData, errorCode); + CEFinalizer finalizer(nodes.getBuffer()); + newBuilder->copyFrom(*dataBuilder, finalizer, errorCode); + if(U_FAILURE(errorCode)) { return; } + delete dataBuilder; + dataBuilder = newBuilder.orphan(); +} + +int32_t +CollationBuilder::ceStrength(int64_t ce) { + return + isTempCE(ce) ? strengthFromTempCE(ce) : + (ce & INT64_C(0xff00000000000000)) != 0 ? UCOL_PRIMARY : + ((uint32_t)ce & 0xff000000) != 0 ? UCOL_SECONDARY : + ce != 0 ? UCOL_TERTIARY : + UCOL_IDENTICAL; +} + +U_NAMESPACE_END + +U_NAMESPACE_USE + +U_CAPI UCollator * U_EXPORT2 +ucol_openRules(const char16_t *rules, int32_t rulesLength, + UColAttributeValue normalizationMode, UCollationStrength strength, + UParseError *parseError, UErrorCode *pErrorCode) { + if(U_FAILURE(*pErrorCode)) { return nullptr; } + if(rules == nullptr && rulesLength != 0) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + RuleBasedCollator *coll = new RuleBasedCollator(); + if(coll == nullptr) { + *pErrorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + UnicodeString r((UBool)(rulesLength < 0), rules, rulesLength); + coll->internalBuildTailoring(r, strength, normalizationMode, parseError, nullptr, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + delete coll; + return nullptr; + } + return coll->toUCollator(); +} + +static const int32_t internalBufferSize = 512; + +// The @internal ucol_getUnsafeSet() was moved here from ucol_sit.cpp +// because it calls UnicodeSet "builder" code that depends on all Unicode properties, +// and the rest of the collation "runtime" code only depends on normalization. +// This function is not related to the collation builder, +// but it did not seem worth moving it into its own .cpp file, +// nor rewriting it to use lower-level UnicodeSet and Normalizer2Impl methods. +U_CAPI int32_t U_EXPORT2 +ucol_getUnsafeSet( const UCollator *coll, + USet *unsafe, + UErrorCode *status) +{ + char16_t buffer[internalBufferSize]; + int32_t len = 0; + + uset_clear(unsafe); + + // cccpattern = "[[:^tccc=0:][:^lccc=0:]]", unfortunately variant + static const char16_t cccpattern[25] = { 0x5b, 0x5b, 0x3a, 0x5e, 0x74, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d, + 0x5b, 0x3a, 0x5e, 0x6c, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d, 0x5d, 0x00 }; + + // add chars that fail the fcd check + uset_applyPattern(unsafe, cccpattern, 24, USET_IGNORE_SPACE, status); + + // add lead/trail surrogates + // (trail surrogates should need to be unsafe only if the caller tests for UTF-16 code *units*, + // not when testing code *points*) + uset_addRange(unsafe, 0xd800, 0xdfff); + + USet *contractions = uset_open(0,0); + + int32_t i = 0, j = 0; + ucol_getContractionsAndExpansions(coll, contractions, nullptr, false, status); + int32_t contsSize = uset_size(contractions); + UChar32 c = 0; + // Contraction set consists only of strings + // to get unsafe code points, we need to + // break the strings apart and add them to the unsafe set + for(i = 0; i < contsSize; i++) { + len = uset_getItem(contractions, i, nullptr, nullptr, buffer, internalBufferSize, status); + if(len > 0) { + j = 0; + while(j < len) { + U16_NEXT(buffer, j, len, c); + if(j < len) { + uset_add(unsafe, c); + } + } + } + } + + uset_close(contractions); + + return uset_size(unsafe); +} + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationbuilder.h b/intl/icu/source/i18n/collationbuilder.h new file mode 100644 index 0000000000..22e24ddb81 --- /dev/null +++ b/intl/icu/source/i18n/collationbuilder.h @@ -0,0 +1,411 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationbuilder.h +* +* created on: 2013may06 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONBUILDER_H__ +#define __COLLATIONBUILDER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "collationrootelements.h" +#include "collationruleparser.h" +#include "uvectr32.h" +#include "uvectr64.h" + +struct UParseError; + +U_NAMESPACE_BEGIN + +struct CollationData; +struct CollationTailoring; + +class CEFinalizer; +class CollationDataBuilder; +class Normalizer2; +class Normalizer2Impl; + +class U_I18N_API CollationBuilder : public CollationRuleParser::Sink { +public: + CollationBuilder(const CollationTailoring *b, UBool icu4xMode, UErrorCode &errorCode); + CollationBuilder(const CollationTailoring *base, UErrorCode &errorCode); + virtual ~CollationBuilder(); + + void disableFastLatin() { fastLatinEnabled = false; } + + CollationTailoring *parseAndBuild(const UnicodeString &ruleString, + const UVersionInfo rulesVersion, + CollationRuleParser::Importer *importer, + UParseError *outParseError, + UErrorCode &errorCode); + + const char *getErrorReason() const { return errorReason; } + +private: + friend class CEFinalizer; + + /** Implements CollationRuleParser::Sink. */ + virtual void addReset(int32_t strength, const UnicodeString &str, + const char *&errorReason, UErrorCode &errorCode) override; + /** + * Returns the secondary or tertiary weight preceding the current node's weight. + * node=nodes[index]. + */ + uint32_t getWeight16Before(int32_t index, int64_t node, int32_t level); + + int64_t getSpecialResetPosition(const UnicodeString &str, + const char *&parserErrorReason, UErrorCode &errorCode); + + /** Implements CollationRuleParser::Sink. */ + virtual void addRelation(int32_t strength, const UnicodeString &prefix, + const UnicodeString &str, const UnicodeString &extension, + const char *&errorReason, UErrorCode &errorCode) override; + + /** + * Picks one of the current CEs and finds or inserts a node in the graph + * for the CE + strength. + */ + int32_t findOrInsertNodeForCEs(int32_t strength, const char *&parserErrorReason, + UErrorCode &errorCode); + int32_t findOrInsertNodeForRootCE(int64_t ce, int32_t strength, UErrorCode &errorCode); + /** Finds or inserts the node for a root CE's primary weight. */ + int32_t findOrInsertNodeForPrimary(uint32_t p, UErrorCode &errorCode); + /** Finds or inserts the node for a secondary or tertiary weight. */ + int32_t findOrInsertWeakNode(int32_t index, uint32_t weight16, int32_t level, + UErrorCode &errorCode); + + /** + * Makes and inserts a new tailored node into the list, after the one at index. + * Skips over nodes of weaker strength to maintain collation order + * ("postpone insertion"). + * @return the new node's index + */ + int32_t insertTailoredNodeAfter(int32_t index, int32_t strength, UErrorCode &errorCode); + + /** + * Inserts a new node into the list, between list-adjacent items. + * The node's previous and next indexes must not be set yet. + * @return the new node's index + */ + int32_t insertNodeBetween(int32_t index, int32_t nextIndex, int64_t node, + UErrorCode &errorCode); + + /** + * Finds the node which implies or contains a common=05 weight of the given strength + * (secondary or tertiary), if the current node is stronger. + * Skips weaker nodes and tailored nodes if the current node is stronger + * and is followed by an explicit-common-weight node. + * Always returns the input index if that node is no stronger than the given strength. + */ + int32_t findCommonNode(int32_t index, int32_t strength) const; + + void setCaseBits(const UnicodeString &nfdString, + const char *&parserErrorReason, UErrorCode &errorCode); + + /** Implements CollationRuleParser::Sink. */ + virtual void suppressContractions(const UnicodeSet &set, const char *&parserErrorReason, + UErrorCode &errorCode) override; + + /** Implements CollationRuleParser::Sink. */ + virtual void optimize(const UnicodeSet &set, const char *&parserErrorReason, + UErrorCode &errorCode) override; + + /** + * Adds the mapping and its canonical closure. + * Takes ce32=dataBuilder->encodeCEs(...) so that the data builder + * need not re-encode the CEs multiple times. + */ + uint32_t addWithClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode); + uint32_t addOnlyClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode); + void addTailComposites(const UnicodeString &nfdPrefix, const UnicodeString &nfdString, + UErrorCode &errorCode); + UBool mergeCompositeIntoString(const UnicodeString &nfdString, int32_t indexAfterLastStarter, + UChar32 composite, const UnicodeString &decomp, + UnicodeString &newNFDString, UnicodeString &newString, + UErrorCode &errorCode) const; + + UBool ignorePrefix(const UnicodeString &s, UErrorCode &errorCode) const; + UBool ignoreString(const UnicodeString &s, UErrorCode &errorCode) const; + UBool isFCD(const UnicodeString &s, UErrorCode &errorCode) const; + + void closeOverComposites(UErrorCode &errorCode); + + uint32_t addIfDifferent(const UnicodeString &prefix, const UnicodeString &str, + const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32, + UErrorCode &errorCode); + static UBool sameCEs(const int64_t ces1[], int32_t ces1Length, + const int64_t ces2[], int32_t ces2Length); + + /** + * Walks the tailoring graph and overwrites tailored nodes with new CEs. + * After this, the graph is destroyed. + * The nodes array can then be used only as a source of tailored CEs. + */ + void makeTailoredCEs(UErrorCode &errorCode); + /** + * Counts the tailored nodes of the given strength up to the next node + * which is either stronger or has an explicit weight of this strength. + */ + static int32_t countTailoredNodes(const int64_t *nodesArray, int32_t i, int32_t strength); + + /** Replaces temporary CEs with the final CEs they point to. */ + void finalizeCEs(UErrorCode &errorCode); + + /** + * Encodes "temporary CE" data into a CE that fits into the CE32 data structure, + * with 2-byte primary, 1-byte secondary and 6-bit tertiary, + * with valid CE byte values. + * + * The index must not exceed 20 bits (0xfffff). + * The strength must fit into 2 bits (UCOL_PRIMARY..UCOL_QUATERNARY). + * + * Temporary CEs are distinguished from real CEs by their use of + * secondary weights 06..45 which are otherwise reserved for compressed sort keys. + * + * The case bits are unused and available. + */ + static inline int64_t tempCEFromIndexAndStrength(int32_t index, int32_t strength) { + return + // CE byte offsets, to ensure valid CE bytes, and case bits 11 + INT64_C(0x4040000006002000) + + // index bits 19..13 -> primary byte 1 = CE bits 63..56 (byte values 40..BF) + ((int64_t)(index & 0xfe000) << 43) + + // index bits 12..6 -> primary byte 2 = CE bits 55..48 (byte values 40..BF) + ((int64_t)(index & 0x1fc0) << 42) + + // index bits 5..0 -> secondary byte 1 = CE bits 31..24 (byte values 06..45) + ((index & 0x3f) << 24) + + // strength bits 1..0 -> tertiary byte 1 = CE bits 13..8 (byte values 20..23) + (strength << 8); + } + static inline int32_t indexFromTempCE(int64_t tempCE) { + tempCE -= INT64_C(0x4040000006002000); + return + ((int32_t)(tempCE >> 43) & 0xfe000) | + ((int32_t)(tempCE >> 42) & 0x1fc0) | + ((int32_t)(tempCE >> 24) & 0x3f); + } + static inline int32_t strengthFromTempCE(int64_t tempCE) { + return ((int32_t)tempCE >> 8) & 3; + } + static inline UBool isTempCE(int64_t ce) { + uint32_t sec = (uint32_t)ce >> 24; + return 6 <= sec && sec <= 0x45; + } + + static inline int32_t indexFromTempCE32(uint32_t tempCE32) { + tempCE32 -= 0x40400620; + return + ((int32_t)(tempCE32 >> 11) & 0xfe000) | + ((int32_t)(tempCE32 >> 10) & 0x1fc0) | + ((int32_t)(tempCE32 >> 8) & 0x3f); + } + static inline UBool isTempCE32(uint32_t ce32) { + return + (ce32 & 0xff) >= 2 && // not a long-primary/long-secondary CE32 + 6 <= ((ce32 >> 8) & 0xff) && ((ce32 >> 8) & 0xff) <= 0x45; + } + + static int32_t ceStrength(int64_t ce); + + /** At most 1M nodes, limited by the 20 bits in node bit fields. */ + static const int32_t MAX_INDEX = 0xfffff; + /** + * Node bit 6 is set on a primary node if there are nodes + * with secondary values below the common secondary weight (05). + */ + static const int32_t HAS_BEFORE2 = 0x40; + /** + * Node bit 5 is set on a primary or secondary node if there are nodes + * with tertiary values below the common tertiary weight (05). + */ + static const int32_t HAS_BEFORE3 = 0x20; + /** + * Node bit 3 distinguishes a tailored node, which has no weight value, + * from a node with an explicit (root or default) weight. + */ + static const int32_t IS_TAILORED = 8; + + static inline int64_t nodeFromWeight32(uint32_t weight32) { + return (int64_t)weight32 << 32; + } + static inline int64_t nodeFromWeight16(uint32_t weight16) { + return (int64_t)weight16 << 48; + } + static inline int64_t nodeFromPreviousIndex(int32_t previous) { + return (int64_t)previous << 28; + } + static inline int64_t nodeFromNextIndex(int32_t next) { + return next << 8; + } + static inline int64_t nodeFromStrength(int32_t strength) { + return strength; + } + + static inline uint32_t weight32FromNode(int64_t node) { + return (uint32_t)(node >> 32); + } + static inline uint32_t weight16FromNode(int64_t node) { + return (uint32_t)(node >> 48) & 0xffff; + } + static inline int32_t previousIndexFromNode(int64_t node) { + return (int32_t)(node >> 28) & MAX_INDEX; + } + static inline int32_t nextIndexFromNode(int64_t node) { + return ((int32_t)node >> 8) & MAX_INDEX; + } + static inline int32_t strengthFromNode(int64_t node) { + return (int32_t)node & 3; + } + + static inline UBool nodeHasBefore2(int64_t node) { + return (node & HAS_BEFORE2) != 0; + } + static inline UBool nodeHasBefore3(int64_t node) { + return (node & HAS_BEFORE3) != 0; + } + static inline UBool nodeHasAnyBefore(int64_t node) { + return (node & (HAS_BEFORE2 | HAS_BEFORE3)) != 0; + } + static inline UBool isTailoredNode(int64_t node) { + return (node & IS_TAILORED) != 0; + } + + static inline int64_t changeNodePreviousIndex(int64_t node, int32_t previous) { + return (node & INT64_C(0xffff00000fffffff)) | nodeFromPreviousIndex(previous); + } + static inline int64_t changeNodeNextIndex(int64_t node, int32_t next) { + return (node & INT64_C(0xfffffffff00000ff)) | nodeFromNextIndex(next); + } + + const Normalizer2 &nfd, &fcd; + const Normalizer2Impl &nfcImpl; + + const CollationTailoring *base; + const CollationData *baseData; + const CollationRootElements rootElements; + uint32_t variableTop; + + CollationDataBuilder *dataBuilder; + UBool fastLatinEnabled; + UBool icu4xMode; + UnicodeSet optimizeSet; + const char *errorReason; + + int64_t ces[Collation::MAX_EXPANSION_LENGTH]; + int32_t cesLength; + + /** + * Indexes of nodes with root primary weights, sorted by primary. + * Compact form of a TreeMap from root primary to node index. + * + * This is a performance optimization for finding reset positions. + * Without this, we would have to search through the entire nodes list. + * It also allows storing root primary weights in list head nodes, + * without previous index, leaving room in root primary nodes for 32-bit primary weights. + */ + UVector32 rootPrimaryIndexes; + /** + * Data structure for assigning tailored weights and CEs. + * Doubly-linked lists of nodes in mostly collation order. + * Each list starts with a root primary node and ends with a nextIndex of 0. + * + * When there are any nodes in the list, then there is always a root primary node at index 0. + * This allows some code not to have to check explicitly for nextIndex==0. + * + * Root primary nodes have 32-bit weights but do not have previous indexes. + * All other nodes have at most 16-bit weights and do have previous indexes. + * + * Nodes with explicit weights store root collator weights, + * or default weak weights (e.g., secondary 05) for stronger nodes. + * "Tailored" nodes, with the IS_TAILORED bit set, + * do not store explicit weights but rather + * create a difference of a certain strength from the preceding node. + * + * A root node is followed by either + * - a root/default node of the same strength, or + * - a root/default node of the next-weaker strength, or + * - a tailored node of the same strength. + * + * A node of a given strength normally implies "common" weights on weaker levels. + * + * A node with HAS_BEFORE2 must be immediately followed by + * a secondary node with an explicit below-common weight, then a secondary tailored node, + * and later an explicit common-secondary node. + * The below-common weight can be a root weight, + * or it can be BEFORE_WEIGHT16 for tailoring before an implied common weight + * or before the lowest root weight. + * (&[before 2] resets to an explicit secondary node so that + * the following addRelation(secondary) tailors right after that. + * If we did not have this node and instead were to reset on the primary node, + * then addRelation(secondary) would skip forward to the the COMMON_WEIGHT16 node.) + * + * If the flag is not set, then there are no explicit secondary nodes + * with the common or lower weights. + * + * Same for HAS_BEFORE3 for tertiary nodes and weights. + * A node must not have both flags set. + * + * Tailored CEs are initially represented in a CollationDataBuilder as temporary CEs + * which point to stable indexes in this list, + * and temporary CEs stored in a CollationDataBuilder only point to tailored nodes. + * + * A temporary CE in the ces[] array may point to a non-tailored reset-before-position node, + * until the next relation is added. + * + * At the end, the tailored weights are allocated as necessary, + * then the tailored nodes are replaced with final CEs, + * and the CollationData is rewritten by replacing temporary CEs with final ones. + * + * We cannot simply insert new nodes in the middle of the array + * because that would invalidate the indexes stored in existing temporary CEs. + * We need to use a linked graph with stable indexes to existing nodes. + * A doubly-linked list seems easiest to maintain. + * + * Each node is stored as an int64_t, with its fields stored as bit fields. + * + * Root primary node: + * - primary weight: 32 bits 63..32 + * - reserved/unused/zero: 4 bits 31..28 + * + * Weaker root nodes & tailored nodes: + * - a weight: 16 bits 63..48 + * + a root or default weight for a non-tailored node + * + unused/zero for a tailored node + * - index to the previous node: 20 bits 47..28 + * + * All types of nodes: + * - index to the next node: 20 bits 27..8 + * + nextIndex=0 in last node per root-primary list + * - reserved/unused/zero bits: bits 7, 4, 2 + * - HAS_BEFORE2: bit 6 + * - HAS_BEFORE3: bit 5 + * - IS_TAILORED: bit 3 + * - the difference strength (primary/secondary/tertiary/quaternary): 2 bits 1..0 + * + * We could allocate structs with pointers, but we would have to store them + * in a pointer list so that they can be indexed from temporary CEs, + * and they would require more memory allocations. + */ + UVector64 nodes; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONBUILDER_H__ diff --git a/intl/icu/source/i18n/collationcompare.cpp b/intl/icu/source/i18n/collationcompare.cpp new file mode 100644 index 0000000000..d9048afc27 --- /dev/null +++ b/intl/icu/source/i18n/collationcompare.cpp @@ -0,0 +1,356 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1996-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationcompare.cpp +* +* created on: 2012feb14 with new and old collation code +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "cmemory.h" +#include "collation.h" +#include "collationcompare.h" +#include "collationiterator.h" +#include "collationsettings.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +UCollationResult +CollationCompare::compareUpToQuaternary(CollationIterator &left, CollationIterator &right, + const CollationSettings &settings, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + + int32_t options = settings.options; + uint32_t variableTop; + if((options & CollationSettings::ALTERNATE_MASK) == 0) { + variableTop = 0; + } else { + // +1 so that we can use "<" and primary ignorables test out early. + variableTop = settings.variableTop + 1; + } + UBool anyVariable = false; + + // Fetch CEs, compare primaries, store secondary & tertiary weights. + for(;;) { + // We fetch CEs until we get a non-ignorable primary or reach the end. + uint32_t leftPrimary; + do { + int64_t ce = left.nextCE(errorCode); + leftPrimary = (uint32_t)(ce >> 32); + if(leftPrimary < variableTop && leftPrimary > Collation::MERGE_SEPARATOR_PRIMARY) { + // Variable CE, shift it to quaternary level. + // Ignore all following primary ignorables, and shift further variable CEs. + anyVariable = true; + do { + // Store only the primary of the variable CE. + left.setCurrentCE(ce & INT64_C(0xffffffff00000000)); + for(;;) { + ce = left.nextCE(errorCode); + leftPrimary = (uint32_t)(ce >> 32); + if(leftPrimary == 0) { + left.setCurrentCE(0); + } else { + break; + } + } + } while(leftPrimary < variableTop && + leftPrimary > Collation::MERGE_SEPARATOR_PRIMARY); + } + } while(leftPrimary == 0); + + uint32_t rightPrimary; + do { + int64_t ce = right.nextCE(errorCode); + rightPrimary = (uint32_t)(ce >> 32); + if(rightPrimary < variableTop && rightPrimary > Collation::MERGE_SEPARATOR_PRIMARY) { + // Variable CE, shift it to quaternary level. + // Ignore all following primary ignorables, and shift further variable CEs. + anyVariable = true; + do { + // Store only the primary of the variable CE. + right.setCurrentCE(ce & INT64_C(0xffffffff00000000)); + for(;;) { + ce = right.nextCE(errorCode); + rightPrimary = (uint32_t)(ce >> 32); + if(rightPrimary == 0) { + right.setCurrentCE(0); + } else { + break; + } + } + } while(rightPrimary < variableTop && + rightPrimary > Collation::MERGE_SEPARATOR_PRIMARY); + } + } while(rightPrimary == 0); + + if(leftPrimary != rightPrimary) { + // Return the primary difference, with script reordering. + if(settings.hasReordering()) { + leftPrimary = settings.reorder(leftPrimary); + rightPrimary = settings.reorder(rightPrimary); + } + return (leftPrimary < rightPrimary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPrimary == Collation::NO_CE_PRIMARY) { break; } + } + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + + // Compare the buffered secondary & tertiary weights. + // We might skip the secondary level but continue with the case level + // which is turned on separately. + if(CollationSettings::getStrength(options) >= UCOL_SECONDARY) { + if((options & CollationSettings::BACKWARD_SECONDARY) == 0) { + int32_t leftIndex = 0; + int32_t rightIndex = 0; + for(;;) { + uint32_t leftSecondary; + do { + leftSecondary = ((uint32_t)left.getCE(leftIndex++)) >> 16; + } while(leftSecondary == 0); + + uint32_t rightSecondary; + do { + rightSecondary = ((uint32_t)right.getCE(rightIndex++)) >> 16; + } while(rightSecondary == 0); + + if(leftSecondary != rightSecondary) { + return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftSecondary == Collation::NO_CE_WEIGHT16) { break; } + } + } else { + // The backwards secondary level compares secondary weights backwards + // within segments separated by the merge separator (U+FFFE, weight 02). + int32_t leftStart = 0; + int32_t rightStart = 0; + for(;;) { + // Find the merge separator or the NO_CE terminator. + uint32_t p; + int32_t leftLimit = leftStart; + while((p = (uint32_t)(left.getCE(leftLimit) >> 32)) > + Collation::MERGE_SEPARATOR_PRIMARY || + p == 0) { + ++leftLimit; + } + int32_t rightLimit = rightStart; + while((p = (uint32_t)(right.getCE(rightLimit) >> 32)) > + Collation::MERGE_SEPARATOR_PRIMARY || + p == 0) { + ++rightLimit; + } + + // Compare the segments. + int32_t leftIndex = leftLimit; + int32_t rightIndex = rightLimit; + for(;;) { + int32_t leftSecondary = 0; + while(leftSecondary == 0 && leftIndex > leftStart) { + leftSecondary = ((uint32_t)left.getCE(--leftIndex)) >> 16; + } + + int32_t rightSecondary = 0; + while(rightSecondary == 0 && rightIndex > rightStart) { + rightSecondary = ((uint32_t)right.getCE(--rightIndex)) >> 16; + } + + if(leftSecondary != rightSecondary) { + return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftSecondary == 0) { break; } + } + + // Did we reach the end of either string? + // Both strings have the same number of merge separators, + // or else there would have been a primary-level difference. + U_ASSERT(left.getCE(leftLimit) == right.getCE(rightLimit)); + if(p == Collation::NO_CE_PRIMARY) { break; } + // Skip both merge separators and continue. + leftStart = leftLimit + 1; + rightStart = rightLimit + 1; + } + } + } + + if((options & CollationSettings::CASE_LEVEL) != 0) { + int32_t strength = CollationSettings::getStrength(options); + int32_t leftIndex = 0; + int32_t rightIndex = 0; + for(;;) { + uint32_t leftCase, leftLower32, rightCase; + if(strength == UCOL_PRIMARY) { + // Primary+caseLevel: Ignore case level weights of primary ignorables. + // Otherwise we would get a-umlaut > a + // which is not desirable for accent-insensitive sorting. + // Check for (lower 32 bits) == 0 as well because variable CEs are stored + // with only primary weights. + int64_t ce; + do { + ce = left.getCE(leftIndex++); + leftCase = (uint32_t)ce; + } while((uint32_t)(ce >> 32) == 0 || leftCase == 0); + leftLower32 = leftCase; + leftCase &= 0xc000; + + do { + ce = right.getCE(rightIndex++); + rightCase = (uint32_t)ce; + } while((uint32_t)(ce >> 32) == 0 || rightCase == 0); + rightCase &= 0xc000; + } else { + // Secondary+caseLevel: By analogy with the above, + // ignore case level weights of secondary ignorables. + // + // Note: A tertiary CE has uppercase case bits (0.0.ut) + // to keep tertiary+caseFirst well-formed. + // + // Tertiary+caseLevel: Also ignore case level weights of secondary ignorables. + // Otherwise a tertiary CE's uppercase would be no greater than + // a primary/secondary CE's uppercase. + // (See UCA well-formedness condition 2.) + // We could construct a special case weight higher than uppercase, + // but it's simpler to always ignore case weights of secondary ignorables, + // turning 0.0.ut into 0.0.0.t. + // (See LDML Collation, Case Parameters.) + do { + leftCase = (uint32_t)left.getCE(leftIndex++); + } while(leftCase <= 0xffff); + leftLower32 = leftCase; + leftCase &= 0xc000; + + do { + rightCase = (uint32_t)right.getCE(rightIndex++); + } while(rightCase <= 0xffff); + rightCase &= 0xc000; + } + + // No need to handle NO_CE and MERGE_SEPARATOR specially: + // There is one case weight for each previous-level weight, + // so level length differences were handled there. + if(leftCase != rightCase) { + if((options & CollationSettings::UPPER_FIRST) == 0) { + return (leftCase < rightCase) ? UCOL_LESS : UCOL_GREATER; + } else { + return (leftCase < rightCase) ? UCOL_GREATER : UCOL_LESS; + } + } + if((leftLower32 >> 16) == Collation::NO_CE_WEIGHT16) { break; } + } + } + if(CollationSettings::getStrength(options) <= UCOL_SECONDARY) { return UCOL_EQUAL; } + + uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options); + + int32_t leftIndex = 0; + int32_t rightIndex = 0; + uint32_t anyQuaternaries = 0; + for(;;) { + uint32_t leftLower32, leftTertiary; + do { + leftLower32 = (uint32_t)left.getCE(leftIndex++); + anyQuaternaries |= leftLower32; + U_ASSERT((leftLower32 & Collation::ONLY_TERTIARY_MASK) != 0 || + (leftLower32 & 0xc0c0) == 0); + leftTertiary = leftLower32 & tertiaryMask; + } while(leftTertiary == 0); + + uint32_t rightLower32, rightTertiary; + do { + rightLower32 = (uint32_t)right.getCE(rightIndex++); + anyQuaternaries |= rightLower32; + U_ASSERT((rightLower32 & Collation::ONLY_TERTIARY_MASK) != 0 || + (rightLower32 & 0xc0c0) == 0); + rightTertiary = rightLower32 & tertiaryMask; + } while(rightTertiary == 0); + + if(leftTertiary != rightTertiary) { + if(CollationSettings::sortsTertiaryUpperCaseFirst(options)) { + // Pass through NO_CE and keep real tertiary weights larger than that. + // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut), + // to keep tertiary CEs well-formed. + // Their case+tertiary weights must be greater than those of + // primary and secondary CEs. + if(leftTertiary > Collation::NO_CE_WEIGHT16) { + if(leftLower32 > 0xffff) { + leftTertiary ^= 0xc000; + } else { + leftTertiary += 0x4000; + } + } + if(rightTertiary > Collation::NO_CE_WEIGHT16) { + if(rightLower32 > 0xffff) { + rightTertiary ^= 0xc000; + } else { + rightTertiary += 0x4000; + } + } + } + return (leftTertiary < rightTertiary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftTertiary == Collation::NO_CE_WEIGHT16) { break; } + } + if(CollationSettings::getStrength(options) <= UCOL_TERTIARY) { return UCOL_EQUAL; } + + if(!anyVariable && (anyQuaternaries & 0xc0) == 0) { + // If there are no "variable" CEs and no non-zero quaternary weights, + // then there are no quaternary differences. + return UCOL_EQUAL; + } + + leftIndex = 0; + rightIndex = 0; + for(;;) { + uint32_t leftQuaternary; + do { + int64_t ce = left.getCE(leftIndex++); + leftQuaternary = (uint32_t)ce & 0xffff; + if(leftQuaternary <= Collation::NO_CE_WEIGHT16) { + // Variable primary or completely ignorable or NO_CE. + leftQuaternary = (uint32_t)(ce >> 32); + } else { + // Regular CE, not tertiary ignorable. + // Preserve the quaternary weight in bits 7..6. + leftQuaternary |= 0xffffff3f; + } + } while(leftQuaternary == 0); + + uint32_t rightQuaternary; + do { + int64_t ce = right.getCE(rightIndex++); + rightQuaternary = (uint32_t)ce & 0xffff; + if(rightQuaternary <= Collation::NO_CE_WEIGHT16) { + // Variable primary or completely ignorable or NO_CE. + rightQuaternary = (uint32_t)(ce >> 32); + } else { + // Regular CE, not tertiary ignorable. + // Preserve the quaternary weight in bits 7..6. + rightQuaternary |= 0xffffff3f; + } + } while(rightQuaternary == 0); + + if(leftQuaternary != rightQuaternary) { + // Return the difference, with script reordering. + if(settings.hasReordering()) { + leftQuaternary = settings.reorder(leftQuaternary); + rightQuaternary = settings.reorder(rightQuaternary); + } + return (leftQuaternary < rightQuaternary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftQuaternary == Collation::NO_CE_PRIMARY) { break; } + } + return UCOL_EQUAL; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationcompare.h b/intl/icu/source/i18n/collationcompare.h new file mode 100644 index 0000000000..6ad2d06704 --- /dev/null +++ b/intl/icu/source/i18n/collationcompare.h @@ -0,0 +1,38 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1996-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationcompare.h +* +* created on: 2012feb14 with new and old collation code +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONCOMPARE_H__ +#define __COLLATIONCOMPARE_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" + +U_NAMESPACE_BEGIN + +class CollationIterator; +struct CollationSettings; + +class U_I18N_API CollationCompare /* not : public UObject because all methods are static */ { +public: + static UCollationResult compareUpToQuaternary(CollationIterator &left, CollationIterator &right, + const CollationSettings &settings, + UErrorCode &errorCode); +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONCOMPARE_H__ diff --git a/intl/icu/source/i18n/collationdata.cpp b/intl/icu/source/i18n/collationdata.cpp new file mode 100644 index 0000000000..1b8b6a76de --- /dev/null +++ b/intl/icu/source/i18n/collationdata.cpp @@ -0,0 +1,390 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdata.cpp +* +* created on: 2012jul28 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/udata.h" +#include "unicode/uscript.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "uassert.h" +#include "utrie2.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +uint32_t +CollationData::getIndirectCE32(uint32_t ce32) const { + U_ASSERT(Collation::isSpecialCE32(ce32)); + int32_t tag = Collation::tagFromCE32(ce32); + if(tag == Collation::DIGIT_TAG) { + // Fetch the non-numeric-collation CE32. + ce32 = ce32s[Collation::indexFromCE32(ce32)]; + } else if(tag == Collation::LEAD_SURROGATE_TAG) { + ce32 = Collation::UNASSIGNED_CE32; + } else if(tag == Collation::U0000_TAG) { + // Fetch the normal ce32 for U+0000. + ce32 = ce32s[0]; + } + return ce32; +} + +uint32_t +CollationData::getFinalCE32(uint32_t ce32) const { + if(Collation::isSpecialCE32(ce32)) { + ce32 = getIndirectCE32(ce32); + } + return ce32; +} + +int64_t +CollationData::getSingleCE(UChar32 c, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + // Keep parallel with CollationDataBuilder::getSingleCE(). + const CollationData *d; + uint32_t ce32 = getCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + d = base; + ce32 = base->getCE32(c); + } else { + d = this; + } + while(Collation::isSpecialCE32(ce32)) { + switch(Collation::tagFromCE32(ce32)) { + case Collation::LATIN_EXPANSION_TAG: + case Collation::BUILDER_DATA_TAG: + case Collation::PREFIX_TAG: + case Collation::CONTRACTION_TAG: + case Collation::HANGUL_TAG: + case Collation::LEAD_SURROGATE_TAG: + errorCode = U_UNSUPPORTED_ERROR; + return 0; + case Collation::FALLBACK_TAG: + case Collation::RESERVED_TAG_3: + errorCode = U_INTERNAL_PROGRAM_ERROR; + return 0; + case Collation::LONG_PRIMARY_TAG: + return Collation::ceFromLongPrimaryCE32(ce32); + case Collation::LONG_SECONDARY_TAG: + return Collation::ceFromLongSecondaryCE32(ce32); + case Collation::EXPANSION32_TAG: + if(Collation::lengthFromCE32(ce32) == 1) { + ce32 = d->ce32s[Collation::indexFromCE32(ce32)]; + break; + } else { + errorCode = U_UNSUPPORTED_ERROR; + return 0; + } + case Collation::EXPANSION_TAG: { + if(Collation::lengthFromCE32(ce32) == 1) { + return d->ces[Collation::indexFromCE32(ce32)]; + } else { + errorCode = U_UNSUPPORTED_ERROR; + return 0; + } + } + case Collation::DIGIT_TAG: + // Fetch the non-numeric-collation CE32 and continue. + ce32 = d->ce32s[Collation::indexFromCE32(ce32)]; + break; + case Collation::U0000_TAG: + U_ASSERT(c == 0); + // Fetch the normal ce32 for U+0000 and continue. + ce32 = d->ce32s[0]; + break; + case Collation::OFFSET_TAG: + return d->getCEFromOffsetCE32(c, ce32); + case Collation::IMPLICIT_TAG: + return Collation::unassignedCEFromCodePoint(c); + } + } + return Collation::ceFromSimpleCE32(ce32); +} + +uint32_t +CollationData::getFirstPrimaryForGroup(int32_t script) const { + int32_t index = getScriptIndex(script); + return index == 0 ? 0 : (uint32_t)scriptStarts[index] << 16; +} + +uint32_t +CollationData::getLastPrimaryForGroup(int32_t script) const { + int32_t index = getScriptIndex(script); + if(index == 0) { + return 0; + } + uint32_t limit = scriptStarts[index + 1]; + return (limit << 16) - 1; +} + +int32_t +CollationData::getGroupForPrimary(uint32_t p) const { + p >>= 16; + if(p < scriptStarts[1] || scriptStarts[scriptStartsLength - 1] <= p) { + return -1; + } + int32_t index = 1; + while(p >= scriptStarts[index + 1]) { ++index; } + for(int32_t i = 0; i < numScripts; ++i) { + if(scriptsIndex[i] == index) { + return i; + } + } + for(int32_t i = 0; i < MAX_NUM_SPECIAL_REORDER_CODES; ++i) { + if(scriptsIndex[numScripts + i] == index) { + return UCOL_REORDER_CODE_FIRST + i; + } + } + return -1; +} + +int32_t +CollationData::getScriptIndex(int32_t script) const { + if(script < 0) { + return 0; + } else if(script < numScripts) { + return scriptsIndex[script]; + } else if(script < UCOL_REORDER_CODE_FIRST) { + return 0; + } else { + script -= UCOL_REORDER_CODE_FIRST; + if(script < MAX_NUM_SPECIAL_REORDER_CODES) { + return scriptsIndex[numScripts + script]; + } else { + return 0; + } + } +} + +int32_t +CollationData::getEquivalentScripts(int32_t script, + int32_t dest[], int32_t capacity, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + int32_t index = getScriptIndex(script); + if(index == 0) { return 0; } + if(script >= UCOL_REORDER_CODE_FIRST) { + // Special groups have no aliases. + if(capacity > 0) { + dest[0] = script; + } else { + errorCode = U_BUFFER_OVERFLOW_ERROR; + } + return 1; + } + + int32_t length = 0; + for(int32_t i = 0; i < numScripts; ++i) { + if(scriptsIndex[i] == index) { + if(length < capacity) { + dest[length] = i; + } + ++length; + } + } + if(length > capacity) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + } + return length; +} + +void +CollationData::makeReorderRanges(const int32_t *reorder, int32_t length, + UVector32 &ranges, UErrorCode &errorCode) const { + makeReorderRanges(reorder, length, false, ranges, errorCode); +} + +void +CollationData::makeReorderRanges(const int32_t *reorder, int32_t length, + UBool latinMustMove, + UVector32 &ranges, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return; } + ranges.removeAllElements(); + if(length == 0 || (length == 1 && reorder[0] == USCRIPT_UNKNOWN)) { + return; + } + + // Maps each script-or-group range to a new lead byte. + uint8_t table[MAX_NUM_SCRIPT_RANGES]; + uprv_memset(table, 0, sizeof(table)); + + { + // Set "don't care" values for reserved ranges. + int32_t index = scriptsIndex[ + numScripts + REORDER_RESERVED_BEFORE_LATIN - UCOL_REORDER_CODE_FIRST]; + if(index != 0) { + table[index] = 0xff; + } + index = scriptsIndex[ + numScripts + REORDER_RESERVED_AFTER_LATIN - UCOL_REORDER_CODE_FIRST]; + if(index != 0) { + table[index] = 0xff; + } + } + + // Never reorder special low and high primary lead bytes. + U_ASSERT(scriptStartsLength >= 2); + U_ASSERT(scriptStarts[0] == 0); + int32_t lowStart = scriptStarts[1]; + U_ASSERT(lowStart == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8)); + int32_t highLimit = scriptStarts[scriptStartsLength - 1]; + U_ASSERT(highLimit == (Collation::TRAIL_WEIGHT_BYTE << 8)); + + // Get the set of special reorder codes in the input list. + // This supports a fixed number of special reorder codes; + // it works for data with codes beyond UCOL_REORDER_CODE_LIMIT. + uint32_t specials = 0; + for(int32_t i = 0; i < length; ++i) { + int32_t reorderCode = reorder[i] - UCOL_REORDER_CODE_FIRST; + if(0 <= reorderCode && reorderCode < MAX_NUM_SPECIAL_REORDER_CODES) { + specials |= (uint32_t)1 << reorderCode; + } + } + + // Start the reordering with the special low reorder codes that do not occur in the input. + for(int32_t i = 0; i < MAX_NUM_SPECIAL_REORDER_CODES; ++i) { + int32_t index = scriptsIndex[numScripts + i]; + if(index != 0 && (specials & ((uint32_t)1 << i)) == 0) { + lowStart = addLowScriptRange(table, index, lowStart); + } + } + + // Skip the reserved range before Latin if Latin is the first script, + // so that we do not move it unnecessarily. + int32_t skippedReserved = 0; + if(specials == 0 && reorder[0] == USCRIPT_LATIN && !latinMustMove) { + int32_t index = scriptsIndex[USCRIPT_LATIN]; + U_ASSERT(index != 0); + int32_t start = scriptStarts[index]; + U_ASSERT(lowStart <= start); + skippedReserved = start - lowStart; + lowStart = start; + } + + // Reorder according to the input scripts, continuing from the bottom of the primary range. + int32_t originalLength = length; // length will be decremented if "others" is in the list. + UBool hasReorderToEnd = false; + for(int32_t i = 0; i < length;) { + int32_t script = reorder[i++]; + if(script == USCRIPT_UNKNOWN) { + // Put the remaining scripts at the top. + hasReorderToEnd = true; + while(i < length) { + script = reorder[--length]; + if(script == USCRIPT_UNKNOWN || // Must occur at most once. + script == UCOL_REORDER_CODE_DEFAULT) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + int32_t index = getScriptIndex(script); + if(index == 0) { continue; } + if(table[index] != 0) { // Duplicate or equivalent script. + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + highLimit = addHighScriptRange(table, index, highLimit); + } + break; + } + if(script == UCOL_REORDER_CODE_DEFAULT) { + // The default code must be the only one in the list, and that is handled by the caller. + // Otherwise it must not be used. + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + int32_t index = getScriptIndex(script); + if(index == 0) { continue; } + if(table[index] != 0) { // Duplicate or equivalent script. + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + lowStart = addLowScriptRange(table, index, lowStart); + } + + // Put all remaining scripts into the middle. + for(int32_t i = 1; i < scriptStartsLength - 1; ++i) { + int32_t leadByte = table[i]; + if(leadByte != 0) { continue; } + int32_t start = scriptStarts[i]; + if(!hasReorderToEnd && start > lowStart) { + // No need to move this script. + lowStart = start; + } + lowStart = addLowScriptRange(table, i, lowStart); + } + if(lowStart > highLimit) { + if((lowStart - (skippedReserved & 0xff00)) <= highLimit) { + // Try not skipping the before-Latin reserved range. + makeReorderRanges(reorder, originalLength, true, ranges, errorCode); + return; + } + // We need more primary lead bytes than available, despite the reserved ranges. + errorCode = U_BUFFER_OVERFLOW_ERROR; + return; + } + + // Turn lead bytes into a list of (limit, offset) pairs. + // Encode each pair in one list element: + // Upper 16 bits = limit, lower 16 = signed lead byte offset. + int32_t offset = 0; + for(int32_t i = 1;; ++i) { + int32_t nextOffset = offset; + while(i < scriptStartsLength - 1) { + int32_t newLeadByte = table[i]; + if(newLeadByte == 0xff) { + // "Don't care" lead byte for reserved range, continue with current offset. + } else { + nextOffset = newLeadByte - (scriptStarts[i] >> 8); + if(nextOffset != offset) { break; } + } + ++i; + } + if(offset != 0 || i < scriptStartsLength - 1) { + ranges.addElement(((int32_t)scriptStarts[i] << 16) | (offset & 0xffff), errorCode); + } + if(i == scriptStartsLength - 1) { break; } + offset = nextOffset; + } +} + +int32_t +CollationData::addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const { + int32_t start = scriptStarts[index]; + if((start & 0xff) < (lowStart & 0xff)) { + lowStart += 0x100; + } + table[index] = (uint8_t)(lowStart >> 8); + int32_t limit = scriptStarts[index + 1]; + lowStart = ((lowStart & 0xff00) + ((limit & 0xff00) - (start & 0xff00))) | (limit & 0xff); + return lowStart; +} + +int32_t +CollationData::addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const { + int32_t limit = scriptStarts[index + 1]; + if((limit & 0xff) > (highLimit & 0xff)) { + highLimit -= 0x100; + } + int32_t start = scriptStarts[index]; + highLimit = ((highLimit & 0xff00) - ((limit & 0xff00) - (start & 0xff00))) | (start & 0xff); + table[index] = (uint8_t)(highLimit >> 8); + return highLimit; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationdata.h b/intl/icu/source/i18n/collationdata.h new file mode 100644 index 0000000000..d4f66828fb --- /dev/null +++ b/intl/icu/source/i18n/collationdata.h @@ -0,0 +1,258 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdata.h +* +* created on: 2010oct27 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONDATA_H__ +#define __COLLATIONDATA_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/uniset.h" +#include "collation.h" +#include "normalizer2impl.h" +#include "utrie2.h" + +struct UDataMemory; + +U_NAMESPACE_BEGIN + +class UVector32; + +/** + * Collation data container. + * Immutable data created by a CollationDataBuilder, or loaded from a file, + * or deserialized from API-provided binary data. + * + * Includes data for the collation base (root/default), aliased if this is not the base. + */ +struct U_I18N_API CollationData : public UMemory { + // Note: The ucadata.icu loader could discover the reserved ranges by setting an array + // parallel with the ranges, and resetting ranges that are indexed. + // The reordering builder code could clone the resulting template array. + static constexpr int32_t REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14; + static constexpr int32_t REORDER_RESERVED_AFTER_LATIN = REORDER_RESERVED_BEFORE_LATIN + 1; + + static constexpr int32_t MAX_NUM_SPECIAL_REORDER_CODES = 8; + /** C++ only, data reader check scriptStartsLength. */ + static constexpr int32_t MAX_NUM_SCRIPT_RANGES = 256; + + CollationData(const Normalizer2Impl &nfc) + : trie(nullptr), + ce32s(nullptr), ces(nullptr), contexts(nullptr), base(nullptr), + jamoCE32s(nullptr), + nfcImpl(nfc), + numericPrimary(0x12000000), + ce32sLength(0), cesLength(0), contextsLength(0), + compressibleBytes(nullptr), + unsafeBackwardSet(nullptr), + fastLatinTable(nullptr), fastLatinTableLength(0), + numScripts(0), scriptsIndex(nullptr), scriptStarts(nullptr), scriptStartsLength(0), + rootElements(nullptr), rootElementsLength(0) {} + + uint32_t getCE32(UChar32 c) const { + return UTRIE2_GET32(trie, c); + } + + uint32_t getCE32FromSupplementary(UChar32 c) const { + return UTRIE2_GET32_FROM_SUPP(trie, c); + } + + UBool isDigit(UChar32 c) const { + return c < 0x660 ? c <= 0x39 && 0x30 <= c : + Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG); + } + + UBool isUnsafeBackward(UChar32 c, UBool numeric) const { + return unsafeBackwardSet->contains(c) || (numeric && isDigit(c)); + } + + UBool isCompressibleLeadByte(uint32_t b) const { + return compressibleBytes[b]; + } + + inline UBool isCompressiblePrimary(uint32_t p) const { + return isCompressibleLeadByte(p >> 24); + } + + /** + * Returns the CE32 from two contexts words. + * Access to the defaultCE32 for contraction and prefix matching. + */ + static uint32_t readCE32(const char16_t *p) { + return ((uint32_t)p[0] << 16) | p[1]; + } + + /** + * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG). + * Requires that ce32 is special. + */ + uint32_t getIndirectCE32(uint32_t ce32) const; + /** + * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG), + * if ce32 is special. + */ + uint32_t getFinalCE32(uint32_t ce32) const; + + /** + * Computes a CE from c's ce32 which has the OFFSET_TAG. + */ + int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const { + int64_t dataCE = ces[Collation::indexFromCE32(ce32)]; + return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE)); + } + + /** + * Returns the single CE that c maps to. + * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE. + */ + int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const; + + /** + * Returns the FCD16 value for code point c. c must be >= 0. + */ + uint16_t getFCD16(UChar32 c) const { + return nfcImpl.getFCD16(c); + } + + /** + * Returns the first primary for the script's reordering group. + * @return the primary with only the first primary lead byte of the group + * (not necessarily an actual root collator primary weight), + * or 0 if the script is unknown + */ + uint32_t getFirstPrimaryForGroup(int32_t script) const; + + /** + * Returns the last primary for the script's reordering group. + * @return the last primary of the group + * (not an actual root collator primary weight), + * or 0 if the script is unknown + */ + uint32_t getLastPrimaryForGroup(int32_t script) const; + + /** + * Finds the reordering group which contains the primary weight. + * @return the first script of the group, or -1 if the weight is beyond the last group + */ + int32_t getGroupForPrimary(uint32_t p) const; + + int32_t getEquivalentScripts(int32_t script, + int32_t dest[], int32_t capacity, UErrorCode &errorCode) const; + + /** + * Writes the permutation of primary-weight ranges + * for the given reordering of scripts and groups. + * The caller checks for illegal arguments and + * takes care of [DEFAULT] and memory allocation. + * + * Each list element will be a (limit, offset) pair as described + * for the CollationSettings::reorderRanges. + * The list will be empty if no ranges are reordered. + */ + void makeReorderRanges(const int32_t *reorder, int32_t length, + UVector32 &ranges, UErrorCode &errorCode) const; + + /** @see jamoCE32s */ + static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27; + + /** Main lookup trie. */ + const UTrie2 *trie; + /** + * Array of CE32 values. + * At index 0 there must be CE32(U+0000) + * to support U+0000's special-tag for NUL-termination handling. + */ + const uint32_t *ce32s; + /** Array of CE values for expansions and OFFSET_TAG. */ + const int64_t *ces; + /** Array of prefix and contraction-suffix matching data. */ + const char16_t *contexts; + /** Base collation data, or nullptr if this data itself is a base. */ + const CollationData *base; + /** + * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T. + * They are normally simple CE32s, rarely expansions. + * For fast handling of HANGUL_TAG. + */ + const uint32_t *jamoCE32s; + const Normalizer2Impl &nfcImpl; + /** The single-byte primary weight (xx000000) for numeric collation. */ + uint32_t numericPrimary; + + int32_t ce32sLength; + int32_t cesLength; + int32_t contextsLength; + + /** 256 flags for which primary-weight lead bytes are compressible. */ + const UBool *compressibleBytes; + /** + * Set of code points that are unsafe for starting string comparison after an identical prefix, + * or in backwards CE iteration. + */ + const UnicodeSet *unsafeBackwardSet; + + /** + * Fast Latin table for common-Latin-text string comparisons. + * Data structure see class CollationFastLatin. + */ + const uint16_t *fastLatinTable; + int32_t fastLatinTableLength; + + /** + * Data for scripts and reordering groups. + * Uses include building a reordering permutation table and + * providing script boundaries to AlphabeticIndex. + */ + int32_t numScripts; + /** + * The length of scriptsIndex is numScripts+16. + * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts. + * 16 special reorder codes (not all used) are mapped starting at numScripts. + * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit. + * There are special codes at the end for reorder-reserved primary ranges. + * + * Multiple scripts may share a range and index, for example Hira & Kana. + */ + const uint16_t *scriptsIndex; + /** + * Start primary weight (top 16 bits only) for a group/script/reserved range + * indexed by scriptsIndex. + * The first range (separators & terminators) and the last range (trailing weights) + * are not reorderable, and no scriptsIndex entry points to them. + */ + const uint16_t *scriptStarts; + int32_t scriptStartsLength; + + /** + * Collation elements in the root collator. + * Used by the CollationRootElements class. The data structure is described there. + * nullptr in a tailoring. + */ + const uint32_t *rootElements; + int32_t rootElementsLength; + +private: + int32_t getScriptIndex(int32_t script) const; + void makeReorderRanges(const int32_t *reorder, int32_t length, + UBool latinMustMove, + UVector32 &ranges, UErrorCode &errorCode) const; + int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const; + int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONDATA_H__ diff --git a/intl/icu/source/i18n/collationdatabuilder.cpp b/intl/icu/source/i18n/collationdatabuilder.cpp new file mode 100644 index 0000000000..7c6f1b881e --- /dev/null +++ b/intl/icu/source/i18n/collationdatabuilder.cpp @@ -0,0 +1,1683 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatabuilder.cpp +* +* (replaced the former ucol_elm.cpp) +* +* created on: 2012apr01 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/localpointer.h" +#include "unicode/uchar.h" +#include "unicode/ucharstrie.h" +#include "unicode/ucharstriebuilder.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/usetiter.h" +#include "unicode/utf16.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationdatabuilder.h" +#include "collationfastlatinbuilder.h" +#include "collationiterator.h" +#include "normalizer2impl.h" +#include "utrie2.h" +#include "uvectr32.h" +#include "uvectr64.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +CollationDataBuilder::CEModifier::~CEModifier() {} + +/** + * Build-time context and CE32 for a code point. + * If a code point has contextual mappings, then the default (no-context) mapping + * and all conditional mappings are stored in a singly-linked list + * of ConditionalCE32, sorted by context strings. + * + * Context strings sort by prefix length, then by prefix, then by contraction suffix. + * Context strings must be unique and in ascending order. + */ +struct ConditionalCE32 : public UMemory { + ConditionalCE32() + : context(), + ce32(0), defaultCE32(Collation::NO_CE32), builtCE32(Collation::NO_CE32), + next(-1) {} + ConditionalCE32(const UnicodeString &ct, uint32_t ce) + : context(ct), + ce32(ce), defaultCE32(Collation::NO_CE32), builtCE32(Collation::NO_CE32), + next(-1) {} + + inline UBool hasContext() const { return context.length() > 1; } + inline int32_t prefixLength() const { return context.charAt(0); } + + /** + * "\0" for the first entry for any code point, with its default CE32. + * + * Otherwise one unit with the length of the prefix string, + * then the prefix string, then the contraction suffix. + */ + UnicodeString context; + /** + * CE32 for the code point and its context. + * Can be special (e.g., for an expansion) but not contextual (prefix or contraction tag). + */ + uint32_t ce32; + /** + * Default CE32 for all contexts with this same prefix. + * Initially NO_CE32. Set only while building runtime data structures, + * and only on one of the nodes of a sub-list with the same prefix. + */ + uint32_t defaultCE32; + /** + * CE32 for the built contexts. + * When fetching CEs from the builder, the contexts are built into their runtime form + * so that the normal collation implementation can process them. + * The result is cached in the list head. It is reset when the contexts are modified. + * All of these builtCE32 are invalidated by clearContexts(), + * via incrementing the contextsEra. + */ + uint32_t builtCE32; + /** + * The "era" of building intermediate contexts when the above builtCE32 was set. + * When the array of cached, temporary contexts overflows, then clearContexts() + * removes them all and invalidates the builtCE32 that used to point to built tries. + */ + int32_t era = -1; + /** + * Index of the next ConditionalCE32. + * Negative for the end of the list. + */ + int32_t next; + // Note: We could create a separate class for all of the contextual mappings for + // a code point, with the builtCE32, the era, and a list of the actual mappings. + // The class that represents one mapping would then not need to + // store those fields in each element. +}; + +U_CDECL_BEGIN + +void U_CALLCONV +uprv_deleteConditionalCE32(void *obj) { + delete static_cast(obj); +} + +U_CDECL_END + +/** + * Build-time collation element and character iterator. + * Uses the runtime CollationIterator for fetching CEs for a string + * but reads from the builder's unfinished data structures. + * In particular, this class reads from the unfinished trie + * and has to avoid CollationIterator::nextCE() and redirect other + * calls to data->getCE32() and data->getCE32FromSupplementary(). + * + * We do this so that we need not implement the collation algorithm + * again for the builder and make it behave exactly like the runtime code. + * That would be more difficult to test and maintain than this indirection. + * + * Some CE32 tags (for example, the DIGIT_TAG) do not occur in the builder data, + * so the data accesses from those code paths need not be modified. + * + * This class iterates directly over whole code points + * so that the CollationIterator does not need the finished trie + * for handling the LEAD_SURROGATE_TAG. + */ +class DataBuilderCollationIterator : public CollationIterator { +public: + DataBuilderCollationIterator(CollationDataBuilder &b); + + virtual ~DataBuilderCollationIterator(); + + int32_t fetchCEs(const UnicodeString &str, int32_t start, int64_t ces[], int32_t cesLength); + + virtual void resetToOffset(int32_t newOffset) override; + virtual int32_t getOffset() const override; + + virtual UChar32 nextCodePoint(UErrorCode &errorCode) override; + virtual UChar32 previousCodePoint(UErrorCode &errorCode) override; + +protected: + virtual void forwardNumCodePoints(int32_t num, UErrorCode &errorCode) override; + virtual void backwardNumCodePoints(int32_t num, UErrorCode &errorCode) override; + + virtual uint32_t getDataCE32(UChar32 c) const override; + virtual uint32_t getCE32FromBuilderData(uint32_t ce32, UErrorCode &errorCode) override; + + CollationDataBuilder &builder; + CollationData builderData; + uint32_t jamoCE32s[CollationData::JAMO_CE32S_LENGTH]; + const UnicodeString *s; + int32_t pos; +}; + +DataBuilderCollationIterator::DataBuilderCollationIterator(CollationDataBuilder &b) + : CollationIterator(&builderData, /*numeric=*/ false), + builder(b), builderData(b.nfcImpl), + s(nullptr), pos(0) { + builderData.base = builder.base; + // Set all of the jamoCE32s[] to indirection CE32s. + for(int32_t j = 0; j < CollationData::JAMO_CE32S_LENGTH; ++j) { // Count across Jamo types. + UChar32 jamo = CollationDataBuilder::jamoCpFromIndex(j); + jamoCE32s[j] = Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, jamo) | + CollationDataBuilder::IS_BUILDER_JAMO_CE32; + } + builderData.jamoCE32s = jamoCE32s; +} + +DataBuilderCollationIterator::~DataBuilderCollationIterator() {} + +int32_t +DataBuilderCollationIterator::fetchCEs(const UnicodeString &str, int32_t start, + int64_t ces[], int32_t cesLength) { + // Set the pointers each time, in case they changed due to reallocation. + builderData.ce32s = reinterpret_cast(builder.ce32s.getBuffer()); + builderData.ces = builder.ce64s.getBuffer(); + builderData.contexts = builder.contexts.getBuffer(); + // Modified copy of CollationIterator::nextCE() and CollationIterator::nextCEFromCE32(). + reset(); + s = &str; + pos = start; + UErrorCode errorCode = U_ZERO_ERROR; + while(U_SUCCESS(errorCode) && pos < s->length()) { + // No need to keep all CEs in the iterator buffer. + clearCEs(); + UChar32 c = s->char32At(pos); + pos += U16_LENGTH(c); + uint32_t ce32 = utrie2_get32(builder.trie, c); + const CollationData *d; + if(ce32 == Collation::FALLBACK_CE32) { + d = builder.base; + ce32 = builder.base->getCE32(c); + } else { + d = &builderData; + } + appendCEsFromCE32(d, c, ce32, /*forward=*/ true, errorCode); + U_ASSERT(U_SUCCESS(errorCode)); + for(int32_t i = 0; i < getCEsLength(); ++i) { + int64_t ce = getCE(i); + if(ce != 0) { + if(cesLength < Collation::MAX_EXPANSION_LENGTH) { + ces[cesLength] = ce; + } + ++cesLength; + } + } + } + return cesLength; +} + +void +DataBuilderCollationIterator::resetToOffset(int32_t newOffset) { + reset(); + pos = newOffset; +} + +int32_t +DataBuilderCollationIterator::getOffset() const { + return pos; +} + +UChar32 +DataBuilderCollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) { + if(pos == s->length()) { + return U_SENTINEL; + } + UChar32 c = s->char32At(pos); + pos += U16_LENGTH(c); + return c; +} + +UChar32 +DataBuilderCollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) { + if(pos == 0) { + return U_SENTINEL; + } + UChar32 c = s->char32At(pos - 1); + pos -= U16_LENGTH(c); + return c; +} + +void +DataBuilderCollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { + pos = s->moveIndex32(pos, num); +} + +void +DataBuilderCollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { + pos = s->moveIndex32(pos, -num); +} + +uint32_t +DataBuilderCollationIterator::getDataCE32(UChar32 c) const { + return utrie2_get32(builder.trie, c); +} + +uint32_t +DataBuilderCollationIterator::getCE32FromBuilderData(uint32_t ce32, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return 0; } + U_ASSERT(Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG)); + if((ce32 & CollationDataBuilder::IS_BUILDER_JAMO_CE32) != 0) { + UChar32 jamo = Collation::indexFromCE32(ce32); + return utrie2_get32(builder.trie, jamo); + } else { + ConditionalCE32 *cond = builder.getConditionalCE32ForCE32(ce32); + if (cond == nullptr) { + errorCode = U_INTERNAL_PROGRAM_ERROR; + // TODO: ICU-21531 figure out why this happens. + return 0; + } + if(cond->builtCE32 == Collation::NO_CE32 || cond->era != builder.contextsEra) { + // Build the context-sensitive mappings into their runtime form and cache the result. + cond->builtCE32 = builder.buildContext(cond, errorCode); + if(errorCode == U_BUFFER_OVERFLOW_ERROR) { + errorCode = U_ZERO_ERROR; + builder.clearContexts(); + cond->builtCE32 = builder.buildContext(cond, errorCode); + } + cond->era = builder.contextsEra; + builderData.contexts = builder.contexts.getBuffer(); + } + return cond->builtCE32; + } +} + +// ------------------------------------------------------------------------- *** + +CollationDataBuilder::CollationDataBuilder(UBool icu4xMode, UErrorCode &errorCode) + : nfcImpl(*Normalizer2Factory::getNFCImpl(errorCode)), + base(nullptr), baseSettings(nullptr), + trie(nullptr), + ce32s(errorCode), ce64s(errorCode), conditionalCE32s(errorCode), + modified(false), + icu4xMode(icu4xMode), + fastLatinEnabled(false), fastLatinBuilder(nullptr), + collIter(nullptr) { + // Reserve the first CE32 for U+0000. + if (!icu4xMode) { + ce32s.addElement(0, errorCode); + } + conditionalCE32s.setDeleter(uprv_deleteConditionalCE32); +} + +CollationDataBuilder::~CollationDataBuilder() { + utrie2_close(trie); + delete fastLatinBuilder; + delete collIter; +} + +void +CollationDataBuilder::initForTailoring(const CollationData *b, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(trie != nullptr) { + errorCode = U_INVALID_STATE_ERROR; + return; + } + if(b == nullptr) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + base = b; + + // For a tailoring, the default is to fall back to the base. + // For ICU4X, use the same value for fallback as for the default + // to avoid having to have different blocks for the two. + trie = utrie2_open(Collation::FALLBACK_CE32, icu4xMode ? Collation::FALLBACK_CE32 : Collation::FFFD_CE32, &errorCode); + + if (!icu4xMode) { + // Set the Latin-1 letters block so that it is allocated first in the data array, + // to try to improve locality of reference when sorting Latin-1 text. + // Do not use utrie2_setRange32() since that will not actually allocate blocks + // that are filled with the default value. + // ASCII (0..7F) is already preallocated anyway. + for(UChar32 c = 0xc0; c <= 0xff; ++c) { + utrie2_set32(trie, c, Collation::FALLBACK_CE32, &errorCode); + } + + // Hangul syllables are not tailorable (except via tailoring Jamos). + // Always set the Hangul tag to help performance. + // Do this here, rather than in buildMappings(), + // so that we see the HANGUL_TAG in various assertions. + uint32_t hangulCE32 = Collation::makeCE32FromTagAndIndex(Collation::HANGUL_TAG, 0); + utrie2_setRange32(trie, Hangul::HANGUL_BASE, Hangul::HANGUL_END, hangulCE32, true, &errorCode); + + // Copy the set contents but don't copy/clone the set as a whole because + // that would copy the isFrozen state too. + unsafeBackwardSet.addAll(*b->unsafeBackwardSet); + } + + if(U_FAILURE(errorCode)) { return; } +} + +UBool +CollationDataBuilder::maybeSetPrimaryRange(UChar32 start, UChar32 end, + uint32_t primary, int32_t step, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + U_ASSERT(start <= end); + // TODO: Do we need to check what values are currently set for start..end? + // An offset range is worth it only if we can achieve an overlap between + // adjacent UTrie2 blocks of 32 code points each. + // An offset CE is also a little more expensive to look up and compute + // than a simple CE. + // If the range spans at least three UTrie2 block boundaries (> 64 code points), + // then we take it. + // If the range spans one or two block boundaries and there are + // at least 4 code points on either side, then we take it. + // (We could additionally require a minimum range length of, say, 16.) + int32_t blockDelta = (end >> 5) - (start >> 5); + if(2 <= step && step <= 0x7f && + (blockDelta >= 3 || + (blockDelta > 0 && (start & 0x1f) <= 0x1c && (end & 0x1f) >= 3))) { + int64_t dataCE = ((int64_t)primary << 32) | (start << 8) | step; + if(isCompressiblePrimary(primary)) { dataCE |= 0x80; } + int32_t index = addCE(dataCE, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + uint32_t offsetCE32 = Collation::makeCE32FromTagAndIndex(Collation::OFFSET_TAG, index); + utrie2_setRange32(trie, start, end, offsetCE32, true, &errorCode); + modified = true; + return true; + } else { + return false; + } +} + +uint32_t +CollationDataBuilder::setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end, + uint32_t primary, int32_t step, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + UBool isCompressible = isCompressiblePrimary(primary); + if(maybeSetPrimaryRange(start, end, primary, step, errorCode)) { + return Collation::incThreeBytePrimaryByOffset(primary, isCompressible, + (end - start + 1) * step); + } else { + // Short range: Set individual CE32s. + for(;;) { + utrie2_set32(trie, start, Collation::makeLongPrimaryCE32(primary), &errorCode); + ++start; + primary = Collation::incThreeBytePrimaryByOffset(primary, isCompressible, step); + if(start > end) { return primary; } + } + modified = true; + } +} + +uint32_t +CollationDataBuilder::getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const { + int32_t i = Collation::indexFromCE32(ce32); + int64_t dataCE = fromBase ? base->ces[i] : ce64s.elementAti(i); + uint32_t p = Collation::getThreeBytePrimaryForOffsetData(c, dataCE); + return Collation::makeLongPrimaryCE32(p); +} + +UBool +CollationDataBuilder::isCompressibleLeadByte(uint32_t b) const { + return base->isCompressibleLeadByte(b); +} + +UBool +CollationDataBuilder::isAssigned(UChar32 c) const { + return Collation::isAssignedCE32(utrie2_get32(trie, c)); +} + +uint32_t +CollationDataBuilder::getLongPrimaryIfSingleCE(UChar32 c) const { + uint32_t ce32 = utrie2_get32(trie, c); + if(Collation::isLongPrimaryCE32(ce32)) { + return Collation::primaryFromLongPrimaryCE32(ce32); + } else { + return 0; + } +} + +int64_t +CollationDataBuilder::getSingleCE(UChar32 c, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + // Keep parallel with CollationData::getSingleCE(). + UBool fromBase = false; + uint32_t ce32 = utrie2_get32(trie, c); + if(ce32 == Collation::FALLBACK_CE32) { + fromBase = true; + ce32 = base->getCE32(c); + } + while(Collation::isSpecialCE32(ce32)) { + switch(Collation::tagFromCE32(ce32)) { + case Collation::LATIN_EXPANSION_TAG: + case Collation::BUILDER_DATA_TAG: + case Collation::PREFIX_TAG: + case Collation::CONTRACTION_TAG: + case Collation::HANGUL_TAG: + case Collation::LEAD_SURROGATE_TAG: + errorCode = U_UNSUPPORTED_ERROR; + return 0; + case Collation::FALLBACK_TAG: + case Collation::RESERVED_TAG_3: + errorCode = U_INTERNAL_PROGRAM_ERROR; + return 0; + case Collation::LONG_PRIMARY_TAG: + return Collation::ceFromLongPrimaryCE32(ce32); + case Collation::LONG_SECONDARY_TAG: + return Collation::ceFromLongSecondaryCE32(ce32); + case Collation::EXPANSION32_TAG: + if(Collation::lengthFromCE32(ce32) == 1) { + int32_t i = Collation::indexFromCE32(ce32); + ce32 = fromBase ? base->ce32s[i] : ce32s.elementAti(i); + break; + } else { + errorCode = U_UNSUPPORTED_ERROR; + return 0; + } + case Collation::EXPANSION_TAG: { + if(Collation::lengthFromCE32(ce32) == 1) { + int32_t i = Collation::indexFromCE32(ce32); + return fromBase ? base->ces[i] : ce64s.elementAti(i); + } else { + errorCode = U_UNSUPPORTED_ERROR; + return 0; + } + } + case Collation::DIGIT_TAG: + // Fetch the non-numeric-collation CE32 and continue. + ce32 = ce32s.elementAti(Collation::indexFromCE32(ce32)); + break; + case Collation::U0000_TAG: + U_ASSERT(c == 0); + // Fetch the normal ce32 for U+0000 and continue. + ce32 = fromBase ? base->ce32s[0] : ce32s.elementAti(0); + break; + case Collation::OFFSET_TAG: + ce32 = getCE32FromOffsetCE32(fromBase, c, ce32); + break; + case Collation::IMPLICIT_TAG: + return Collation::unassignedCEFromCodePoint(c); + } + } + return Collation::ceFromSimpleCE32(ce32); +} + +int32_t +CollationDataBuilder::addCE(int64_t ce, UErrorCode &errorCode) { + int32_t length = ce64s.size(); + for(int32_t i = 0; i < length; ++i) { + if(ce == ce64s.elementAti(i)) { return i; } + } + ce64s.addElement(ce, errorCode); + return length; +} + +int32_t +CollationDataBuilder::addCE32(uint32_t ce32, UErrorCode &errorCode) { + int32_t length = ce32s.size(); + for(int32_t i = 0; i < length; ++i) { + if(ce32 == (uint32_t)ce32s.elementAti(i)) { return i; } + } + ce32s.addElement((int32_t)ce32, errorCode); + return length; +} + +int32_t +CollationDataBuilder::addConditionalCE32(const UnicodeString &context, uint32_t ce32, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return -1; } + U_ASSERT(!context.isEmpty()); + int32_t index = conditionalCE32s.size(); + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return -1; + } + LocalPointer cond(new ConditionalCE32(context, ce32), errorCode); + conditionalCE32s.adoptElement(cond.orphan(), errorCode); + if(U_FAILURE(errorCode)) { + return -1; + } + return index; +} + +void +CollationDataBuilder::add(const UnicodeString &prefix, const UnicodeString &s, + const int64_t ces[], int32_t cesLength, + UErrorCode &errorCode) { + uint32_t ce32 = encodeCEs(ces, cesLength, errorCode); + addCE32(prefix, s, ce32, errorCode); +} + +void +CollationDataBuilder::addCE32(const UnicodeString &prefix, const UnicodeString &s, + uint32_t ce32, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(s.isEmpty()) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if(trie == nullptr || utrie2_isFrozen(trie)) { + errorCode = U_INVALID_STATE_ERROR; + return; + } + UChar32 c = s.char32At(0); + int32_t cLength = U16_LENGTH(c); + uint32_t oldCE32 = utrie2_get32(trie, c); + UBool hasContext = !prefix.isEmpty() || s.length() > cLength; + + if (icu4xMode) { + if (base && c >= 0x1100 && c < 0x1200) { + // Omit jamo tailorings. + // TODO(https://github.com/unicode-org/icu4x/issues/1941). + } + const Normalizer2* nfdNormalizer = Normalizer2::getNFDInstance(errorCode); + UnicodeString sInNfd; + nfdNormalizer->normalize(s, sInNfd, errorCode); + if (s != sInNfd) { + // s is not in NFD, so it cannot match in ICU4X, since ICU4X only + // does NFD lookups. + // Now check that we're only rejecting known cases. + if (s.length() == 2) { + char16_t second = s.charAt(1); + if (second == 0x0F73 || second == 0x0F75 || second == 0x0F81) { + // Second is a special decomposing Tibetan vowel sign. + // These also get added in the decomposed form, so ignoring + // this instance is OK. + return; + } + if (c == 0xFDD1 && second == 0xAC00) { + // This strange contraction exists in the root and + // doesn't have a decomposed counterpart there. + // This won't match in ICU4X anyway and is very strange: + // Unassigned Arabic presentation form contracting with + // the very first Hangul syllable. Let's ignore this + // explicitly. + return; + } + } + // Unknown case worth investigating if ever found. + errorCode = U_UNSUPPORTED_ERROR; + return; + } + + if (!prefix.isEmpty()) { + UnicodeString prefixInNfd; + nfdNormalizer->normalize(prefix, prefixInNfd, errorCode); + if (prefix != prefixInNfd) { + errorCode = U_UNSUPPORTED_ERROR; + return; + } + + int32_t count = prefix.countChar32(); + if (count > 2) { + // Prefix too long for ICU4X. + errorCode = U_UNSUPPORTED_ERROR; + return; + } + UChar32 utf32[4]; + int32_t len = prefix.toUTF32(utf32, 4, errorCode); + if (len != count) { + errorCode = U_INVALID_STATE_ERROR; + return; + } + UChar32 c = utf32[0]; + if (u_getCombiningClass(c)) { + // Prefix must start with as starter for ICU4X. + errorCode = U_UNSUPPORTED_ERROR; + return; + } + // XXX: Korean searchjl has jamo in prefix, so commenting out this + // check for now. ICU4X currently ignores non-root jamo tables anyway. + // searchjl was added in + // https://unicode-org.atlassian.net/browse/CLDR-3560 + // Contractions were changed to prefixes in + // https://unicode-org.atlassian.net/browse/CLDR-6546 + // + // if ((c >= 0x1100 && c < 0x1200) || (c >= 0xAC00 && c < 0xD7A4)) { + // errorCode = U_UNSUPPORTED_ERROR; + // return; + // } + if ((len > 1) && !(utf32[1] == 0x3099 || utf32[1] == 0x309A)) { + // Second character in prefix, if present, must be a kana voicing mark for ICU4X. + errorCode = U_UNSUPPORTED_ERROR; + return; + } + } + + if (s.length() > cLength) { + // Check that there's no modern Hangul in contractions. + for (int32_t i = 0; i < s.length(); ++i) { + char16_t c = s.charAt(i); + if ((c >= 0x1100 && c < 0x1100 + 19) || (c >= 0x1161 && c < 0x1161 + 21) || (c >= 0x11A7 && c < 0x11A7 + 28) || (c >= 0xAC00 && c < 0xD7A4)) { + errorCode = U_UNSUPPORTED_ERROR; + return; + } + } + } + } + + if(oldCE32 == Collation::FALLBACK_CE32) { + // First tailoring for c. + // If c has contextual base mappings or if we add a contextual mapping, + // then copy the base mappings. + // Otherwise we just override the base mapping. + uint32_t baseCE32 = base->getFinalCE32(base->getCE32(c)); + if(hasContext || Collation::ce32HasContext(baseCE32)) { + oldCE32 = copyFromBaseCE32(c, baseCE32, true, errorCode); + utrie2_set32(trie, c, oldCE32, &errorCode); + if(U_FAILURE(errorCode)) { return; } + } + } + if(!hasContext) { + // No prefix, no contraction. + if(!isBuilderContextCE32(oldCE32)) { + utrie2_set32(trie, c, ce32, &errorCode); + } else { + ConditionalCE32 *cond = getConditionalCE32ForCE32(oldCE32); + cond->builtCE32 = Collation::NO_CE32; + cond->ce32 = ce32; + } + } else { + ConditionalCE32 *cond; + if(!isBuilderContextCE32(oldCE32)) { + // Replace the simple oldCE32 with a builder context CE32 + // pointing to a new ConditionalCE32 list head. + int32_t index = addConditionalCE32(UnicodeString((char16_t)0), oldCE32, errorCode); + if(U_FAILURE(errorCode)) { return; } + uint32_t contextCE32 = makeBuilderContextCE32(index); + utrie2_set32(trie, c, contextCE32, &errorCode); + contextChars.add(c); + cond = getConditionalCE32(index); + } else { + cond = getConditionalCE32ForCE32(oldCE32); + cond->builtCE32 = Collation::NO_CE32; + } + UnicodeString suffix(s, cLength); + UnicodeString context((char16_t)prefix.length()); + context.append(prefix).append(suffix); + unsafeBackwardSet.addAll(suffix); + for(;;) { + // invariant: context > cond->context + int32_t next = cond->next; + if(next < 0) { + // Append a new ConditionalCE32 after cond. + int32_t index = addConditionalCE32(context, ce32, errorCode); + if(U_FAILURE(errorCode)) { return; } + cond->next = index; + break; + } + ConditionalCE32 *nextCond = getConditionalCE32(next); + int8_t cmp = context.compare(nextCond->context); + if(cmp < 0) { + // Insert a new ConditionalCE32 between cond and nextCond. + int32_t index = addConditionalCE32(context, ce32, errorCode); + if(U_FAILURE(errorCode)) { return; } + cond->next = index; + getConditionalCE32(index)->next = next; + break; + } else if(cmp == 0) { + // Same context as before, overwrite its ce32. + nextCond->ce32 = ce32; + break; + } + cond = nextCond; + } + } + modified = true; +} + +uint32_t +CollationDataBuilder::encodeOneCEAsCE32(int64_t ce) { + uint32_t p = (uint32_t)(ce >> 32); + uint32_t lower32 = (uint32_t)ce; + uint32_t t = (uint32_t)(ce & 0xffff); + U_ASSERT((t & 0xc000) != 0xc000); // Impossible case bits 11 mark special CE32s. + if((ce & INT64_C(0xffff00ff00ff)) == 0) { + // normal form ppppsstt + return p | (lower32 >> 16) | (t >> 8); + } else if((ce & INT64_C(0xffffffffff)) == Collation::COMMON_SEC_AND_TER_CE) { + // long-primary form ppppppC1 + return Collation::makeLongPrimaryCE32(p); + } else if(p == 0 && (t & 0xff) == 0) { + // long-secondary form ssssttC2 + return Collation::makeLongSecondaryCE32(lower32); + } + return Collation::NO_CE32; +} + +uint32_t +CollationDataBuilder::encodeOneCE(int64_t ce, UErrorCode &errorCode) { + // Try to encode one CE as one CE32. + uint32_t ce32 = encodeOneCEAsCE32(ce); + if(ce32 != Collation::NO_CE32) { return ce32; } + int32_t index = addCE(ce, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + return Collation::makeCE32FromTagIndexAndLength(Collation::EXPANSION_TAG, index, 1); +} + +uint32_t +CollationDataBuilder::encodeCEs(const int64_t ces[], int32_t cesLength, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + if(cesLength < 0 || cesLength > Collation::MAX_EXPANSION_LENGTH) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(trie == nullptr || utrie2_isFrozen(trie)) { + errorCode = U_INVALID_STATE_ERROR; + return 0; + } + if(cesLength == 0) { + // Convenience: We cannot map to nothing, but we can map to a completely ignorable CE. + // Do this here so that callers need not do it. + return encodeOneCEAsCE32(0); + } else if(cesLength == 1) { + return encodeOneCE(ces[0], errorCode); + } else if(cesLength == 2 && !icu4xMode) { + // Try to encode two CEs as one CE32. + // Turn this off for ICU4X, because without the canonical closure + // these are so rare that it doesn't make sense to spend a branch + // on checking this tag when using the data. + int64_t ce0 = ces[0]; + int64_t ce1 = ces[1]; + uint32_t p0 = (uint32_t)(ce0 >> 32); + if((ce0 & INT64_C(0xffffffffff00ff)) == Collation::COMMON_SECONDARY_CE && + (ce1 & INT64_C(0xffffffff00ffffff)) == Collation::COMMON_TERTIARY_CE && + p0 != 0) { + // Latin mini expansion + return + p0 | + (((uint32_t)ce0 & 0xff00u) << 8) | + (uint32_t)(ce1 >> 16) | + Collation::SPECIAL_CE32_LOW_BYTE | + Collation::LATIN_EXPANSION_TAG; + } + } + // Try to encode two or more CEs as CE32s. + int32_t newCE32s[Collation::MAX_EXPANSION_LENGTH]; + for(int32_t i = 0;; ++i) { + if(i == cesLength) { + return encodeExpansion32(newCE32s, cesLength, errorCode); + } + uint32_t ce32 = encodeOneCEAsCE32(ces[i]); + if(ce32 == Collation::NO_CE32) { break; } + newCE32s[i] = (int32_t)ce32; + } + return encodeExpansion(ces, cesLength, errorCode); +} + +uint32_t +CollationDataBuilder::encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + // See if this sequence of CEs has already been stored. + int64_t first = ces[0]; + int32_t ce64sMax = ce64s.size() - length; + for(int32_t i = 0; i <= ce64sMax; ++i) { + if(first == ce64s.elementAti(i)) { + if(i > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + for(int32_t j = 1;; ++j) { + if(j == length) { + return Collation::makeCE32FromTagIndexAndLength( + Collation::EXPANSION_TAG, i, length); + } + if(ce64s.elementAti(i + j) != ces[j]) { break; } + } + } + } + // Store the new sequence. + int32_t i = ce64s.size(); + if(i > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + for(int32_t j = 0; j < length; ++j) { + ce64s.addElement(ces[j], errorCode); + } + return Collation::makeCE32FromTagIndexAndLength(Collation::EXPANSION_TAG, i, length); +} + +uint32_t +CollationDataBuilder::encodeExpansion32(const int32_t newCE32s[], int32_t length, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + // See if this sequence of CE32s has already been stored. + int32_t first = newCE32s[0]; + int32_t ce32sMax = ce32s.size() - length; + for(int32_t i = 0; i <= ce32sMax; ++i) { + if(first == ce32s.elementAti(i)) { + if(i > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + for(int32_t j = 1;; ++j) { + if(j == length) { + return Collation::makeCE32FromTagIndexAndLength( + Collation::EXPANSION32_TAG, i, length); + } + if(ce32s.elementAti(i + j) != newCE32s[j]) { break; } + } + } + } + // Store the new sequence. + int32_t i = ce32s.size(); + if(i > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + for(int32_t j = 0; j < length; ++j) { + ce32s.addElement(newCE32s[j], errorCode); + } + return Collation::makeCE32FromTagIndexAndLength(Collation::EXPANSION32_TAG, i, length); +} + +uint32_t +CollationDataBuilder::copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + if(!Collation::isSpecialCE32(ce32)) { return ce32; } + switch(Collation::tagFromCE32(ce32)) { + case Collation::LONG_PRIMARY_TAG: + case Collation::LONG_SECONDARY_TAG: + case Collation::LATIN_EXPANSION_TAG: + // copy as is + break; + case Collation::EXPANSION32_TAG: { + const uint32_t *baseCE32s = base->ce32s + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + ce32 = encodeExpansion32( + reinterpret_cast(baseCE32s), length, errorCode); + break; + } + case Collation::EXPANSION_TAG: { + const int64_t *baseCEs = base->ces + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + ce32 = encodeExpansion(baseCEs, length, errorCode); + break; + } + case Collation::PREFIX_TAG: { + // Flatten prefixes and nested suffixes (contractions) + // into a linear list of ConditionalCE32. + const char16_t *p = base->contexts + Collation::indexFromCE32(ce32); + ce32 = CollationData::readCE32(p); // Default if no prefix match. + if(!withContext) { + return copyFromBaseCE32(c, ce32, false, errorCode); + } + ConditionalCE32 head; + UnicodeString context((char16_t)0); + int32_t index; + if(Collation::isContractionCE32(ce32)) { + index = copyContractionsFromBaseCE32(context, c, ce32, &head, errorCode); + } else { + ce32 = copyFromBaseCE32(c, ce32, true, errorCode); + head.next = index = addConditionalCE32(context, ce32, errorCode); + } + if(U_FAILURE(errorCode)) { return 0; } + ConditionalCE32 *cond = getConditionalCE32(index); // the last ConditionalCE32 so far + UCharsTrie::Iterator prefixes(p + 2, 0, errorCode); + while(prefixes.next(errorCode)) { + context = prefixes.getString(); + context.reverse(); + context.insert(0, (char16_t)context.length()); + ce32 = (uint32_t)prefixes.getValue(); + if(Collation::isContractionCE32(ce32)) { + index = copyContractionsFromBaseCE32(context, c, ce32, cond, errorCode); + } else { + ce32 = copyFromBaseCE32(c, ce32, true, errorCode); + cond->next = index = addConditionalCE32(context, ce32, errorCode); + } + if(U_FAILURE(errorCode)) { return 0; } + cond = getConditionalCE32(index); + } + ce32 = makeBuilderContextCE32(head.next); + contextChars.add(c); + break; + } + case Collation::CONTRACTION_TAG: { + if(!withContext) { + const char16_t *p = base->contexts + Collation::indexFromCE32(ce32); + ce32 = CollationData::readCE32(p); // Default if no suffix match. + return copyFromBaseCE32(c, ce32, false, errorCode); + } + ConditionalCE32 head; + UnicodeString context((char16_t)0); + copyContractionsFromBaseCE32(context, c, ce32, &head, errorCode); + ce32 = makeBuilderContextCE32(head.next); + contextChars.add(c); + break; + } + case Collation::HANGUL_TAG: + errorCode = U_UNSUPPORTED_ERROR; // We forbid tailoring of Hangul syllables. + break; + case Collation::OFFSET_TAG: + ce32 = getCE32FromOffsetCE32(true, c, ce32); + break; + case Collation::IMPLICIT_TAG: + ce32 = encodeOneCE(Collation::unassignedCEFromCodePoint(c), errorCode); + break; + default: + UPRV_UNREACHABLE_EXIT; // require ce32 == base->getFinalCE32(ce32) + } + return ce32; +} + +int32_t +CollationDataBuilder::copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32, + ConditionalCE32 *cond, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + const char16_t *p = base->contexts + Collation::indexFromCE32(ce32); + int32_t index; + if((ce32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) { + // No match on the single code point. + // We are underneath a prefix, and the default mapping is just + // a fallback to the mappings for a shorter prefix. + U_ASSERT(context.length() > 1); + index = -1; + } else { + ce32 = CollationData::readCE32(p); // Default if no suffix match. + U_ASSERT(!Collation::isContractionCE32(ce32)); + ce32 = copyFromBaseCE32(c, ce32, true, errorCode); + cond->next = index = addConditionalCE32(context, ce32, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + cond = getConditionalCE32(index); + } + + int32_t suffixStart = context.length(); + UCharsTrie::Iterator suffixes(p + 2, 0, errorCode); + while(suffixes.next(errorCode)) { + context.append(suffixes.getString()); + ce32 = copyFromBaseCE32(c, (uint32_t)suffixes.getValue(), true, errorCode); + cond->next = index = addConditionalCE32(context, ce32, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + // No need to update the unsafeBackwardSet because the tailoring set + // is already a copy of the base set. + cond = getConditionalCE32(index); + context.truncate(suffixStart); + } + U_ASSERT(index >= 0); + return index; +} + +class CopyHelper { +public: + CopyHelper(const CollationDataBuilder &s, CollationDataBuilder &d, + const CollationDataBuilder::CEModifier &m, UErrorCode &initialErrorCode) + : src(s), dest(d), modifier(m), + errorCode(initialErrorCode) {} + + UBool copyRangeCE32(UChar32 start, UChar32 end, uint32_t ce32) { + ce32 = copyCE32(ce32); + utrie2_setRange32(dest.trie, start, end, ce32, true, &errorCode); + if(CollationDataBuilder::isBuilderContextCE32(ce32)) { + dest.contextChars.add(start, end); + } + return U_SUCCESS(errorCode); + } + + uint32_t copyCE32(uint32_t ce32) { + if(!Collation::isSpecialCE32(ce32)) { + int64_t ce = modifier.modifyCE32(ce32); + if(ce != Collation::NO_CE) { + ce32 = dest.encodeOneCE(ce, errorCode); + } + } else { + int32_t tag = Collation::tagFromCE32(ce32); + if(tag == Collation::EXPANSION32_TAG) { + const uint32_t *srcCE32s = reinterpret_cast(src.ce32s.getBuffer()); + srcCE32s += Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + // Inspect the source CE32s. Just copy them if none are modified. + // Otherwise copy to modifiedCEs, with modifications. + UBool isModified = false; + for(int32_t i = 0; i < length; ++i) { + ce32 = srcCE32s[i]; + int64_t ce; + if(Collation::isSpecialCE32(ce32) || + (ce = modifier.modifyCE32(ce32)) == Collation::NO_CE) { + if(isModified) { + modifiedCEs[i] = Collation::ceFromCE32(ce32); + } + } else { + if(!isModified) { + for(int32_t j = 0; j < i; ++j) { + modifiedCEs[j] = Collation::ceFromCE32(srcCE32s[j]); + } + isModified = true; + } + modifiedCEs[i] = ce; + } + } + if(isModified) { + ce32 = dest.encodeCEs(modifiedCEs, length, errorCode); + } else { + ce32 = dest.encodeExpansion32( + reinterpret_cast(srcCE32s), length, errorCode); + } + } else if(tag == Collation::EXPANSION_TAG) { + const int64_t *srcCEs = src.ce64s.getBuffer(); + srcCEs += Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + // Inspect the source CEs. Just copy them if none are modified. + // Otherwise copy to modifiedCEs, with modifications. + UBool isModified = false; + for(int32_t i = 0; i < length; ++i) { + int64_t srcCE = srcCEs[i]; + int64_t ce = modifier.modifyCE(srcCE); + if(ce == Collation::NO_CE) { + if(isModified) { + modifiedCEs[i] = srcCE; + } + } else { + if(!isModified) { + for(int32_t j = 0; j < i; ++j) { + modifiedCEs[j] = srcCEs[j]; + } + isModified = true; + } + modifiedCEs[i] = ce; + } + } + if(isModified) { + ce32 = dest.encodeCEs(modifiedCEs, length, errorCode); + } else { + ce32 = dest.encodeExpansion(srcCEs, length, errorCode); + } + } else if(tag == Collation::BUILDER_DATA_TAG) { + // Copy the list of ConditionalCE32. + ConditionalCE32 *cond = src.getConditionalCE32ForCE32(ce32); + U_ASSERT(!cond->hasContext()); + int32_t destIndex = dest.addConditionalCE32( + cond->context, copyCE32(cond->ce32), errorCode); + ce32 = CollationDataBuilder::makeBuilderContextCE32(destIndex); + while(cond->next >= 0) { + cond = src.getConditionalCE32(cond->next); + ConditionalCE32 *prevDestCond = dest.getConditionalCE32(destIndex); + destIndex = dest.addConditionalCE32( + cond->context, copyCE32(cond->ce32), errorCode); + int32_t suffixStart = cond->prefixLength() + 1; + dest.unsafeBackwardSet.addAll(cond->context.tempSubString(suffixStart)); + prevDestCond->next = destIndex; + } + } else { + // Just copy long CEs and Latin mini expansions (and other expected values) as is, + // assuming that the modifier would not modify them. + U_ASSERT(tag == Collation::LONG_PRIMARY_TAG || + tag == Collation::LONG_SECONDARY_TAG || + tag == Collation::LATIN_EXPANSION_TAG || + tag == Collation::HANGUL_TAG); + } + } + return ce32; + } + + const CollationDataBuilder &src; + CollationDataBuilder &dest; + const CollationDataBuilder::CEModifier &modifier; + int64_t modifiedCEs[Collation::MAX_EXPANSION_LENGTH]; + UErrorCode errorCode; +}; + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumRangeForCopy(const void *context, UChar32 start, UChar32 end, uint32_t value) { + return + value == Collation::UNASSIGNED_CE32 || value == Collation::FALLBACK_CE32 || + ((CopyHelper *)context)->copyRangeCE32(start, end, value); +} + +U_CDECL_END + +void +CollationDataBuilder::copyFrom(const CollationDataBuilder &src, const CEModifier &modifier, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(trie == nullptr || utrie2_isFrozen(trie)) { + errorCode = U_INVALID_STATE_ERROR; + return; + } + CopyHelper helper(src, *this, modifier, errorCode); + utrie2_enum(src.trie, nullptr, enumRangeForCopy, &helper); + errorCode = helper.errorCode; + // Update the contextChars and the unsafeBackwardSet while copying, + // in case a character had conditional mappings in the source builder + // and they were removed later. + modified |= src.modified; +} + +void +CollationDataBuilder::optimize(const UnicodeSet &set, UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || set.isEmpty()) { return; } + UnicodeSetIterator iter(set); + while(iter.next() && !iter.isString()) { + UChar32 c = iter.getCodepoint(); + uint32_t ce32 = utrie2_get32(trie, c); + if(ce32 == Collation::FALLBACK_CE32) { + ce32 = base->getFinalCE32(base->getCE32(c)); + ce32 = copyFromBaseCE32(c, ce32, true, errorCode); + utrie2_set32(trie, c, ce32, &errorCode); + } + } + modified = true; +} + +void +CollationDataBuilder::suppressContractions(const UnicodeSet &set, UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || set.isEmpty()) { return; } + UnicodeSetIterator iter(set); + while(iter.next() && !iter.isString()) { + UChar32 c = iter.getCodepoint(); + uint32_t ce32 = utrie2_get32(trie, c); + if(ce32 == Collation::FALLBACK_CE32) { + ce32 = base->getFinalCE32(base->getCE32(c)); + if(Collation::ce32HasContext(ce32)) { + ce32 = copyFromBaseCE32(c, ce32, false /* without context */, errorCode); + utrie2_set32(trie, c, ce32, &errorCode); + } + } else if(isBuilderContextCE32(ce32)) { + ce32 = getConditionalCE32ForCE32(ce32)->ce32; + // Simply abandon the list of ConditionalCE32. + // The caller will copy this builder in the end, + // eliminating unreachable data. + utrie2_set32(trie, c, ce32, &errorCode); + contextChars.remove(c); + } + } + modified = true; +} + +UBool +CollationDataBuilder::getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + UBool anyJamoAssigned = base == nullptr; // always set jamoCE32s in the base data + UBool needToCopyFromBase = false; + for(int32_t j = 0; j < CollationData::JAMO_CE32S_LENGTH; ++j) { // Count across Jamo types. + UChar32 jamo = jamoCpFromIndex(j); + UBool fromBase = false; + uint32_t ce32 = utrie2_get32(trie, jamo); + anyJamoAssigned |= Collation::isAssignedCE32(ce32); + // TODO: Try to prevent [optimize [Jamo]] from counting as anyJamoAssigned. + // (As of CLDR 24 [2013] the Korean tailoring does not optimize conjoining Jamo.) + if(ce32 == Collation::FALLBACK_CE32) { + fromBase = true; + ce32 = base->getCE32(jamo); + } + if(Collation::isSpecialCE32(ce32)) { + switch(Collation::tagFromCE32(ce32)) { + case Collation::LONG_PRIMARY_TAG: + case Collation::LONG_SECONDARY_TAG: + case Collation::LATIN_EXPANSION_TAG: + // Copy the ce32 as-is. + break; + case Collation::EXPANSION32_TAG: + case Collation::EXPANSION_TAG: + case Collation::PREFIX_TAG: + case Collation::CONTRACTION_TAG: + if(fromBase) { + // Defer copying until we know if anyJamoAssigned. + ce32 = Collation::FALLBACK_CE32; + needToCopyFromBase = true; + } + break; + case Collation::IMPLICIT_TAG: + // An unassigned Jamo should only occur in tests with incomplete bases. + U_ASSERT(fromBase); + ce32 = Collation::FALLBACK_CE32; + needToCopyFromBase = true; + break; + case Collation::OFFSET_TAG: + ce32 = getCE32FromOffsetCE32(fromBase, jamo, ce32); + break; + case Collation::FALLBACK_TAG: + case Collation::RESERVED_TAG_3: + case Collation::BUILDER_DATA_TAG: + case Collation::DIGIT_TAG: + case Collation::U0000_TAG: + case Collation::HANGUL_TAG: + case Collation::LEAD_SURROGATE_TAG: + errorCode = U_INTERNAL_PROGRAM_ERROR; + return false; + } + } + jamoCE32s[j] = ce32; + } + if(anyJamoAssigned && needToCopyFromBase) { + for(int32_t j = 0; j < CollationData::JAMO_CE32S_LENGTH; ++j) { + if(jamoCE32s[j] == Collation::FALLBACK_CE32) { + UChar32 jamo = jamoCpFromIndex(j); + jamoCE32s[j] = copyFromBaseCE32(jamo, base->getCE32(jamo), + /*withContext=*/ true, errorCode); + } + } + } + return anyJamoAssigned && U_SUCCESS(errorCode); +} + +void +CollationDataBuilder::setDigitTags(UErrorCode &errorCode) { + UnicodeSet digits(UNICODE_STRING_SIMPLE("[:Nd:]"), errorCode); + if(U_FAILURE(errorCode)) { return; } + UnicodeSetIterator iter(digits); + while(iter.next()) { + U_ASSERT(!iter.isString()); + UChar32 c = iter.getCodepoint(); + uint32_t ce32 = utrie2_get32(trie, c); + if(ce32 != Collation::FALLBACK_CE32 && ce32 != Collation::UNASSIGNED_CE32) { + int32_t index = addCE32(ce32, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return; + } + ce32 = Collation::makeCE32FromTagIndexAndLength( + Collation::DIGIT_TAG, index, u_charDigitValue(c)); + utrie2_set32(trie, c, ce32, &errorCode); + } + } +} + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumRangeLeadValue(const void *context, UChar32 /*start*/, UChar32 /*end*/, uint32_t value) { + int32_t *pValue = (int32_t *)context; + if(value == Collation::UNASSIGNED_CE32) { + value = Collation::LEAD_ALL_UNASSIGNED; + } else if(value == Collation::FALLBACK_CE32) { + value = Collation::LEAD_ALL_FALLBACK; + } else { + *pValue = Collation::LEAD_MIXED; + return false; + } + if(*pValue < 0) { + *pValue = (int32_t)value; + } else if(*pValue != (int32_t)value) { + *pValue = Collation::LEAD_MIXED; + return false; + } + return true; +} + +U_CDECL_END + +void +CollationDataBuilder::setLeadSurrogates(UErrorCode &errorCode) { + for(char16_t lead = 0xd800; lead < 0xdc00; ++lead) { + int32_t value = -1; + utrie2_enumForLeadSurrogate(trie, lead, nullptr, enumRangeLeadValue, &value); + utrie2_set32ForLeadSurrogateCodeUnit( + trie, lead, + Collation::makeCE32FromTagAndIndex(Collation::LEAD_SURROGATE_TAG, 0) | (uint32_t)value, + &errorCode); + } +} + +void +CollationDataBuilder::build(CollationData &data, UErrorCode &errorCode) { + buildMappings(data, errorCode); + if(base != nullptr) { + data.numericPrimary = base->numericPrimary; + data.compressibleBytes = base->compressibleBytes; + data.numScripts = base->numScripts; + data.scriptsIndex = base->scriptsIndex; + data.scriptStarts = base->scriptStarts; + data.scriptStartsLength = base->scriptStartsLength; + } + buildFastLatinTable(data, errorCode); +} + +void +CollationDataBuilder::buildMappings(CollationData &data, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(trie == nullptr || utrie2_isFrozen(trie)) { + errorCode = U_INVALID_STATE_ERROR; + return; + } + + buildContexts(errorCode); + + uint32_t jamoCE32s[CollationData::JAMO_CE32S_LENGTH]; + int32_t jamoIndex = -1; + if(getJamoCE32s(jamoCE32s, errorCode)) { + jamoIndex = ce32s.size(); + for(int32_t i = 0; i < CollationData::JAMO_CE32S_LENGTH; ++i) { + ce32s.addElement((int32_t)jamoCE32s[i], errorCode); + } + // Small optimization: Use a bit in the Hangul ce32 + // to indicate that none of the Jamo CE32s are isSpecialCE32() + // (as it should be in the root collator). + // It allows CollationIterator to avoid recursive function calls and per-Jamo tests. + // In order to still have good trie compression and keep this code simple, + // we only set this flag if a whole block of 588 Hangul syllables starting with + // a common leading consonant (Jamo L) has this property. + UBool isAnyJamoVTSpecial = false; + for(int32_t i = Hangul::JAMO_L_COUNT; i < CollationData::JAMO_CE32S_LENGTH; ++i) { + if(Collation::isSpecialCE32(jamoCE32s[i])) { + isAnyJamoVTSpecial = true; + break; + } + } + uint32_t hangulCE32 = Collation::makeCE32FromTagAndIndex(Collation::HANGUL_TAG, 0); + UChar32 c = Hangul::HANGUL_BASE; + for(int32_t i = 0; i < Hangul::JAMO_L_COUNT; ++i) { // iterate over the Jamo L + uint32_t ce32 = hangulCE32; + if(!isAnyJamoVTSpecial && !Collation::isSpecialCE32(jamoCE32s[i])) { + ce32 |= Collation::HANGUL_NO_SPECIAL_JAMO; + } + UChar32 limit = c + Hangul::JAMO_VT_COUNT; + utrie2_setRange32(trie, c, limit - 1, ce32, true, &errorCode); + c = limit; + } + } else { + // Copy the Hangul CE32s from the base in blocks per Jamo L, + // assuming that HANGUL_NO_SPECIAL_JAMO is set or not set for whole blocks. + for(UChar32 c = Hangul::HANGUL_BASE; c < Hangul::HANGUL_LIMIT;) { + uint32_t ce32 = base->getCE32(c); + U_ASSERT(Collation::hasCE32Tag(ce32, Collation::HANGUL_TAG)); + UChar32 limit = c + Hangul::JAMO_VT_COUNT; + utrie2_setRange32(trie, c, limit - 1, ce32, true, &errorCode); + c = limit; + } + } + + setDigitTags(errorCode); + setLeadSurrogates(errorCode); + + if (!icu4xMode) { + // For U+0000, move its normal ce32 into CE32s[0] and set U0000_TAG. + ce32s.setElementAt((int32_t)utrie2_get32(trie, 0), 0); + utrie2_set32(trie, 0, Collation::makeCE32FromTagAndIndex(Collation::U0000_TAG, 0), &errorCode); + } + + utrie2_freeze(trie, UTRIE2_32_VALUE_BITS, &errorCode); + if(U_FAILURE(errorCode)) { return; } + + // Mark each lead surrogate as "unsafe" + // if any of its 1024 associated supplementary code points is "unsafe". + UChar32 c = 0x10000; + for(char16_t lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) { + if(unsafeBackwardSet.containsSome(c, c + 0x3ff)) { + unsafeBackwardSet.add(lead); + } + } + unsafeBackwardSet.freeze(); + + data.trie = trie; + data.ce32s = reinterpret_cast(ce32s.getBuffer()); + data.ces = ce64s.getBuffer(); + data.contexts = contexts.getBuffer(); + + data.ce32sLength = ce32s.size(); + data.cesLength = ce64s.size(); + data.contextsLength = contexts.length(); + + data.base = base; + if(jamoIndex >= 0) { + data.jamoCE32s = data.ce32s + jamoIndex; + } else { + data.jamoCE32s = base->jamoCE32s; + } + data.unsafeBackwardSet = &unsafeBackwardSet; +} + +void +CollationDataBuilder::clearContexts() { + contexts.remove(); + // Incrementing the contexts build "era" invalidates all of the builtCE32 + // from before this clearContexts() call. + // Simpler than finding and resetting all of those fields. + ++contextsEra; +} + +void +CollationDataBuilder::buildContexts(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + // Ignore abandoned lists and the cached builtCE32, + // and build all contexts from scratch. + clearContexts(); + UnicodeSetIterator iter(contextChars); + while(U_SUCCESS(errorCode) && iter.next()) { + U_ASSERT(!iter.isString()); + UChar32 c = iter.getCodepoint(); + uint32_t ce32 = utrie2_get32(trie, c); + if(!isBuilderContextCE32(ce32)) { + // Impossible: No context data for c in contextChars. + errorCode = U_INTERNAL_PROGRAM_ERROR; + return; + } + ConditionalCE32 *cond = getConditionalCE32ForCE32(ce32); + ce32 = buildContext(cond, errorCode); + utrie2_set32(trie, c, ce32, &errorCode); + } +} + +uint32_t +CollationDataBuilder::buildContext(ConditionalCE32 *head, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + // The list head must have no context. + U_ASSERT(!head->hasContext()); + // The list head must be followed by one or more nodes that all do have context. + U_ASSERT(head->next >= 0); + UCharsTrieBuilder prefixBuilder(errorCode); + UCharsTrieBuilder contractionBuilder(errorCode); + // This outer loop goes from each prefix to the next. + // For each prefix it finds the one or more same-prefix entries (firstCond..lastCond). + // If there are multiple suffixes for the same prefix, + // then an inner loop builds a contraction trie for them. + for(ConditionalCE32 *cond = head;; cond = getConditionalCE32(cond->next)) { + if(U_FAILURE(errorCode)) { return 0; } // early out for memory allocation errors + // After the list head, the prefix or suffix can be empty, but not both. + U_ASSERT(cond == head || cond->hasContext()); + int32_t prefixLength = cond->prefixLength(); + UnicodeString prefix(cond->context, 0, prefixLength + 1); + // Collect all contraction suffixes for one prefix. + ConditionalCE32 *firstCond = cond; + ConditionalCE32 *lastCond; + do { + lastCond = cond; + // Clear the defaultCE32 fields as we go. + // They are left over from building a previous version of this list of contexts. + // + // One of the code paths below may copy a preceding defaultCE32 + // into its emptySuffixCE32. + // If a new suffix has been inserted before what used to be + // the firstCond for its prefix, then that previous firstCond could still + // contain an outdated defaultCE32 from an earlier buildContext() and + // result in an incorrect emptySuffixCE32. + // So we reset all defaultCE32 before reading and setting new values. + cond->defaultCE32 = Collation::NO_CE32; + } while(cond->next >= 0 && + (cond = getConditionalCE32(cond->next))->context.startsWith(prefix)); + uint32_t ce32; + int32_t suffixStart = prefixLength + 1; // == prefix.length() + if(lastCond->context.length() == suffixStart) { + // One prefix without contraction suffix. + U_ASSERT(firstCond == lastCond); + ce32 = lastCond->ce32; + cond = lastCond; + } else { + // Build the contractions trie. + contractionBuilder.clear(); + // Entry for an empty suffix, to be stored before the trie. + uint32_t emptySuffixCE32 = 0; + uint32_t flags = 0; + if(firstCond->context.length() == suffixStart) { + // There is a mapping for the prefix and the single character c. (p|c) + // If no other suffix matches, then we return this value. + emptySuffixCE32 = firstCond->ce32; + cond = getConditionalCE32(firstCond->next); + } else { + // There is no mapping for the prefix and just the single character. + // (There is no p|c, only p|cd, p|ce etc.) + flags |= Collation::CONTRACT_SINGLE_CP_NO_MATCH; + // When the prefix matches but none of the prefix-specific suffixes, + // then we fall back to the mappings with the next-longest prefix, + // and ultimately to mappings with no prefix. + // Each fallback might be another set of contractions. + // For example, if there are mappings for ch, p|cd, p|ce, but not for p|c, + // then in text "pch" we find the ch contraction. + for(cond = head;; cond = getConditionalCE32(cond->next)) { + int32_t length = cond->prefixLength(); + if(length == prefixLength) { break; } + if(cond->defaultCE32 != Collation::NO_CE32 && + (length==0 || prefix.endsWith(cond->context, 1, length))) { + emptySuffixCE32 = cond->defaultCE32; + } + } + cond = firstCond; + } + // Optimization: Set a flag when + // the first character of every contraction suffix has lccc!=0. + // Short-circuits contraction matching when a normal letter follows. + flags |= Collation::CONTRACT_NEXT_CCC; + // Add all of the non-empty suffixes into the contraction trie. + for(;;) { + UnicodeString suffix(cond->context, suffixStart); + uint16_t fcd16 = nfcImpl.getFCD16(suffix.char32At(0)); + if(fcd16 <= 0xff) { + flags &= ~Collation::CONTRACT_NEXT_CCC; + } + fcd16 = nfcImpl.getFCD16(suffix.char32At(suffix.length() - 1)); + if(fcd16 > 0xff) { + // The last suffix character has lccc!=0, allowing for discontiguous contractions. + flags |= Collation::CONTRACT_TRAILING_CCC; + } + if (icu4xMode && (flags & Collation::CONTRACT_HAS_STARTER) == 0) { + for (int32_t i = 0; i < suffix.length();) { + UChar32 c = suffix.char32At(i); + if (!u_getCombiningClass(c)) { + flags |= Collation::CONTRACT_HAS_STARTER; + break; + } + if (c > 0xFFFF) { + i += 2; + } else { + ++i; + } + } + } + contractionBuilder.add(suffix, (int32_t)cond->ce32, errorCode); + if(cond == lastCond) { break; } + cond = getConditionalCE32(cond->next); + } + int32_t index = addContextTrie(emptySuffixCE32, contractionBuilder, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + ce32 = Collation::makeCE32FromTagAndIndex(Collation::CONTRACTION_TAG, index) | flags; + } + U_ASSERT(cond == lastCond); + firstCond->defaultCE32 = ce32; + if(prefixLength == 0) { + if(cond->next < 0) { + // No non-empty prefixes, only contractions. + return ce32; + } + } else { + prefix.remove(0, 1); // Remove the length unit. + prefix.reverse(); + prefixBuilder.add(prefix, (int32_t)ce32, errorCode); + if(cond->next < 0) { break; } + } + } + U_ASSERT(head->defaultCE32 != Collation::NO_CE32); + int32_t index = addContextTrie(head->defaultCE32, prefixBuilder, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + if(index > Collation::MAX_INDEX) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return 0; + } + return Collation::makeCE32FromTagAndIndex(Collation::PREFIX_TAG, index); +} + +int32_t +CollationDataBuilder::addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder, + UErrorCode &errorCode) { + UnicodeString context; + context.append((char16_t)(defaultCE32 >> 16)).append((char16_t)defaultCE32); + UnicodeString trieString; + context.append(trieBuilder.buildUnicodeString(USTRINGTRIE_BUILD_SMALL, trieString, errorCode)); + if(U_FAILURE(errorCode)) { return -1; } + int32_t index = contexts.indexOf(context); + if(index < 0) { + index = contexts.length(); + contexts.append(context); + } + return index; +} + +void +CollationDataBuilder::buildFastLatinTable(CollationData &data, UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || !fastLatinEnabled) { return; } + + delete fastLatinBuilder; + fastLatinBuilder = new CollationFastLatinBuilder(errorCode); + if(fastLatinBuilder == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + if(fastLatinBuilder->forData(data, errorCode)) { + const uint16_t *table = fastLatinBuilder->getTable(); + int32_t length = fastLatinBuilder->lengthOfTable(); + if(base != nullptr && length == base->fastLatinTableLength && + uprv_memcmp(table, base->fastLatinTable, length * 2) == 0) { + // Same fast Latin table as in the base, use that one instead. + delete fastLatinBuilder; + fastLatinBuilder = nullptr; + table = base->fastLatinTable; + } + data.fastLatinTable = table; + data.fastLatinTableLength = length; + } else { + delete fastLatinBuilder; + fastLatinBuilder = nullptr; + } +} + +int32_t +CollationDataBuilder::getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength) { + return getCEs(s, 0, ces, cesLength); +} + +int32_t +CollationDataBuilder::getCEs(const UnicodeString &prefix, const UnicodeString &s, + int64_t ces[], int32_t cesLength) { + int32_t prefixLength = prefix.length(); + if(prefixLength == 0) { + return getCEs(s, 0, ces, cesLength); + } else { + return getCEs(prefix + s, prefixLength, ces, cesLength); + } +} + +int32_t +CollationDataBuilder::getCEs(const UnicodeString &s, int32_t start, + int64_t ces[], int32_t cesLength) { + if(collIter == nullptr) { + collIter = new DataBuilderCollationIterator(*this); + if(collIter == nullptr) { return 0; } + } + return collIter->fetchCEs(s, start, ces, cesLength); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationdatabuilder.h b/intl/icu/source/i18n/collationdatabuilder.h new file mode 100644 index 0000000000..cbbd8f264b --- /dev/null +++ b/intl/icu/source/i18n/collationdatabuilder.h @@ -0,0 +1,269 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatabuilder.h +* +* created on: 2012apr01 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONDATABUILDER_H__ +#define __COLLATIONDATABUILDER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/uversion.h" +#include "collation.h" +#include "collationdata.h" +#include "collationsettings.h" +#include "normalizer2impl.h" +#include "utrie2.h" +#include "uvectr32.h" +#include "uvectr64.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +struct ConditionalCE32; + +class CollationFastLatinBuilder; +class CopyHelper; +class DataBuilderCollationIterator; +class UCharsTrieBuilder; + +/** + * Low-level CollationData builder. + * Takes (character, CE) pairs and builds them into runtime data structures. + * Supports characters with context prefixes and contraction suffixes. + */ +class U_I18N_API CollationDataBuilder : public UObject { +public: + /** + * Collation element modifier. Interface class for a modifier + * that changes a tailoring builder's temporary CEs to final CEs. + * Called for every non-special CE32 and every expansion CE. + */ + class CEModifier : public UObject { + public: + virtual ~CEModifier(); + /** Returns a new CE to replace the non-special input CE32, or else Collation::NO_CE. */ + virtual int64_t modifyCE32(uint32_t ce32) const = 0; + /** Returns a new CE to replace the input CE, or else Collation::NO_CE. */ + virtual int64_t modifyCE(int64_t ce) const = 0; + }; + + CollationDataBuilder(UBool icu4xMode, UErrorCode &errorCode); + + virtual ~CollationDataBuilder(); + + void initForTailoring(const CollationData *b, UErrorCode &errorCode); + + virtual UBool isCompressibleLeadByte(uint32_t b) const; + + inline UBool isCompressiblePrimary(uint32_t p) const { + return isCompressibleLeadByte(p >> 24); + } + + /** + * @return true if this builder has mappings (e.g., add() has been called) + */ + UBool hasMappings() const { return modified; } + + /** + * @return true if c has CEs in this builder + */ + UBool isAssigned(UChar32 c) const; + + /** + * @return the three-byte primary if c maps to a single such CE and has no context data, + * otherwise returns 0. + */ + uint32_t getLongPrimaryIfSingleCE(UChar32 c) const; + + /** + * @return the single CE for c. + * Sets an error code if c does not have a single CE. + */ + int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const; + + void add(const UnicodeString &prefix, const UnicodeString &s, + const int64_t ces[], int32_t cesLength, + UErrorCode &errorCode); + + /** + * Encodes the ces as either the returned ce32 by itself, + * or by storing an expansion, with the returned ce32 referring to that. + * + * add(p, s, ces, cesLength) = addCE32(p, s, encodeCEs(ces, cesLength)) + */ + virtual uint32_t encodeCEs(const int64_t ces[], int32_t cesLength, UErrorCode &errorCode); + void addCE32(const UnicodeString &prefix, const UnicodeString &s, + uint32_t ce32, UErrorCode &errorCode); + + /** + * Sets three-byte-primary CEs for a range of code points in code point order, + * if it is worth doing; otherwise no change is made. + * None of the code points in the range should have complex mappings so far + * (expansions/contractions/prefixes). + * @param start first code point + * @param end last code point (inclusive) + * @param primary primary weight for 'start' + * @param step per-code point primary-weight increment + * @param errorCode ICU in/out error code + * @return true if an OFFSET_TAG range was used for start..end + */ + UBool maybeSetPrimaryRange(UChar32 start, UChar32 end, + uint32_t primary, int32_t step, + UErrorCode &errorCode); + + /** + * Sets three-byte-primary CEs for a range of code points in code point order. + * Sets range values if that is worth doing, or else individual values. + * None of the code points in the range should have complex mappings so far + * (expansions/contractions/prefixes). + * @param start first code point + * @param end last code point (inclusive) + * @param primary primary weight for 'start' + * @param step per-code point primary-weight increment + * @param errorCode ICU in/out error code + * @return the next primary after 'end': start primary incremented by ((end-start)+1)*step + */ + uint32_t setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end, + uint32_t primary, int32_t step, + UErrorCode &errorCode); + + /** + * Copies all mappings from the src builder, with modifications. + * This builder here must not be built yet, and should be empty. + */ + void copyFrom(const CollationDataBuilder &src, const CEModifier &modifier, + UErrorCode &errorCode); + + void optimize(const UnicodeSet &set, UErrorCode &errorCode); + void suppressContractions(const UnicodeSet &set, UErrorCode &errorCode); + + void enableFastLatin() { fastLatinEnabled = true; } + virtual void build(CollationData &data, UErrorCode &errorCode); + + /** + * Looks up CEs for s and appends them to the ces array. + * Does not handle normalization: s should be in FCD form. + * + * Does not write completely ignorable CEs. + * Does not write beyond Collation::MAX_EXPANSION_LENGTH. + * + * @return incremented cesLength + */ + int32_t getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength); + int32_t getCEs(const UnicodeString &prefix, const UnicodeString &s, + int64_t ces[], int32_t cesLength); + +protected: + friend class CopyHelper; + friend class DataBuilderCollationIterator; + + uint32_t getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const; + + int32_t addCE(int64_t ce, UErrorCode &errorCode); + int32_t addCE32(uint32_t ce32, UErrorCode &errorCode); + int32_t addConditionalCE32(const UnicodeString &context, uint32_t ce32, UErrorCode &errorCode); + + inline ConditionalCE32 *getConditionalCE32(int32_t index) const { + return static_cast(conditionalCE32s[index]); + } + inline ConditionalCE32 *getConditionalCE32ForCE32(uint32_t ce32) const { + return getConditionalCE32(Collation::indexFromCE32(ce32)); + } + + static uint32_t makeBuilderContextCE32(int32_t index) { + return Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, index); + } + static inline UBool isBuilderContextCE32(uint32_t ce32) { + return Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG); + } + + static uint32_t encodeOneCEAsCE32(int64_t ce); + uint32_t encodeOneCE(int64_t ce, UErrorCode &errorCode); + uint32_t encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode); + uint32_t encodeExpansion32(const int32_t newCE32s[], int32_t length, UErrorCode &errorCode); + + uint32_t copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, UErrorCode &errorCode); + /** + * Copies base contractions to a list of ConditionalCE32. + * Sets cond->next to the index of the first new item + * and returns the index of the last new item. + */ + int32_t copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32, + ConditionalCE32 *cond, UErrorCode &errorCode); + + UBool getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode); + void setDigitTags(UErrorCode &errorCode); + void setLeadSurrogates(UErrorCode &errorCode); + + void buildMappings(CollationData &data, UErrorCode &errorCode); + + void clearContexts(); + void buildContexts(UErrorCode &errorCode); + uint32_t buildContext(ConditionalCE32 *head, UErrorCode &errorCode); + int32_t addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder, + UErrorCode &errorCode); + + void buildFastLatinTable(CollationData &data, UErrorCode &errorCode); + + int32_t getCEs(const UnicodeString &s, int32_t start, int64_t ces[], int32_t cesLength); + + static UChar32 jamoCpFromIndex(int32_t i) { + // 0 <= i < CollationData::JAMO_CE32S_LENGTH = 19 + 21 + 27 + if(i < Hangul::JAMO_L_COUNT) { return Hangul::JAMO_L_BASE + i; } + i -= Hangul::JAMO_L_COUNT; + if(i < Hangul::JAMO_V_COUNT) { return Hangul::JAMO_V_BASE + i; } + i -= Hangul::JAMO_V_COUNT; + // i < 27 + return Hangul::JAMO_T_BASE + 1 + i; + } + + /** @see Collation::BUILDER_DATA_TAG */ + static const uint32_t IS_BUILDER_JAMO_CE32 = 0x100; + + const Normalizer2Impl &nfcImpl; + const CollationData *base; + const CollationSettings *baseSettings; + UTrie2 *trie; + UVector32 ce32s; + UVector64 ce64s; + UVector conditionalCE32s; // vector of ConditionalCE32 + // Characters that have context (prefixes or contraction suffixes). + UnicodeSet contextChars; + // Serialized UCharsTrie structures for finalized contexts. + UnicodeString contexts; +private: + /** + * The "era" of building intermediate contexts. + * When the array of cached, temporary contexts overflows, then clearContexts() + * removes them all and invalidates the builtCE32 that used to point to built tries. + * See ConditionalCE32::era. + */ + int32_t contextsEra = 0; +protected: + UnicodeSet unsafeBackwardSet; + UBool modified; + UBool icu4xMode; + + UBool fastLatinEnabled; + CollationFastLatinBuilder *fastLatinBuilder; + + DataBuilderCollationIterator *collIter; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONDATABUILDER_H__ diff --git a/intl/icu/source/i18n/collationdatareader.cpp b/intl/icu/source/i18n/collationdatareader.cpp new file mode 100644 index 0000000000..1884208eaa --- /dev/null +++ b/intl/icu/source/i18n/collationdatareader.cpp @@ -0,0 +1,482 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatareader.cpp +* +* created on: 2013feb07 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/udata.h" +#include "unicode/uscript.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationdatareader.h" +#include "collationfastlatin.h" +#include "collationkeys.h" +#include "collationrootelements.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "collunsafe.h" +#include "normalizer2impl.h" +#include "uassert.h" +#include "ucmndata.h" +#include "utrie2.h" + +U_NAMESPACE_BEGIN + +namespace { + +int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) { + return (i < length) ? indexes[i] : -1; +} + +} // namespace + +void +CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength, + CollationTailoring &tailoring, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(base != nullptr) { + if(inBytes == nullptr || (0 <= inLength && inLength < 24)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + const DataHeader *header = reinterpret_cast(inBytes); + if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 && + isAcceptable(tailoring.version, nullptr, nullptr, &header->info))) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + if(base->getUCAVersion() != tailoring.getUCAVersion()) { + errorCode = U_COLLATOR_VERSION_MISMATCH; + return; + } + int32_t headerLength = header->dataHeader.headerSize; + inBytes += headerLength; + if(inLength >= 0) { + inLength -= headerLength; + } + } + + if(inBytes == nullptr || (0 <= inLength && inLength < 8)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + const int32_t *inIndexes = reinterpret_cast(inBytes); + int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH]; + if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) { + errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes. + return; + } + + // Assume that the tailoring data is in initial state, + // with nullptr pointers and 0 lengths. + + // Set pointers to non-empty data parts. + // Do this in order of their byte offsets. (Should help porting to Java.) + + int32_t index; // one of the indexes[] slots + int32_t offset; // byte offset for the index part + int32_t length; // number of bytes in the index part + + if(indexesLength > IX_TOTAL_SIZE) { + length = inIndexes[IX_TOTAL_SIZE]; + } else if(indexesLength > IX_REORDER_CODES_OFFSET) { + length = inIndexes[indexesLength - 1]; + } else { + length = 0; // only indexes, and inLength was already checked for them + } + if(0 <= inLength && inLength < length) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + + const CollationData *baseData = base == nullptr ? nullptr : base->data; + const int32_t *reorderCodes = nullptr; + int32_t reorderCodesLength = 0; + const uint32_t *reorderRanges = nullptr; + int32_t reorderRangesLength = 0; + index = IX_REORDER_CODES_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 4) { + if(baseData == nullptr) { + // We assume for collation settings that + // the base data does not have a reordering. + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + reorderCodes = reinterpret_cast(inBytes + offset); + reorderCodesLength = length / 4; + + // The reorderRanges (if any) are the trailing reorderCodes entries. + // Split the array at the boundary. + // Script or reorder codes do not exceed 16-bit values. + // Range limits are stored in the upper 16 bits, and are never 0. + while(reorderRangesLength < reorderCodesLength && + (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) { + ++reorderRangesLength; + } + U_ASSERT(reorderRangesLength < reorderCodesLength); + if(reorderRangesLength != 0) { + reorderCodesLength -= reorderRangesLength; + reorderRanges = reinterpret_cast(reorderCodes + reorderCodesLength); + } + } + + // There should be a reorder table only if there are reorder codes. + // However, when there are reorder codes the reorder table may be omitted to reduce + // the data size. + const uint8_t *reorderTable = nullptr; + index = IX_REORDER_TABLE_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 256) { + if(reorderCodesLength == 0) { + errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes. + return; + } + reorderTable = inBytes + offset; + } else { + // If we have reorder codes, then build the reorderTable at the end, + // when the CollationData is otherwise complete. + } + + if(baseData != nullptr && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + CollationData *data = nullptr; // Remains nullptr if there are no mappings. + + index = IX_TRIE_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 8) { + if(!tailoring.ensureOwnedData(errorCode)) { return; } + data = tailoring.ownedData; + data->base = baseData; + data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000; + data->trie = tailoring.trie = utrie2_openFromSerialized( + UTRIE2_32_VALUE_BITS, inBytes + offset, length, nullptr, + &errorCode); + if(U_FAILURE(errorCode)) { return; } + } else if(baseData != nullptr) { + // Use the base data. Only the settings are tailored. + tailoring.data = baseData; + } else { + errorCode = U_INVALID_FORMAT_ERROR; // No mappings. + return; + } + + index = IX_CES_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 8) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie. + return; + } + data->ces = reinterpret_cast(inBytes + offset); + data->cesLength = length / 8; + } + + index = IX_CE32S_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 4) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie. + return; + } + data->ce32s = reinterpret_cast(inBytes + offset); + data->ce32sLength = length / 4; + } + + int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START); + if(jamoCE32sStart >= 0) { + if(data == nullptr || data->ce32s == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[]. + return; + } + data->jamoCE32s = data->ce32s + jamoCE32sStart; + } else if(data == nullptr) { + // Nothing to do. + } else if(baseData != nullptr) { + data->jamoCE32s = baseData->jamoCE32s; + } else { + errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing. + return; + } + + index = IX_ROOT_ELEMENTS_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 4) { + length /= 4; + if(data == nullptr || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + data->rootElements = reinterpret_cast(inBytes + offset); + data->rootElementsLength = length; + uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE]; + if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES]; + if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) { + // [fixed last secondary common byte] is too low, + // and secondary weights would collide with compressed common secondaries. + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + } + + index = IX_CONTEXTS_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 2) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie. + return; + } + data->contexts = reinterpret_cast(inBytes + offset); + data->contextsLength = length / 2; + } + + index = IX_UNSAFE_BWD_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 2) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + if(baseData == nullptr) { +#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE) + tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode); + if(tailoring.unsafeBackwardSet == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } else if (U_FAILURE(errorCode)) { + return; + } +#else + // Create the unsafe-backward set for the root collator. + // Include all non-zero combining marks and trail surrogates. + // We do this at load time, rather than at build time, + // to simplify Unicode version bootstrapping: + // The root data builder only needs the new FractionalUCA.txt data, + // but it need not be built with a version of ICU already updated to + // the corresponding new Unicode Character Database. + // + // The following is an optimized version of + // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]"). + // It is faster and requires fewer code dependencies. + tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates + if(tailoring.unsafeBackwardSet == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet); +#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION + } else { + // Clone the root collator's set contents. + tailoring.unsafeBackwardSet = static_cast( + baseData->unsafeBackwardSet->cloneAsThawed()); + if(tailoring.unsafeBackwardSet == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + // Add the ranges from the data file to the unsafe-backward set. + USerializedSet sset; + const uint16_t *unsafeData = reinterpret_cast(inBytes + offset); + if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + int32_t count = uset_getSerializedRangeCount(&sset); + for(int32_t i = 0; i < count; ++i) { + UChar32 start, end; + uset_getSerializedRange(&sset, i, &start, &end); + tailoring.unsafeBackwardSet->add(start, end); + } + // Mark each lead surrogate as "unsafe" + // if any of its 1024 associated supplementary code points is "unsafe". + UChar32 c = 0x10000; + for(char16_t lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) { + if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) { + tailoring.unsafeBackwardSet->add(lead); + } + } + tailoring.unsafeBackwardSet->freeze(); + data->unsafeBackwardSet = tailoring.unsafeBackwardSet; + } else if(data == nullptr) { + // Nothing to do. + } else if(baseData != nullptr) { + // No tailoring-specific data: Alias the root collator's set. + data->unsafeBackwardSet = baseData->unsafeBackwardSet; + } else { + errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet. + return; + } + + // If the fast Latin format version is different, + // or the version is set to 0 for "no fast Latin table", + // then just always use the normal string comparison path. + if(data != nullptr) { + data->fastLatinTable = nullptr; + data->fastLatinTableLength = 0; + if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) { + index = IX_FAST_LATIN_TABLE_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 2) { + data->fastLatinTable = reinterpret_cast(inBytes + offset); + data->fastLatinTableLength = length / 2; + if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) { + errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch + return; + } + } else if(baseData != nullptr) { + data->fastLatinTable = baseData->fastLatinTable; + data->fastLatinTableLength = baseData->fastLatinTableLength; + } + } + } + + index = IX_SCRIPTS_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 2) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + const uint16_t *scripts = reinterpret_cast(inBytes + offset); + int32_t scriptsLength = length / 2; + data->numScripts = scripts[0]; + // There must be enough entries for both arrays, including more than two range starts. + data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16); + if(data->scriptStartsLength <= 2 || + CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + data->scriptsIndex = scripts + 1; + data->scriptStarts = scripts + 1 + data->numScripts + 16; + if(!(data->scriptStarts[0] == 0 && + data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) && + data->scriptStarts[data->scriptStartsLength - 1] == + (Collation::TRAIL_WEIGHT_BYTE << 8))) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + } else if(data == nullptr) { + // Nothing to do. + } else if(baseData != nullptr) { + data->numScripts = baseData->numScripts; + data->scriptsIndex = baseData->scriptsIndex; + data->scriptStarts = baseData->scriptStarts; + data->scriptStartsLength = baseData->scriptStartsLength; + } + + index = IX_COMPRESSIBLE_BYTES_OFFSET; + offset = getIndex(inIndexes, indexesLength, index); + length = getIndex(inIndexes, indexesLength, index + 1) - offset; + if(length >= 256) { + if(data == nullptr) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + data->compressibleBytes = reinterpret_cast(inBytes + offset); + } else if(data == nullptr) { + // Nothing to do. + } else if(baseData != nullptr) { + data->compressibleBytes = baseData->compressibleBytes; + } else { + errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[]. + return; + } + + const CollationSettings &ts = *tailoring.settings; + int32_t options = inIndexes[IX_OPTIONS] & 0xffff; + uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT]; + int32_t fastLatinOptions = CollationFastLatin::getOptions( + tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries)); + if(options == ts.options && ts.variableTop != 0 && + reorderCodesLength == ts.reorderCodesLength && + (reorderCodesLength == 0 || + uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) && + fastLatinOptions == ts.fastLatinOptions && + (fastLatinOptions < 0 || + uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries, + sizeof(fastLatinPrimaries)) == 0)) { + return; + } + + CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings); + if(settings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + settings->options = options; + // Set variableTop from options and scripts data. + settings->variableTop = tailoring.data->getLastPrimaryForGroup( + UCOL_REORDER_CODE_FIRST + int32_t{settings->getMaxVariable()}); + if(settings->variableTop == 0) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + + if(reorderCodesLength != 0) { + settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength, + reorderRanges, reorderRangesLength, + reorderTable, errorCode); + } + + settings->fastLatinOptions = CollationFastLatin::getOptions( + tailoring.data, *settings, + settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries)); +} + +UBool U_CALLCONV +CollationDataReader::isAcceptable(void *context, + const char * /* type */, const char * /*name*/, + const UDataInfo *pInfo) { + if( + pInfo->size >= 20 && + pInfo->isBigEndian == U_IS_BIG_ENDIAN && + pInfo->charsetFamily == U_CHARSET_FAMILY && + pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol" + pInfo->dataFormat[1] == 0x43 && + pInfo->dataFormat[2] == 0x6f && + pInfo->dataFormat[3] == 0x6c && + pInfo->formatVersion[0] == 5 + ) { + UVersionInfo *version = static_cast(context); + if(version != nullptr) { + uprv_memcpy(version, pInfo->dataVersion, 4); + } + return true; + } else { + return false; + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationdatareader.h b/intl/icu/source/i18n/collationdatareader.h new file mode 100644 index 0000000000..5030f6c852 --- /dev/null +++ b/intl/icu/source/i18n/collationdatareader.h @@ -0,0 +1,253 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatareader.h +* +* created on: 2013feb07 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONDATAREADER_H__ +#define __COLLATIONDATAREADER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/udata.h" + +struct UDataMemory; + +U_NAMESPACE_BEGIN + +struct CollationTailoring; + +/** + * Collation binary data reader. + */ +struct U_I18N_API CollationDataReader /* all static */ { + // The following constants are also copied into source/common/ucol_swp.cpp. + // Keep them in sync! + enum { + /** + * Number of int32_t indexes. + * + * Can be 2 if there are only options. + * Can be 7 or 8 if there are only options and a script reordering. + * The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0. + */ + IX_INDEXES_LENGTH, // 0 + /** + * Bits 31..24: numericPrimary, for numeric collation + * 23..16: fast Latin format version (0 = no fast Latin table) + * 15.. 0: options bit set + */ + IX_OPTIONS, + IX_RESERVED2, + IX_RESERVED3, + + /** Array offset to Jamo CE32s in ce32s[], or <0 if none. */ + IX_JAMO_CE32S_START, // 4 + + // Byte offsets from the start of the data, after the generic header. + // The indexes[] are at byte offset 0, other data follows. + // Each data item is aligned properly. + // The data items should be in descending order of unit size, + // to minimize the need for padding. + // Each item's byte length is given by the difference between its offset and + // the next index/offset value. + /** Byte offset to int32_t reorderCodes[]. */ + IX_REORDER_CODES_OFFSET, + /** + * Byte offset to uint8_t reorderTable[]. + * Empty table if <256 bytes (padding only). + * Otherwise 256 bytes or more (with padding). + */ + IX_REORDER_TABLE_OFFSET, + /** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */ + IX_TRIE_OFFSET, + + IX_RESERVED8_OFFSET, // 8 + /** Byte offset to int64_t ces[]. */ + IX_CES_OFFSET, + IX_RESERVED10_OFFSET, + /** Byte offset to uint32_t ce32s[]. */ + IX_CE32S_OFFSET, + + /** Byte offset to uint32_t rootElements[]. */ + IX_ROOT_ELEMENTS_OFFSET, // 12 + /** Byte offset to char16_t *contexts[]. */ + IX_CONTEXTS_OFFSET, + /** Byte offset to uint16_t [] with serialized unsafeBackwardSet. */ + IX_UNSAFE_BWD_OFFSET, + /** Byte offset to uint16_t fastLatinTable[]. */ + IX_FAST_LATIN_TABLE_OFFSET, + + /** Byte offset to uint16_t scripts[]. */ + IX_SCRIPTS_OFFSET, // 16 + /** + * Byte offset to UBool compressibleBytes[]. + * Empty table if <256 bytes (padding only). + * Otherwise 256 bytes or more (with padding). + */ + IX_COMPRESSIBLE_BYTES_OFFSET, + IX_RESERVED18_OFFSET, + IX_TOTAL_SIZE + }; + + static void read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength, + CollationTailoring &tailoring, UErrorCode &errorCode); + + static UBool U_CALLCONV + isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo); + +private: + CollationDataReader() = delete; // no constructor +}; + +/* + * Format of collation data (ucadata.icu, binary data in coll/ *.res files). + * Format version 5. + * + * The root collation data is stored in the ucadata.icu file. + * Tailorings are stored inside .res resource bundle files, with a complete file header. + * + * Collation data begins with a standard ICU data file header + * (DataHeader, see ucmndata.h and unicode/udata.h). + * The UDataInfo.dataVersion field contains the UCA and other version numbers, + * see the comments for CollationTailoring.version. + * + * After the header, the file contains the following parts. + * Constants are defined as enum values of the CollationDataReader class. + * See also the Collation class. + * + * int32_t indexes[indexesLength]; + * The indexes array has variable length. + * Some tailorings only need the length and the options, + * others only add reorderCodes and the reorderTable, + * some need to store mappings. + * Only as many indexes are stored as needed to read all of the data. + * + * Index 0: indexesLength + * Index 1: numericPrimary, CollationFastLatin::VERSION, and options: see IX_OPTIONS + * Index 2..3: Unused/reserved/0. + * Index 4: Index into the ce32s array where the CE32s of the conjoining Jamo + * are stored in a short, contiguous part of the ce32s array. + * + * Indexes 5..19 are byte offsets in ascending order. + * Each byte offset marks the start of the next part in the data file, + * and the end of the previous one. + * When two consecutive byte offsets are the same (or too short), + * then the corresponding part is empty. + * Byte offsets are offsets from after the header, + * that is, from the beginning of the indexes[]. + * Each part starts at an offset with proper alignment for its data. + * If necessary, the previous part may include padding bytes to achieve this alignment. + * The last byte offset that is stored in the indexes indicates the total size of the data + * (starting with the indexes). + * + * int32_t reorderCodes[]; -- empty in root + * The list of script and reordering codes. + * + * Beginning with format version 5, this array may optionally + * have trailing entries with a full list of reorder ranges + * as described for CollationSettings::reorderRanges. + * + * Script or reorder codes are first and do not exceed 16-bit values. + * Range limits are stored in the upper 16 bits, and are never 0. + * Split this array into reorder codes and ranges at the first entry + * with non-zero upper 16 bits. + * + * If the ranges are missing but needed for split-reordered primary lead bytes, + * then they are regenerated at load time. + * + * uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes + * Primary-weight lead byte permutation table. + * Normally present when the reorderCodes are, but can be built at load time. + * + * Beginning with format version 5, a 0 entry at a non-zero index + * (which is otherwise an illegal value) + * means that the primary lead byte is "split" + * (there are different offsets for primaries that share that lead byte) + * and the reordering offset must be determined via the reorder ranges + * that are either stored as part of the reorderCodes array + * or regenerated at load time. + * + * UTrie2 trie; -- see utrie2_impl.h and utrie2.h + * The trie holds the main collation data. Each code point is mapped to a 32-bit value. + * It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set, + * in which case it is a special CE32 and contains a 4-bit tag and further data. + * See the Collation class for details. + * + * The trie has a value for each lead surrogate code unit with some bits encoding + * collective properties of the 1024 supplementary characters whose UTF-16 form starts with + * the lead surrogate. See Collation::LEAD_SURROGATE_TAG.. + * + * int64_t ces[]; + * 64-bit CEs and expansions that cannot be stored in a more compact form. + * + * uint32_t ce32s[]; + * CE32s for expansions in compact form, and for characters whose trie values + * contain special data. + * + * uint32_t rootElements[]; -- empty in all tailorings + * Compact storage for all of the CEs that occur in the root collation. + * See the CollationRootElements class. + * + * char16_t *contexts[]; + * Serialized UCharsTrie structures with prefix (pre-context) and contraction mappings. + * + * uint16_t unsafeBackwardSet[]; -- see UnicodeSet::serialize() + * Serialized form of characters that are unsafe when iterating backwards, + * and at the end of an identical string prefix. + * Back up to a safe character. + * Lead surrogates are "unsafe" when any of their corresponding supplementary + * code points are unsafe. + * Does not include [:^lccc=0:][:^tccc=0:]. + * For each tailoring, the root unsafeBackwardSet is subtracted. + * (As a result, in many tailorings no set needs to be stored.) + * + * uint16_t fastLatinTable[]; + * Optional optimization for Latin text. + * See the CollationFastLatin class. + * + * uint16_t scripts[]; -- empty in all tailorings + * Format version 5: + * uint16_t numScripts; + * uint16_t scriptsIndex[numScripts+16]; + * uint16_t scriptStarts[]; + * See CollationData::numScripts etc. + * + * Format version 4: + * Table of the reordering groups with their first and last lead bytes, + * and their script and reordering codes. + * See CollationData::scripts. + * + * UBool compressibleBytes[]; -- empty in all tailorings + * Flag for getSortKey(), indicating primary weight lead bytes that are compressible. + * + * ----------------- + * Changes for formatVersion 5 (ICU 55) + * + * Reordering moves single scripts, not groups of scripts. + * Reorder ranges are optionally appended to the reorderCodes, + * and a 0 entry in the reorderTable indicates a split lead byte. + * The scripts data has a new format. + * + * The rootElements may contain secondary and tertiary weights below common=05. + * (Used for small Hiragana letters.) + * Where is occurs, there is also an explicit unit with common secondary & tertiary weights. + * There are no other data structure changes, but builder code needs to be able to handle such data. + * + * The collation element for the merge separator code point U+FFFE + * does not necessarily have special, unique secondary/tertiary weights any more. + */ + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONDATAREADER_H__ diff --git a/intl/icu/source/i18n/collationdatawriter.cpp b/intl/icu/source/i18n/collationdatawriter.cpp new file mode 100644 index 0000000000..ce78a0526a --- /dev/null +++ b/intl/icu/source/i18n/collationdatawriter.cpp @@ -0,0 +1,352 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatawriter.cpp +* +* created on: 2013aug06 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/tblcoll.h" +#include "unicode/udata.h" +#include "unicode/uniset.h" +#include "cmemory.h" +#include "collationdata.h" +#include "collationdatabuilder.h" +#include "collationdatareader.h" +#include "collationdatawriter.h" +#include "collationfastlatin.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "uassert.h" +#include "ucmndata.h" + +U_NAMESPACE_BEGIN + +uint8_t * +RuleBasedCollator::cloneRuleData(int32_t &length, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return nullptr; } + LocalMemory buffer((uint8_t *)uprv_malloc(20000)); + if(buffer.isNull()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + length = cloneBinary(buffer.getAlias(), 20000, errorCode); + if(errorCode == U_BUFFER_OVERFLOW_ERROR) { + if(buffer.allocateInsteadAndCopy(length, 0) == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + errorCode = U_ZERO_ERROR; + length = cloneBinary(buffer.getAlias(), length, errorCode); + } + if(U_FAILURE(errorCode)) { return nullptr; } + return buffer.orphan(); +} + +int32_t +RuleBasedCollator::cloneBinary(uint8_t *dest, int32_t capacity, UErrorCode &errorCode) const { + int32_t indexes[CollationDataReader::IX_TOTAL_SIZE + 1]; + return CollationDataWriter::writeTailoring( + *tailoring, *settings, indexes, dest, capacity, + errorCode); +} + +static const UDataInfo dataInfo = { + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + { 0x55, 0x43, 0x6f, 0x6c }, // dataFormat="UCol" + { 5, 0, 0, 0 }, // formatVersion + { 6, 3, 0, 0 } // dataVersion +}; + +int32_t +CollationDataWriter::writeBase(const CollationData &data, const CollationSettings &settings, + const void *rootElements, int32_t rootElementsLength, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode) { + return write(true, nullptr, + data, settings, + rootElements, rootElementsLength, + indexes, dest, capacity, errorCode); +} + +int32_t +CollationDataWriter::writeTailoring(const CollationTailoring &t, const CollationSettings &settings, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode) { + return write(false, t.version, + *t.data, settings, + nullptr, 0, + indexes, dest, capacity, errorCode); +} + +int32_t +CollationDataWriter::write(UBool isBase, const UVersionInfo dataVersion, + const CollationData &data, const CollationSettings &settings, + const void *rootElements, int32_t rootElementsLength, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + if(capacity < 0 || (capacity > 0 && dest == nullptr)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // Figure out which data items to write before settling on + // the indexes length and writing offsets. + // For any data item, we need to write the start and limit offsets, + // so the indexes length must be at least index-of-start-offset + 2. + int32_t indexesLength; + UBool hasMappings; + UnicodeSet unsafeBackwardSet; + const CollationData *baseData = data.base; + + int32_t fastLatinVersion; + if(data.fastLatinTable != nullptr) { + fastLatinVersion = (int32_t)CollationFastLatin::VERSION << 16; + } else { + fastLatinVersion = 0; + } + int32_t fastLatinTableLength = 0; + + if(isBase) { + // For the root collator, we write an even number of indexes + // so that we start with an 8-aligned offset. + indexesLength = CollationDataReader::IX_TOTAL_SIZE + 1; + U_ASSERT(settings.reorderCodesLength == 0); + hasMappings = true; + unsafeBackwardSet = *data.unsafeBackwardSet; + fastLatinTableLength = data.fastLatinTableLength; + } else if(baseData == nullptr) { + hasMappings = false; + if(settings.reorderCodesLength == 0) { + // only options + indexesLength = CollationDataReader::IX_OPTIONS + 1; // no limit offset here + } else { + // only options, reorder codes, and the reorder table + indexesLength = CollationDataReader::IX_REORDER_TABLE_OFFSET + 2; + } + } else { + hasMappings = true; + // Tailored mappings, and what else? + // Check in ascending order of optional tailoring data items. + indexesLength = CollationDataReader::IX_CE32S_OFFSET + 2; + if(data.contextsLength != 0) { + indexesLength = CollationDataReader::IX_CONTEXTS_OFFSET + 2; + } + unsafeBackwardSet.addAll(*data.unsafeBackwardSet).removeAll(*baseData->unsafeBackwardSet); + if(!unsafeBackwardSet.isEmpty()) { + indexesLength = CollationDataReader::IX_UNSAFE_BWD_OFFSET + 2; + } + if(data.fastLatinTable != baseData->fastLatinTable) { + fastLatinTableLength = data.fastLatinTableLength; + indexesLength = CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET + 2; + } + } + + UVector32 codesAndRanges(errorCode); + const int32_t *reorderCodes = settings.reorderCodes; + int32_t reorderCodesLength = settings.reorderCodesLength; + if(settings.hasReordering() && + CollationSettings::reorderTableHasSplitBytes(settings.reorderTable)) { + // Rebuild the full list of reorder ranges. + // The list in the settings is truncated for efficiency. + data.makeReorderRanges(reorderCodes, reorderCodesLength, codesAndRanges, errorCode); + // Write the codes, then the ranges. + for(int32_t i = 0; i < reorderCodesLength; ++i) { + codesAndRanges.insertElementAt(reorderCodes[i], i, errorCode); + } + if(U_FAILURE(errorCode)) { return 0; } + reorderCodes = codesAndRanges.getBuffer(); + reorderCodesLength = codesAndRanges.size(); + } + + int32_t headerSize; + if(isBase) { + headerSize = 0; // udata_create() writes the header + } else { + DataHeader header; + header.dataHeader.magic1 = 0xda; + header.dataHeader.magic2 = 0x27; + uprv_memcpy(&header.info, &dataInfo, sizeof(UDataInfo)); + uprv_memcpy(header.info.dataVersion, dataVersion, sizeof(UVersionInfo)); + headerSize = (int32_t)sizeof(header); + U_ASSERT((headerSize & 3) == 0); // multiple of 4 bytes + if(hasMappings && data.cesLength != 0) { + // Sum of the sizes of the data items which are + // not automatically multiples of 8 bytes and which are placed before the CEs. + int32_t sum = headerSize + (indexesLength + reorderCodesLength) * 4; + if((sum & 7) != 0) { + // We need to add padding somewhere so that the 64-bit CEs are 8-aligned. + // We add to the header size here. + // Alternatively, we could increment the indexesLength + // or add a few bytes to the reorderTable. + headerSize += 4; + } + } + header.dataHeader.headerSize = (uint16_t)headerSize; + if(headerSize <= capacity) { + uprv_memcpy(dest, &header, sizeof(header)); + // Write 00 bytes so that the padding is not mistaken for a copyright string. + uprv_memset(dest + sizeof(header), 0, headerSize - (int32_t)sizeof(header)); + dest += headerSize; + capacity -= headerSize; + } else { + dest = nullptr; + capacity = 0; + } + } + + indexes[CollationDataReader::IX_INDEXES_LENGTH] = indexesLength; + U_ASSERT((settings.options & ~0xffff) == 0); + indexes[CollationDataReader::IX_OPTIONS] = + data.numericPrimary | fastLatinVersion | settings.options; + indexes[CollationDataReader::IX_RESERVED2] = 0; + indexes[CollationDataReader::IX_RESERVED3] = 0; + + // Byte offsets of data items all start from the start of the indexes. + // We add the headerSize at the very end. + int32_t totalSize = indexesLength * 4; + + if(hasMappings && (isBase || data.jamoCE32s != baseData->jamoCE32s)) { + indexes[CollationDataReader::IX_JAMO_CE32S_START] = static_cast(data.jamoCE32s - data.ce32s); + } else { + indexes[CollationDataReader::IX_JAMO_CE32S_START] = -1; + } + + indexes[CollationDataReader::IX_REORDER_CODES_OFFSET] = totalSize; + totalSize += reorderCodesLength * 4; + + indexes[CollationDataReader::IX_REORDER_TABLE_OFFSET] = totalSize; + if(settings.reorderTable != nullptr) { + totalSize += 256; + } + + indexes[CollationDataReader::IX_TRIE_OFFSET] = totalSize; + if(hasMappings) { + UErrorCode errorCode2 = U_ZERO_ERROR; + int32_t length; + if(totalSize < capacity) { + length = utrie2_serialize(data.trie, dest + totalSize, + capacity - totalSize, &errorCode2); + } else { + length = utrie2_serialize(data.trie, nullptr, 0, &errorCode2); + } + if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) { + errorCode = errorCode2; + return 0; + } + // The trie size should be a multiple of 8 bytes due to the way + // compactIndex2(UNewTrie2 *trie) currently works. + U_ASSERT((length & 7) == 0); + totalSize += length; + } + + indexes[CollationDataReader::IX_RESERVED8_OFFSET] = totalSize; + indexes[CollationDataReader::IX_CES_OFFSET] = totalSize; + if(hasMappings && data.cesLength != 0) { + U_ASSERT(((headerSize + totalSize) & 7) == 0); + totalSize += data.cesLength * 8; + } + + indexes[CollationDataReader::IX_RESERVED10_OFFSET] = totalSize; + indexes[CollationDataReader::IX_CE32S_OFFSET] = totalSize; + if(hasMappings) { + totalSize += data.ce32sLength * 4; + } + + indexes[CollationDataReader::IX_ROOT_ELEMENTS_OFFSET] = totalSize; + totalSize += rootElementsLength * 4; + + indexes[CollationDataReader::IX_CONTEXTS_OFFSET] = totalSize; + if(hasMappings) { + totalSize += data.contextsLength * 2; + } + + indexes[CollationDataReader::IX_UNSAFE_BWD_OFFSET] = totalSize; + if(hasMappings && !unsafeBackwardSet.isEmpty()) { + UErrorCode errorCode2 = U_ZERO_ERROR; + int32_t length; + if(totalSize < capacity) { + uint16_t *p = reinterpret_cast(dest + totalSize); + length = unsafeBackwardSet.serialize( + p, (capacity - totalSize) / 2, errorCode2); + } else { + length = unsafeBackwardSet.serialize(nullptr, 0, errorCode2); + } + if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) { + errorCode = errorCode2; + return 0; + } + totalSize += length * 2; + } + + indexes[CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET] = totalSize; + totalSize += fastLatinTableLength * 2; + + UnicodeString scripts; + indexes[CollationDataReader::IX_SCRIPTS_OFFSET] = totalSize; + if(isBase) { + scripts.append((char16_t)data.numScripts); + scripts.append(reinterpret_cast(data.scriptsIndex), data.numScripts + 16); + scripts.append(reinterpret_cast(data.scriptStarts), data.scriptStartsLength); + totalSize += scripts.length() * 2; + } + + indexes[CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET] = totalSize; + if(isBase) { + totalSize += 256; + } + + indexes[CollationDataReader::IX_RESERVED18_OFFSET] = totalSize; + indexes[CollationDataReader::IX_TOTAL_SIZE] = totalSize; + + if(totalSize > capacity) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return headerSize + totalSize; + } + + uprv_memcpy(dest, indexes, indexesLength * 4); + copyData(indexes, CollationDataReader::IX_REORDER_CODES_OFFSET, reorderCodes, dest); + copyData(indexes, CollationDataReader::IX_REORDER_TABLE_OFFSET, settings.reorderTable, dest); + // The trie has already been serialized into the dest buffer. + copyData(indexes, CollationDataReader::IX_CES_OFFSET, data.ces, dest); + copyData(indexes, CollationDataReader::IX_CE32S_OFFSET, data.ce32s, dest); + copyData(indexes, CollationDataReader::IX_ROOT_ELEMENTS_OFFSET, rootElements, dest); + copyData(indexes, CollationDataReader::IX_CONTEXTS_OFFSET, data.contexts, dest); + // The unsafeBackwardSet has already been serialized into the dest buffer. + copyData(indexes, CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET, data.fastLatinTable, dest); + copyData(indexes, CollationDataReader::IX_SCRIPTS_OFFSET, scripts.getBuffer(), dest); + copyData(indexes, CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET, data.compressibleBytes, dest); + + return headerSize + totalSize; +} + +void +CollationDataWriter::copyData(const int32_t indexes[], int32_t startIndex, + const void *src, uint8_t *dest) { + int32_t start = indexes[startIndex]; + int32_t limit = indexes[startIndex + 1]; + if(start < limit) { + uprv_memcpy(dest + start, src, limit - start); + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationdatawriter.h b/intl/icu/source/i18n/collationdatawriter.h new file mode 100644 index 0000000000..6ba9a9c2c7 --- /dev/null +++ b/intl/icu/source/i18n/collationdatawriter.h @@ -0,0 +1,57 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationdatawriter.h +* +* created on: 2013aug06 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONDATAWRITER_H__ +#define __COLLATIONDATAWRITER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +U_NAMESPACE_BEGIN + +struct CollationData; +struct CollationSettings; +struct CollationTailoring; + +/** + * Collation-related code for tools & demos. + */ +class U_I18N_API CollationDataWriter /* all static */ { +public: + static int32_t writeBase(const CollationData &data, const CollationSettings &settings, + const void *rootElements, int32_t rootElementsLength, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode); + + static int32_t writeTailoring(const CollationTailoring &t, const CollationSettings &settings, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode); + +private: + CollationDataWriter() = delete; // no constructor + + static int32_t write(UBool isBase, const UVersionInfo dataVersion, + const CollationData &data, const CollationSettings &settings, + const void *rootElements, int32_t rootElementsLength, + int32_t indexes[], uint8_t *dest, int32_t capacity, + UErrorCode &errorCode); + + static void copyData(const int32_t indexes[], int32_t startIndex, + const void *src, uint8_t *dest); +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONDATAWRITER_H__ diff --git a/intl/icu/source/i18n/collationfastlatin.cpp b/intl/icu/source/i18n/collationfastlatin.cpp new file mode 100644 index 0000000000..f40781a117 --- /dev/null +++ b/intl/icu/source/i18n/collationfastlatin.cpp @@ -0,0 +1,1099 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationfastlatin.cpp +* +* created on: 2013aug18 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "collationdata.h" +#include "collationfastlatin.h" +#include "collationsettings.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +int32_t +CollationFastLatin::getOptions(const CollationData *data, const CollationSettings &settings, + uint16_t *primaries, int32_t capacity) { + const uint16_t *table = data->fastLatinTable; + if(table == nullptr) { return -1; } + U_ASSERT(capacity == LATIN_LIMIT); + if(capacity != LATIN_LIMIT) { return -1; } + + uint32_t miniVarTop; + if((settings.options & CollationSettings::ALTERNATE_MASK) == 0) { + // No mini primaries are variable, set a variableTop just below the + // lowest long mini primary. + miniVarTop = MIN_LONG - 1; + } else { + int32_t headerLength = *table & 0xff; + int32_t i = 1 + settings.getMaxVariable(); + if(i >= headerLength) { + return -1; // variableTop >= digits, should not occur + } + miniVarTop = table[i]; + } + + UBool digitsAreReordered = false; + if(settings.hasReordering()) { + uint32_t prevStart = 0; + uint32_t beforeDigitStart = 0; + uint32_t digitStart = 0; + uint32_t afterDigitStart = 0; + for(int32_t group = UCOL_REORDER_CODE_FIRST; + group < UCOL_REORDER_CODE_FIRST + CollationData::MAX_NUM_SPECIAL_REORDER_CODES; + ++group) { + uint32_t start = data->getFirstPrimaryForGroup(group); + start = settings.reorder(start); + if(group == UCOL_REORDER_CODE_DIGIT) { + beforeDigitStart = prevStart; + digitStart = start; + } else if(start != 0) { + if(start < prevStart) { + // The permutation affects the groups up to Latin. + return -1; + } + // In the future, there might be a special group between digits & Latin. + if(digitStart != 0 && afterDigitStart == 0 && prevStart == beforeDigitStart) { + afterDigitStart = start; + } + prevStart = start; + } + } + uint32_t latinStart = data->getFirstPrimaryForGroup(USCRIPT_LATIN); + latinStart = settings.reorder(latinStart); + if(latinStart < prevStart) { + return -1; + } + if(afterDigitStart == 0) { + afterDigitStart = latinStart; + } + if(!(beforeDigitStart < digitStart && digitStart < afterDigitStart)) { + digitsAreReordered = true; + } + } + + table += (table[0] & 0xff); // skip the header + for(UChar32 c = 0; c < LATIN_LIMIT; ++c) { + uint32_t p = table[c]; + if(p >= MIN_SHORT) { + p &= SHORT_PRIMARY_MASK; + } else if(p > miniVarTop) { + p &= LONG_PRIMARY_MASK; + } else { + p = 0; + } + primaries[c] = (uint16_t)p; + } + if(digitsAreReordered || (settings.options & CollationSettings::NUMERIC) != 0) { + // Bail out for digits. + for(UChar32 c = 0x30; c <= 0x39; ++c) { primaries[c] = 0; } + } + + // Shift the miniVarTop above other options. + return ((int32_t)miniVarTop << 16) | settings.options; +} + +int32_t +CollationFastLatin::compareUTF16(const uint16_t *table, const uint16_t *primaries, int32_t options, + const char16_t *left, int32_t leftLength, + const char16_t *right, int32_t rightLength) { + // This is a modified copy of CollationCompare::compareUpToQuaternary(), + // optimized for common Latin text. + // Keep them in sync! + // Keep compareUTF16() and compareUTF8() in sync very closely! + + U_ASSERT((table[0] >> 8) == VERSION); + table += (table[0] & 0xff); // skip the header + uint32_t variableTop = (uint32_t)options >> 16; // see getOptions() + options &= 0xffff; // needed for CollationSettings::getStrength() to work + + // Check for supported characters, fetch mini CEs, and compare primaries. + int32_t leftIndex = 0, rightIndex = 0; + /** + * Single mini CE or a pair. + * The current mini CE is in the lower 16 bits, the next one is in the upper 16 bits. + * If there is only one, then it is in the lower bits, and the upper bits are 0. + */ + uint32_t leftPair = 0, rightPair = 0; + for(;;) { + // We fetch CEs until we get a non-ignorable primary or reach the end. + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + if(c <= LATIN_MAX) { + leftPair = primaries[c]; + if(leftPair != 0) { break; } + if(c <= 0x39 && c >= 0x30 && (options & CollationSettings::NUMERIC) != 0) { + return BAIL_OUT_RESULT; + } + leftPair = table[c]; + } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { + leftPair = table[c - PUNCT_START + LATIN_LIMIT]; + } else { + leftPair = lookup(table, c); + } + if(leftPair >= MIN_SHORT) { + leftPair &= SHORT_PRIMARY_MASK; + break; + } else if(leftPair > variableTop) { + leftPair &= LONG_PRIMARY_MASK; + break; + } else { + leftPair = nextPair(table, c, leftPair, left, nullptr, leftIndex, leftLength); + if(leftPair == BAIL_OUT) { return BAIL_OUT_RESULT; } + leftPair = getPrimaries(variableTop, leftPair); + } + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + if(c <= LATIN_MAX) { + rightPair = primaries[c]; + if(rightPair != 0) { break; } + if(c <= 0x39 && c >= 0x30 && (options & CollationSettings::NUMERIC) != 0) { + return BAIL_OUT_RESULT; + } + rightPair = table[c]; + } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { + rightPair = table[c - PUNCT_START + LATIN_LIMIT]; + } else { + rightPair = lookup(table, c); + } + if(rightPair >= MIN_SHORT) { + rightPair &= SHORT_PRIMARY_MASK; + break; + } else if(rightPair > variableTop) { + rightPair &= LONG_PRIMARY_MASK; + break; + } else { + rightPair = nextPair(table, c, rightPair, right, nullptr, rightIndex, rightLength); + if(rightPair == BAIL_OUT) { return BAIL_OUT_RESULT; } + rightPair = getPrimaries(variableTop, rightPair); + } + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftPrimary = leftPair & 0xffff; + uint32_t rightPrimary = rightPair & 0xffff; + if(leftPrimary != rightPrimary) { + // Return the primary difference. + return (leftPrimary < rightPrimary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + // In the following, we need to re-fetch each character because we did not buffer the CEs, + // but we know that the string is well-formed and + // only contains supported characters and mappings. + + // We might skip the secondary level but continue with the case level + // which is turned on separately. + if(CollationSettings::getStrength(options) >= UCOL_SECONDARY) { + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + if(c <= LATIN_MAX) { + leftPair = table[c]; + } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { + leftPair = table[c - PUNCT_START + LATIN_LIMIT]; + } else { + leftPair = lookup(table, c); + } + if(leftPair >= MIN_SHORT) { + leftPair = getSecondariesFromOneShortCE(leftPair); + break; + } else if(leftPair > variableTop) { + leftPair = COMMON_SEC_PLUS_OFFSET; + break; + } else { + leftPair = nextPair(table, c, leftPair, left, nullptr, leftIndex, leftLength); + leftPair = getSecondaries(variableTop, leftPair); + } + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + if(c <= LATIN_MAX) { + rightPair = table[c]; + } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { + rightPair = table[c - PUNCT_START + LATIN_LIMIT]; + } else { + rightPair = lookup(table, c); + } + if(rightPair >= MIN_SHORT) { + rightPair = getSecondariesFromOneShortCE(rightPair); + break; + } else if(rightPair > variableTop) { + rightPair = COMMON_SEC_PLUS_OFFSET; + break; + } else { + rightPair = nextPair(table, c, rightPair, right, nullptr, rightIndex, rightLength); + rightPair = getSecondaries(variableTop, rightPair); + } + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftSecondary = leftPair & 0xffff; + uint32_t rightSecondary = rightPair & 0xffff; + if(leftSecondary != rightSecondary) { + if((options & CollationSettings::BACKWARD_SECONDARY) != 0) { + // Full support for backwards secondary requires backwards contraction matching + // and moving backwards between merge separators. + return BAIL_OUT_RESULT; + } + return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + } + + if((options & CollationSettings::CASE_LEVEL) != 0) { + UBool strengthIsPrimary = CollationSettings::getStrength(options) == UCOL_PRIMARY; + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, left, nullptr, leftIndex, leftLength); + } + leftPair = getCases(variableTop, strengthIsPrimary, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, right, nullptr, rightIndex, rightLength); + } + rightPair = getCases(variableTop, strengthIsPrimary, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftCase = leftPair & 0xffff; + uint32_t rightCase = rightPair & 0xffff; + if(leftCase != rightCase) { + if((options & CollationSettings::UPPER_FIRST) == 0) { + return (leftCase < rightCase) ? UCOL_LESS : UCOL_GREATER; + } else { + return (leftCase < rightCase) ? UCOL_GREATER : UCOL_LESS; + } + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + } + if(CollationSettings::getStrength(options) <= UCOL_SECONDARY) { return UCOL_EQUAL; } + + // Remove the case bits from the tertiary weight when caseLevel is on or caseFirst is off. + UBool withCaseBits = CollationSettings::isTertiaryWithCaseBits(options); + + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, left, nullptr, leftIndex, leftLength); + } + leftPair = getTertiaries(variableTop, withCaseBits, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, right, nullptr, rightIndex, rightLength); + } + rightPair = getTertiaries(variableTop, withCaseBits, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftTertiary = leftPair & 0xffff; + uint32_t rightTertiary = rightPair & 0xffff; + if(leftTertiary != rightTertiary) { + if(CollationSettings::sortsTertiaryUpperCaseFirst(options)) { + // Pass through EOS and MERGE_WEIGHT + // and keep real tertiary weights larger than the MERGE_WEIGHT. + // Tertiary CEs (secondary ignorables) are not supported in fast Latin. + if(leftTertiary > MERGE_WEIGHT) { + leftTertiary ^= CASE_MASK; + } + if(rightTertiary > MERGE_WEIGHT) { + rightTertiary ^= CASE_MASK; + } + } + return (leftTertiary < rightTertiary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + if(CollationSettings::getStrength(options) <= UCOL_TERTIARY) { return UCOL_EQUAL; } + + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, left, nullptr, leftIndex, leftLength); + } + leftPair = getQuaternaries(variableTop, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= LATIN_MAX) ? table[c] : lookup(table, c); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, right, nullptr, rightIndex, rightLength); + } + rightPair = getQuaternaries(variableTop, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftQuaternary = leftPair & 0xffff; + uint32_t rightQuaternary = rightPair & 0xffff; + if(leftQuaternary != rightQuaternary) { + return (leftQuaternary < rightQuaternary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + return UCOL_EQUAL; +} + +int32_t +CollationFastLatin::compareUTF8(const uint16_t *table, const uint16_t *primaries, int32_t options, + const uint8_t *left, int32_t leftLength, + const uint8_t *right, int32_t rightLength) { + // Keep compareUTF16() and compareUTF8() in sync very closely! + + U_ASSERT((table[0] >> 8) == VERSION); + table += (table[0] & 0xff); // skip the header + uint32_t variableTop = (uint32_t)options >> 16; // see RuleBasedCollator::getFastLatinOptions() + options &= 0xffff; // needed for CollationSettings::getStrength() to work + + // Check for supported characters, fetch mini CEs, and compare primaries. + int32_t leftIndex = 0, rightIndex = 0; + /** + * Single mini CE or a pair. + * The current mini CE is in the lower 16 bits, the next one is in the upper 16 bits. + * If there is only one, then it is in the lower bits, and the upper bits are 0. + */ + uint32_t leftPair = 0, rightPair = 0; + // Note: There is no need to assemble the code point. + // We only need to look up the table entry for the character, + // and nextPair() looks for whether c==0. + for(;;) { + // We fetch CEs until we get a non-ignorable primary or reach the end. + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + uint8_t t; + if(c <= 0x7f) { + leftPair = primaries[c]; + if(leftPair != 0) { break; } + if(c <= 0x39 && c >= 0x30 && (options & CollationSettings::NUMERIC) != 0) { + return BAIL_OUT_RESULT; + } + leftPair = table[c]; + } else if(c <= LATIN_MAX_UTF8_LEAD && 0xc2 <= c && leftIndex != leftLength && + 0x80 <= (t = left[leftIndex]) && t <= 0xbf) { + ++leftIndex; + c = ((c - 0xc2) << 6) + t; + leftPair = primaries[c]; + if(leftPair != 0) { break; } + leftPair = table[c]; + } else { + leftPair = lookupUTF8(table, c, left, leftIndex, leftLength); + } + if(leftPair >= MIN_SHORT) { + leftPair &= SHORT_PRIMARY_MASK; + break; + } else if(leftPair > variableTop) { + leftPair &= LONG_PRIMARY_MASK; + break; + } else { + leftPair = nextPair(table, c, leftPair, nullptr, left, leftIndex, leftLength); + if(leftPair == BAIL_OUT) { return BAIL_OUT_RESULT; } + leftPair = getPrimaries(variableTop, leftPair); + } + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + uint8_t t; + if(c <= 0x7f) { + rightPair = primaries[c]; + if(rightPair != 0) { break; } + if(c <= 0x39 && c >= 0x30 && (options & CollationSettings::NUMERIC) != 0) { + return BAIL_OUT_RESULT; + } + rightPair = table[c]; + } else if(c <= LATIN_MAX_UTF8_LEAD && 0xc2 <= c && rightIndex != rightLength && + 0x80 <= (t = right[rightIndex]) && t <= 0xbf) { + ++rightIndex; + c = ((c - 0xc2) << 6) + t; + rightPair = primaries[c]; + if(rightPair != 0) { break; } + rightPair = table[c]; + } else { + rightPair = lookupUTF8(table, c, right, rightIndex, rightLength); + } + if(rightPair >= MIN_SHORT) { + rightPair &= SHORT_PRIMARY_MASK; + break; + } else if(rightPair > variableTop) { + rightPair &= LONG_PRIMARY_MASK; + break; + } else { + rightPair = nextPair(table, c, rightPair, nullptr, right, rightIndex, rightLength); + if(rightPair == BAIL_OUT) { return BAIL_OUT_RESULT; } + rightPair = getPrimaries(variableTop, rightPair); + } + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftPrimary = leftPair & 0xffff; + uint32_t rightPrimary = rightPair & 0xffff; + if(leftPrimary != rightPrimary) { + // Return the primary difference. + return (leftPrimary < rightPrimary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + // In the following, we need to re-fetch each character because we did not buffer the CEs, + // but we know that the string is well-formed and + // only contains supported characters and mappings. + + // We might skip the secondary level but continue with the case level + // which is turned on separately. + if(CollationSettings::getStrength(options) >= UCOL_SECONDARY) { + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + if(c <= 0x7f) { + leftPair = table[c]; + } else if(c <= LATIN_MAX_UTF8_LEAD) { + leftPair = table[((c - 0xc2) << 6) + left[leftIndex++]]; + } else { + leftPair = lookupUTF8Unsafe(table, c, left, leftIndex); + } + if(leftPair >= MIN_SHORT) { + leftPair = getSecondariesFromOneShortCE(leftPair); + break; + } else if(leftPair > variableTop) { + leftPair = COMMON_SEC_PLUS_OFFSET; + break; + } else { + leftPair = nextPair(table, c, leftPair, nullptr, left, leftIndex, leftLength); + leftPair = getSecondaries(variableTop, leftPair); + } + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + if(c <= 0x7f) { + rightPair = table[c]; + } else if(c <= LATIN_MAX_UTF8_LEAD) { + rightPair = table[((c - 0xc2) << 6) + right[rightIndex++]]; + } else { + rightPair = lookupUTF8Unsafe(table, c, right, rightIndex); + } + if(rightPair >= MIN_SHORT) { + rightPair = getSecondariesFromOneShortCE(rightPair); + break; + } else if(rightPair > variableTop) { + rightPair = COMMON_SEC_PLUS_OFFSET; + break; + } else { + rightPair = nextPair(table, c, rightPair, nullptr, right, rightIndex, rightLength); + rightPair = getSecondaries(variableTop, rightPair); + } + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftSecondary = leftPair & 0xffff; + uint32_t rightSecondary = rightPair & 0xffff; + if(leftSecondary != rightSecondary) { + if((options & CollationSettings::BACKWARD_SECONDARY) != 0) { + // Full support for backwards secondary requires backwards contraction matching + // and moving backwards between merge separators. + return BAIL_OUT_RESULT; + } + return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + } + + if((options & CollationSettings::CASE_LEVEL) != 0) { + UBool strengthIsPrimary = CollationSettings::getStrength(options) == UCOL_PRIMARY; + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, left, leftIndex); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, nullptr, left, leftIndex, leftLength); + } + leftPair = getCases(variableTop, strengthIsPrimary, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, right, rightIndex); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, nullptr, right, rightIndex, rightLength); + } + rightPair = getCases(variableTop, strengthIsPrimary, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftCase = leftPair & 0xffff; + uint32_t rightCase = rightPair & 0xffff; + if(leftCase != rightCase) { + if((options & CollationSettings::UPPER_FIRST) == 0) { + return (leftCase < rightCase) ? UCOL_LESS : UCOL_GREATER; + } else { + return (leftCase < rightCase) ? UCOL_GREATER : UCOL_LESS; + } + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + } + if(CollationSettings::getStrength(options) <= UCOL_SECONDARY) { return UCOL_EQUAL; } + + // Remove the case bits from the tertiary weight when caseLevel is on or caseFirst is off. + UBool withCaseBits = CollationSettings::isTertiaryWithCaseBits(options); + + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, left, leftIndex); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, nullptr, left, leftIndex, leftLength); + } + leftPair = getTertiaries(variableTop, withCaseBits, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, right, rightIndex); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, nullptr, right, rightIndex, rightLength); + } + rightPair = getTertiaries(variableTop, withCaseBits, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftTertiary = leftPair & 0xffff; + uint32_t rightTertiary = rightPair & 0xffff; + if(leftTertiary != rightTertiary) { + if(CollationSettings::sortsTertiaryUpperCaseFirst(options)) { + // Pass through EOS and MERGE_WEIGHT + // and keep real tertiary weights larger than the MERGE_WEIGHT. + // Tertiary CEs (secondary ignorables) are not supported in fast Latin. + if(leftTertiary > MERGE_WEIGHT) { + leftTertiary ^= CASE_MASK; + } + if(rightTertiary > MERGE_WEIGHT) { + rightTertiary ^= CASE_MASK; + } + } + return (leftTertiary < rightTertiary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + if(CollationSettings::getStrength(options) <= UCOL_TERTIARY) { return UCOL_EQUAL; } + + leftIndex = rightIndex = 0; + leftPair = rightPair = 0; + for(;;) { + while(leftPair == 0) { + if(leftIndex == leftLength) { + leftPair = EOS; + break; + } + UChar32 c = left[leftIndex++]; + leftPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, left, leftIndex); + if(leftPair < MIN_LONG) { + leftPair = nextPair(table, c, leftPair, nullptr, left, leftIndex, leftLength); + } + leftPair = getQuaternaries(variableTop, leftPair); + } + + while(rightPair == 0) { + if(rightIndex == rightLength) { + rightPair = EOS; + break; + } + UChar32 c = right[rightIndex++]; + rightPair = (c <= 0x7f) ? table[c] : lookupUTF8Unsafe(table, c, right, rightIndex); + if(rightPair < MIN_LONG) { + rightPair = nextPair(table, c, rightPair, nullptr, right, rightIndex, rightLength); + } + rightPair = getQuaternaries(variableTop, rightPair); + } + + if(leftPair == rightPair) { + if(leftPair == EOS) { break; } + leftPair = rightPair = 0; + continue; + } + uint32_t leftQuaternary = leftPair & 0xffff; + uint32_t rightQuaternary = rightPair & 0xffff; + if(leftQuaternary != rightQuaternary) { + return (leftQuaternary < rightQuaternary) ? UCOL_LESS : UCOL_GREATER; + } + if(leftPair == EOS) { break; } + leftPair >>= 16; + rightPair >>= 16; + } + return UCOL_EQUAL; +} + +uint32_t +CollationFastLatin::lookup(const uint16_t *table, UChar32 c) { + U_ASSERT(c > LATIN_MAX); + if(PUNCT_START <= c && c < PUNCT_LIMIT) { + return table[c - PUNCT_START + LATIN_LIMIT]; + } else if(c == 0xfffe) { + return MERGE_WEIGHT; + } else if(c == 0xffff) { + return MAX_SHORT | COMMON_SEC | LOWER_CASE | COMMON_TER; + } else { + return BAIL_OUT; + } +} + +uint32_t +CollationFastLatin::lookupUTF8(const uint16_t *table, UChar32 c, + const uint8_t *s8, int32_t &sIndex, int32_t sLength) { + // The caller handled ASCII and valid/supported Latin. + U_ASSERT(c > 0x7f); + int32_t i2 = sIndex + 1; + if(i2 < sLength || sLength < 0) { + uint8_t t1 = s8[sIndex]; + uint8_t t2 = s8[i2]; + sIndex += 2; + if(c == 0xe2 && t1 == 0x80 && 0x80 <= t2 && t2 <= 0xbf) { + return table[(LATIN_LIMIT - 0x80) + t2]; // 2000..203F -> 0180..01BF + } else if(c == 0xef && t1 == 0xbf) { + if(t2 == 0xbe) { + return MERGE_WEIGHT; // U+FFFE + } else if(t2 == 0xbf) { + return MAX_SHORT | COMMON_SEC | LOWER_CASE | COMMON_TER; // U+FFFF + } + } + } + return BAIL_OUT; +} + +uint32_t +CollationFastLatin::lookupUTF8Unsafe(const uint16_t *table, UChar32 c, + const uint8_t *s8, int32_t &sIndex) { + // The caller handled ASCII. + // The string is well-formed and contains only supported characters. + U_ASSERT(c > 0x7f); + if(c <= LATIN_MAX_UTF8_LEAD) { + return table[((c - 0xc2) << 6) + s8[sIndex++]]; // 0080..017F + } + uint8_t t2 = s8[sIndex + 1]; + sIndex += 2; + if(c == 0xe2) { + return table[(LATIN_LIMIT - 0x80) + t2]; // 2000..203F -> 0180..01BF + } else if(t2 == 0xbe) { + return MERGE_WEIGHT; // U+FFFE + } else { + return MAX_SHORT | COMMON_SEC | LOWER_CASE | COMMON_TER; // U+FFFF + } +} + +uint32_t +CollationFastLatin::nextPair(const uint16_t *table, UChar32 c, uint32_t ce, + const char16_t *s16, const uint8_t *s8, int32_t &sIndex, int32_t &sLength) { + if(ce >= MIN_LONG || ce < CONTRACTION) { + return ce; // simple or special mini CE + } else if(ce >= EXPANSION) { + int32_t index = NUM_FAST_CHARS + (ce & INDEX_MASK); + return ((uint32_t)table[index + 1] << 16) | table[index]; + } else /* ce >= CONTRACTION */ { + if(c == 0 && sLength < 0) { + sLength = sIndex - 1; + return EOS; + } + // Contraction list: Default mapping followed by + // 0 or more single-character contraction suffix mappings. + int32_t index = NUM_FAST_CHARS + (ce & INDEX_MASK); + if(sIndex != sLength) { + // Read the next character. + int32_t c2; + int32_t nextIndex = sIndex; + if(s16 != nullptr) { + c2 = s16[nextIndex++]; + if(c2 > LATIN_MAX) { + if(PUNCT_START <= c2 && c2 < PUNCT_LIMIT) { + c2 = c2 - PUNCT_START + LATIN_LIMIT; // 2000..203F -> 0180..01BF + } else if(c2 == 0xfffe || c2 == 0xffff) { + c2 = -1; // U+FFFE & U+FFFF cannot occur in contractions. + } else { + return BAIL_OUT; + } + } + } else { + c2 = s8[nextIndex++]; + if(c2 > 0x7f) { + uint8_t t; + if(c2 <= 0xc5 && 0xc2 <= c2 && nextIndex != sLength && + 0x80 <= (t = s8[nextIndex]) && t <= 0xbf) { + c2 = ((c2 - 0xc2) << 6) + t; // 0080..017F + ++nextIndex; + } else { + int32_t i2 = nextIndex + 1; + if(i2 < sLength || sLength < 0) { + if(c2 == 0xe2 && s8[nextIndex] == 0x80 && + 0x80 <= (t = s8[i2]) && t <= 0xbf) { + c2 = (LATIN_LIMIT - 0x80) + t; // 2000..203F -> 0180..01BF + } else if(c2 == 0xef && s8[nextIndex] == 0xbf && + ((t = s8[i2]) == 0xbe || t == 0xbf)) { + c2 = -1; // U+FFFE & U+FFFF cannot occur in contractions. + } else { + return BAIL_OUT; + } + } else { + return BAIL_OUT; + } + nextIndex += 2; + } + } + } + if(c2 == 0 && sLength < 0) { + sLength = sIndex; + c2 = -1; + } + // Look for the next character in the contraction suffix list, + // which is in ascending order of single suffix characters. + int32_t i = index; + int32_t head = table[i]; // first skip the default mapping + int32_t x; + do { + i += head >> CONTR_LENGTH_SHIFT; + head = table[i]; + x = head & CONTR_CHAR_MASK; + } while(x < c2); + if(x == c2) { + index = i; + sIndex = nextIndex; + } + } + // Return the CE or CEs for the default or contraction mapping. + int32_t length = table[index] >> CONTR_LENGTH_SHIFT; + if(length == 1) { + return BAIL_OUT; + } + ce = table[index + 1]; + if(length == 2) { + return ce; + } else { + return ((uint32_t)table[index + 2] << 16) | ce; + } + } +} + +uint32_t +CollationFastLatin::getSecondaries(uint32_t variableTop, uint32_t pair) { + if(pair <= 0xffff) { + // one mini CE + if(pair >= MIN_SHORT) { + pair = getSecondariesFromOneShortCE(pair); + } else if(pair > variableTop) { + pair = COMMON_SEC_PLUS_OFFSET; + } else if(pair >= MIN_LONG) { + pair = 0; // variable + } + // else special mini CE + } else { + uint32_t ce = pair & 0xffff; + if(ce >= MIN_SHORT) { + pair = (pair & TWO_SECONDARIES_MASK) + TWO_SEC_OFFSETS; + } else if(ce > variableTop) { + pair = TWO_COMMON_SEC_PLUS_OFFSET; + } else { + U_ASSERT(ce >= MIN_LONG); + pair = 0; // variable + } + } + return pair; +} + +uint32_t +CollationFastLatin::getCases(uint32_t variableTop, UBool strengthIsPrimary, uint32_t pair) { + // Primary+caseLevel: Ignore case level weights of primary ignorables. + // Otherwise: Ignore case level weights of secondary ignorables. + // For details see the comments in the CollationCompare class. + // Tertiary CEs (secondary ignorables) are not supported in fast Latin. + if(pair <= 0xffff) { + // one mini CE + if(pair >= MIN_SHORT) { + // A high secondary weight means we really have two CEs, + // a primary CE and a secondary CE. + uint32_t ce = pair; + pair &= CASE_MASK; // explicit weight of primary CE + if(!strengthIsPrimary && (ce & SECONDARY_MASK) >= MIN_SEC_HIGH) { + pair |= LOWER_CASE << 16; // implied weight of secondary CE + } + } else if(pair > variableTop) { + pair = LOWER_CASE; + } else if(pair >= MIN_LONG) { + pair = 0; // variable + } + // else special mini CE + } else { + // two mini CEs, same primary groups, neither expands like above + uint32_t ce = pair & 0xffff; + if(ce >= MIN_SHORT) { + if(strengthIsPrimary && (pair & (SHORT_PRIMARY_MASK << 16)) == 0) { + pair &= CASE_MASK; + } else { + pair &= TWO_CASES_MASK; + } + } else if(ce > variableTop) { + pair = TWO_LOWER_CASES; + } else { + U_ASSERT(ce >= MIN_LONG); + pair = 0; // variable + } + } + return pair; +} + +uint32_t +CollationFastLatin::getTertiaries(uint32_t variableTop, UBool withCaseBits, uint32_t pair) { + if(pair <= 0xffff) { + // one mini CE + if(pair >= MIN_SHORT) { + // A high secondary weight means we really have two CEs, + // a primary CE and a secondary CE. + uint32_t ce = pair; + if(withCaseBits) { + pair = (pair & CASE_AND_TERTIARY_MASK) + TER_OFFSET; + if((ce & SECONDARY_MASK) >= MIN_SEC_HIGH) { + pair |= (LOWER_CASE | COMMON_TER_PLUS_OFFSET) << 16; + } + } else { + pair = (pair & TERTIARY_MASK) + TER_OFFSET; + if((ce & SECONDARY_MASK) >= MIN_SEC_HIGH) { + pair |= COMMON_TER_PLUS_OFFSET << 16; + } + } + } else if(pair > variableTop) { + pair = (pair & TERTIARY_MASK) + TER_OFFSET; + if(withCaseBits) { + pair |= LOWER_CASE; + } + } else if(pair >= MIN_LONG) { + pair = 0; // variable + } + // else special mini CE + } else { + // two mini CEs, same primary groups, neither expands like above + uint32_t ce = pair & 0xffff; + if(ce >= MIN_SHORT) { + if(withCaseBits) { + pair &= TWO_CASES_MASK | TWO_TERTIARIES_MASK; + } else { + pair &= TWO_TERTIARIES_MASK; + } + pair += TWO_TER_OFFSETS; + } else if(ce > variableTop) { + pair = (pair & TWO_TERTIARIES_MASK) + TWO_TER_OFFSETS; + if(withCaseBits) { + pair |= TWO_LOWER_CASES; + } + } else { + U_ASSERT(ce >= MIN_LONG); + pair = 0; // variable + } + } + return pair; +} + +uint32_t +CollationFastLatin::getQuaternaries(uint32_t variableTop, uint32_t pair) { + // Return the primary weight of a variable CE, + // or the maximum primary weight for a non-variable, not-completely-ignorable CE. + if(pair <= 0xffff) { + // one mini CE + if(pair >= MIN_SHORT) { + // A high secondary weight means we really have two CEs, + // a primary CE and a secondary CE. + if((pair & SECONDARY_MASK) >= MIN_SEC_HIGH) { + pair = TWO_SHORT_PRIMARIES_MASK; + } else { + pair = SHORT_PRIMARY_MASK; + } + } else if(pair > variableTop) { + pair = SHORT_PRIMARY_MASK; + } else if(pair >= MIN_LONG) { + pair &= LONG_PRIMARY_MASK; // variable + } + // else special mini CE + } else { + // two mini CEs, same primary groups, neither expands like above + uint32_t ce = pair & 0xffff; + if(ce > variableTop) { + pair = TWO_SHORT_PRIMARIES_MASK; + } else { + U_ASSERT(ce >= MIN_LONG); + pair &= TWO_LONG_PRIMARIES_MASK; // variable + } + } + return pair; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationfastlatin.h b/intl/icu/source/i18n/collationfastlatin.h new file mode 100644 index 0000000000..d4caddcb63 --- /dev/null +++ b/intl/icu/source/i18n/collationfastlatin.h @@ -0,0 +1,319 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationfastlatin.h +* +* created on: 2013aug09 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONFASTLATIN_H__ +#define __COLLATIONFASTLATIN_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +U_NAMESPACE_BEGIN + +struct CollationData; +struct CollationSettings; + +class U_I18N_API CollationFastLatin /* all static */ { +public: + /** + * Fast Latin format version (one byte 1..FF). + * Must be incremented for any runtime-incompatible changes, + * in particular, for changes to any of the following constants. + * + * When the major version number of the main data format changes, + * we can reset this fast Latin version to 1. + */ + static const uint16_t VERSION = 2; + + static const int32_t LATIN_MAX = 0x17f; + static const int32_t LATIN_LIMIT = LATIN_MAX + 1; + + static const int32_t LATIN_MAX_UTF8_LEAD = 0xc5; // UTF-8 lead byte of LATIN_MAX + + static const int32_t PUNCT_START = 0x2000; + static const int32_t PUNCT_LIMIT = 0x2040; + + // excludes U+FFFE & U+FFFF + static const int32_t NUM_FAST_CHARS = LATIN_LIMIT + (PUNCT_LIMIT - PUNCT_START); + + // Note on the supported weight ranges: + // Analysis of UCA 6.3 and CLDR 23 non-search tailorings shows that + // the CEs for characters in the above ranges, excluding expansions with length >2, + // excluding contractions of >2 characters, and other restrictions + // (see the builder's getCEsFromCE32()), + // use at most about 150 primary weights, + // where about 94 primary weights are possibly-variable (space/punct/symbol/currency), + // at most 4 secondary before-common weights, + // at most 4 secondary after-common weights, + // at most 16 secondary high weights (in secondary CEs), and + // at most 4 tertiary after-common weights. + // The following ranges are designed to support slightly more weights than that. + // (en_US_POSIX is unusual: It creates about 64 variable + 116 Latin primaries.) + + // Digits may use long primaries (preserving more short ones) + // or short primaries (faster) without changing this data structure. + // (If we supported numeric collation, then digits would have to have long primaries + // so that special handling does not affect the fast path.) + + static const uint32_t SHORT_PRIMARY_MASK = 0xfc00; // bits 15..10 + static const uint32_t INDEX_MASK = 0x3ff; // bits 9..0 for expansions & contractions + static const uint32_t SECONDARY_MASK = 0x3e0; // bits 9..5 + static const uint32_t CASE_MASK = 0x18; // bits 4..3 + static const uint32_t LONG_PRIMARY_MASK = 0xfff8; // bits 15..3 + static const uint32_t TERTIARY_MASK = 7; // bits 2..0 + static const uint32_t CASE_AND_TERTIARY_MASK = CASE_MASK | TERTIARY_MASK; + + static const uint32_t TWO_SHORT_PRIMARIES_MASK = + (SHORT_PRIMARY_MASK << 16) | SHORT_PRIMARY_MASK; // 0xfc00fc00 + static const uint32_t TWO_LONG_PRIMARIES_MASK = + (LONG_PRIMARY_MASK << 16) | LONG_PRIMARY_MASK; // 0xfff8fff8 + static const uint32_t TWO_SECONDARIES_MASK = + (SECONDARY_MASK << 16) | SECONDARY_MASK; // 0x3e003e0 + static const uint32_t TWO_CASES_MASK = + (CASE_MASK << 16) | CASE_MASK; // 0x180018 + static const uint32_t TWO_TERTIARIES_MASK = + (TERTIARY_MASK << 16) | TERTIARY_MASK; // 0x70007 + + /** + * Contraction with one fast Latin character. + * Use INDEX_MASK to find the start of the contraction list after the fixed table. + * The first entry contains the default mapping. + * Otherwise use CONTR_CHAR_MASK for the contraction character index + * (in ascending order). + * Use CONTR_LENGTH_SHIFT for the length of the entry + * (1=BAIL_OUT, 2=one CE, 3=two CEs). + * + * Also, U+0000 maps to a contraction entry, so that the fast path need not + * check for NUL termination. + * It usually maps to a contraction list with only the completely ignorable default value. + */ + static const uint32_t CONTRACTION = 0x400; + /** + * An expansion encodes two CEs. + * Use INDEX_MASK to find the pair of CEs after the fixed table. + * + * The higher a mini CE value, the easier it is to process. + * For expansions and higher, no context needs to be considered. + */ + static const uint32_t EXPANSION = 0x800; + /** + * Encodes one CE with a long/low mini primary (there are 128). + * All potentially-variable primaries must be in this range, + * to make the short-primary path as fast as possible. + */ + static const uint32_t MIN_LONG = 0xc00; + static const uint32_t LONG_INC = 8; + static const uint32_t MAX_LONG = 0xff8; + /** + * Encodes one CE with a short/high primary (there are 60), + * plus a secondary CE if the secondary weight is high. + * Fast handling: At least all letter primaries should be in this range. + */ + static const uint32_t MIN_SHORT = 0x1000; + static const uint32_t SHORT_INC = 0x400; + /** The highest primary weight is reserved for U+FFFF. */ + static const uint32_t MAX_SHORT = SHORT_PRIMARY_MASK; + + static const uint32_t MIN_SEC_BEFORE = 0; // must add SEC_OFFSET + static const uint32_t SEC_INC = 0x20; + static const uint32_t MAX_SEC_BEFORE = MIN_SEC_BEFORE + 4 * SEC_INC; // 5 before common + static const uint32_t COMMON_SEC = MAX_SEC_BEFORE + SEC_INC; + static const uint32_t MIN_SEC_AFTER = COMMON_SEC + SEC_INC; + static const uint32_t MAX_SEC_AFTER = MIN_SEC_AFTER + 5 * SEC_INC; // 6 after common + static const uint32_t MIN_SEC_HIGH = MAX_SEC_AFTER + SEC_INC; // 20 high secondaries + static const uint32_t MAX_SEC_HIGH = SECONDARY_MASK; + + /** + * Lookup: Add this offset to secondary weights, except for completely ignorable CEs. + * Must be greater than any special value, e.g., MERGE_WEIGHT. + * The exact value is not relevant for the format version. + */ + static const uint32_t SEC_OFFSET = SEC_INC; + static const uint32_t COMMON_SEC_PLUS_OFFSET = COMMON_SEC + SEC_OFFSET; + + static const uint32_t TWO_SEC_OFFSETS = + (SEC_OFFSET << 16) | SEC_OFFSET; // 0x200020 + static const uint32_t TWO_COMMON_SEC_PLUS_OFFSET = + (COMMON_SEC_PLUS_OFFSET << 16) | COMMON_SEC_PLUS_OFFSET; + + static const uint32_t LOWER_CASE = 8; // case bits include this offset + static const uint32_t TWO_LOWER_CASES = (LOWER_CASE << 16) | LOWER_CASE; // 0x80008 + + static const uint32_t COMMON_TER = 0; // must add TER_OFFSET + static const uint32_t MAX_TER_AFTER = 7; // 7 after common + + /** + * Lookup: Add this offset to tertiary weights, except for completely ignorable CEs. + * Must be greater than any special value, e.g., MERGE_WEIGHT. + * Must be greater than case bits as well, so that with combined case+tertiary weights + * plus the offset the tertiary bits does not spill over into the case bits. + * The exact value is not relevant for the format version. + */ + static const uint32_t TER_OFFSET = SEC_OFFSET; + static const uint32_t COMMON_TER_PLUS_OFFSET = COMMON_TER + TER_OFFSET; + + static const uint32_t TWO_TER_OFFSETS = (TER_OFFSET << 16) | TER_OFFSET; + static const uint32_t TWO_COMMON_TER_PLUS_OFFSET = + (COMMON_TER_PLUS_OFFSET << 16) | COMMON_TER_PLUS_OFFSET; + + static const uint32_t MERGE_WEIGHT = 3; + static const uint32_t EOS = 2; // end of string + static const uint32_t BAIL_OUT = 1; + + /** + * Contraction result first word bits 8..0 contain the + * second contraction character, as a char index 0..NUM_FAST_CHARS-1. + * Each contraction list is terminated with a word containing CONTR_CHAR_MASK. + */ + static const uint32_t CONTR_CHAR_MASK = 0x1ff; + /** + * Contraction result first word bits 10..9 contain the result length: + * 1=bail out, 2=one mini CE, 3=two mini CEs + */ + static const uint32_t CONTR_LENGTH_SHIFT = 9; + + /** + * Comparison return value when the regular comparison must be used. + * The exact value is not relevant for the format version. + */ + static const int32_t BAIL_OUT_RESULT = -2; + + static inline int32_t getCharIndex(char16_t c) { + if(c <= LATIN_MAX) { + return c; + } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { + return c - (PUNCT_START - LATIN_LIMIT); + } else { + // Not a fast Latin character. + // Note: U+FFFE & U+FFFF are forbidden in tailorings + // and thus do not occur in any contractions. + return -1; + } + } + + /** + * Computes the options value for the compare functions + * and writes the precomputed primary weights. + * Returns -1 if the Latin fastpath is not supported for the data and settings. + * The capacity must be LATIN_LIMIT. + */ + static int32_t getOptions(const CollationData *data, const CollationSettings &settings, + uint16_t *primaries, int32_t capacity); + + static int32_t compareUTF16(const uint16_t *table, const uint16_t *primaries, int32_t options, + const char16_t *left, int32_t leftLength, + const char16_t *right, int32_t rightLength); + + static int32_t compareUTF8(const uint16_t *table, const uint16_t *primaries, int32_t options, + const uint8_t *left, int32_t leftLength, + const uint8_t *right, int32_t rightLength); + +private: + static uint32_t lookup(const uint16_t *table, UChar32 c); + static uint32_t lookupUTF8(const uint16_t *table, UChar32 c, + const uint8_t *s8, int32_t &sIndex, int32_t sLength); + static uint32_t lookupUTF8Unsafe(const uint16_t *table, UChar32 c, + const uint8_t *s8, int32_t &sIndex); + + static uint32_t nextPair(const uint16_t *table, UChar32 c, uint32_t ce, + const char16_t *s16, const uint8_t *s8, int32_t &sIndex, int32_t &sLength); + + static inline uint32_t getPrimaries(uint32_t variableTop, uint32_t pair) { + uint32_t ce = pair & 0xffff; + if(ce >= MIN_SHORT) { return pair & TWO_SHORT_PRIMARIES_MASK; } + if(ce > variableTop) { return pair & TWO_LONG_PRIMARIES_MASK; } + if(ce >= MIN_LONG) { return 0; } // variable + return pair; // special mini CE + } + static inline uint32_t getSecondariesFromOneShortCE(uint32_t ce) { + ce &= SECONDARY_MASK; + if(ce < MIN_SEC_HIGH) { + return ce + SEC_OFFSET; + } else { + return ((ce + SEC_OFFSET) << 16) | COMMON_SEC_PLUS_OFFSET; + } + } + static uint32_t getSecondaries(uint32_t variableTop, uint32_t pair); + static uint32_t getCases(uint32_t variableTop, UBool strengthIsPrimary, uint32_t pair); + static uint32_t getTertiaries(uint32_t variableTop, UBool withCaseBits, uint32_t pair); + static uint32_t getQuaternaries(uint32_t variableTop, uint32_t pair); + +private: + CollationFastLatin() = delete; // no constructor +}; + +/* + * Format of the CollationFastLatin data table. + * CollationFastLatin::VERSION = 2. + * + * This table contains data for a Latin-text collation fastpath. + * The data is stored as an array of uint16_t which contains the following parts. + * + * uint16_t -- version & header length + * Bits 15..8: version, must match the VERSION + * 7..0: length of the header + * + * uint16_t varTops[header length - 1] + * Version 2: + * varTops[m] is the highest CollationFastLatin long-primary weight + * of supported maxVariable group m + * (special reorder group space, punct, symbol, currency). + * + * Version 1: + * Each of these values maps the variable top lead byte of a supported maxVariable group + * to the highest CollationFastLatin long-primary weight. + * The values are stored in ascending order. + * Bits 15..7: max fast-Latin long-primary weight (bits 11..3 shifted left by 4 bits) + * 6..0: regular primary lead byte + * + * uint16_t miniCEs[0x1c0] + * A mini collation element for each character U+0000..U+017F and U+2000..U+203F. + * Each value encodes one or two mini CEs (two are possible if the first one + * has a short mini primary and the second one is a secondary CE, i.e., primary == 0), + * or points to an expansion or to a contraction table. + * U+0000 always has a contraction entry, + * so that NUL-termination need not be tested in the fastpath. + * If the collation elements for a character or contraction cannot be encoded in this format, + * then the BAIL_OUT value is stored. + * For details see the comments for the class constants. + * + * uint16_t expansions[variable length]; + * Expansion mini CEs contain an offset relative to just after the miniCEs table. + * An expansions contains exactly 2 mini CEs. + * + * uint16_t contractions[variable length]; + * Contraction mini CEs contain an offset relative to just after the miniCEs table. + * It points to a list of tuples which map from a contraction suffix character to a result. + * First uint16_t of each tuple: + * Bits 10..9: Length of the result (1..3), see comments on CONTR_LENGTH_SHIFT. + * Bits 8..0: Contraction character, see comments on CONTR_CHAR_MASK. + * This is followed by 0, 1, or 2 uint16_t according to the length. + * Each list is terminated by an entry with CONTR_CHAR_MASK. + * Each list starts with such an entry which also contains the default result + * for when there is no contraction match. + * + * ----------------- + * Changes for version 2 (ICU 55) + * + * Special reorder groups do not necessarily start on whole primary lead bytes any more. + * Therefore, the varTops data has a new format: + * Version 1 stored the lead bytes of the highest root primaries for + * the maxVariable-supported special reorder groups. + * Now the top 16 bits would need to be stored, + * and it is simpler to store only the fast-Latin weights. + */ + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONFASTLATIN_H__ diff --git a/intl/icu/source/i18n/collationfastlatinbuilder.cpp b/intl/icu/source/i18n/collationfastlatinbuilder.cpp new file mode 100644 index 0000000000..b1fd3af70a --- /dev/null +++ b/intl/icu/source/i18n/collationfastlatinbuilder.cpp @@ -0,0 +1,717 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationfastlatinbuilder.cpp +* +* created on: 2013aug09 +* created by: Markus W. Scherer +*/ + +#define DEBUG_COLLATION_FAST_LATIN_BUILDER 0 // 0 or 1 or 2 +#if DEBUG_COLLATION_FAST_LATIN_BUILDER +#include +#include +#endif + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/ucharstrie.h" +#include "unicode/unistr.h" +#include "unicode/uobject.h" +#include "unicode/uscript.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationfastlatin.h" +#include "collationfastlatinbuilder.h" +#include "uassert.h" +#include "uvectr64.h" + +U_NAMESPACE_BEGIN + +struct CollationData; + +namespace { + +/** + * Compare two signed int64_t values as if they were unsigned. + */ +int32_t +compareInt64AsUnsigned(int64_t a, int64_t b) { + if((uint64_t)a < (uint64_t)b) { + return -1; + } else if((uint64_t)a > (uint64_t)b) { + return 1; + } else { + return 0; + } +} + +// TODO: Merge this with the near-identical version in collationbasedatabuilder.cpp +/** + * Like Java Collections.binarySearch(List, String, Comparator). + * + * @return the index>=0 where the item was found, + * or the index<0 for inserting the string at ~index in sorted order + */ +int32_t +binarySearch(const int64_t list[], int32_t limit, int64_t ce) { + if (limit == 0) { return ~0; } + int32_t start = 0; + for (;;) { + int32_t i = (start + limit) / 2; + int32_t cmp = compareInt64AsUnsigned(ce, list[i]); + if (cmp == 0) { + return i; + } else if (cmp < 0) { + if (i == start) { + return ~start; // insert ce before i + } + limit = i; + } else { + if (i == start) { + return ~(start + 1); // insert ce after i + } + start = i; + } + } +} + +} // namespace + +CollationFastLatinBuilder::CollationFastLatinBuilder(UErrorCode &errorCode) + : ce0(0), ce1(0), + contractionCEs(errorCode), uniqueCEs(errorCode), + miniCEs(nullptr), + firstDigitPrimary(0), firstLatinPrimary(0), lastLatinPrimary(0), + firstShortPrimary(0), shortPrimaryOverflow(false), + headerLength(0) { +} + +CollationFastLatinBuilder::~CollationFastLatinBuilder() { + uprv_free(miniCEs); +} + +UBool +CollationFastLatinBuilder::forData(const CollationData &data, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + if(!result.isEmpty()) { // This builder is not reusable. + errorCode = U_INVALID_STATE_ERROR; + return false; + } + if(!loadGroups(data, errorCode)) { return false; } + + // Fast handling of digits. + firstShortPrimary = firstDigitPrimary; + getCEs(data, errorCode); + if(!encodeUniqueCEs(errorCode)) { return false; } + if(shortPrimaryOverflow) { + // Give digits long mini primaries, + // so that there are more short primaries for letters. + firstShortPrimary = firstLatinPrimary; + resetCEs(); + getCEs(data, errorCode); + if(!encodeUniqueCEs(errorCode)) { return false; } + } + // Note: If we still have a short-primary overflow but not a long-primary overflow, + // then we could calculate how many more long primaries would fit, + // and set the firstShortPrimary to that many after the current firstShortPrimary, + // and try again. + // However, this might only benefit the en_US_POSIX tailoring, + // and it is simpler to suppress building fast Latin data for it in genrb, + // or by returning false here if shortPrimaryOverflow. + + UBool ok = !shortPrimaryOverflow && + encodeCharCEs(errorCode) && encodeContractions(errorCode); + contractionCEs.removeAllElements(); // might reduce heap memory usage + uniqueCEs.removeAllElements(); + return ok; +} + +UBool +CollationFastLatinBuilder::loadGroups(const CollationData &data, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + headerLength = 1 + NUM_SPECIAL_GROUPS; + uint32_t r0 = (CollationFastLatin::VERSION << 8) | headerLength; + result.append((char16_t)r0); + // The first few reordering groups should be special groups + // (space, punct, ..., digit) followed by Latn, then Grek and other scripts. + for(int32_t i = 0; i < NUM_SPECIAL_GROUPS; ++i) { + lastSpecialPrimaries[i] = data.getLastPrimaryForGroup(UCOL_REORDER_CODE_FIRST + i); + if(lastSpecialPrimaries[i] == 0) { + // missing data + return false; + } + result.append((char16_t)0); // reserve a slot for this group + } + + firstDigitPrimary = data.getFirstPrimaryForGroup(UCOL_REORDER_CODE_DIGIT); + firstLatinPrimary = data.getFirstPrimaryForGroup(USCRIPT_LATIN); + lastLatinPrimary = data.getLastPrimaryForGroup(USCRIPT_LATIN); + if(firstDigitPrimary == 0 || firstLatinPrimary == 0) { + // missing data + return false; + } + return true; +} + +UBool +CollationFastLatinBuilder::inSameGroup(uint32_t p, uint32_t q) const { + // Both or neither need to be encoded as short primaries, + // so that we can test only one and use the same bit mask. + if(p >= firstShortPrimary) { + return q >= firstShortPrimary; + } else if(q >= firstShortPrimary) { + return false; + } + // Both or neither must be potentially-variable, + // so that we can test only one and determine if both are variable. + uint32_t lastVariablePrimary = lastSpecialPrimaries[NUM_SPECIAL_GROUPS - 1]; + if(p > lastVariablePrimary) { + return q > lastVariablePrimary; + } else if(q > lastVariablePrimary) { + return false; + } + // Both will be encoded with long mini primaries. + // They must be in the same special reordering group, + // so that we can test only one and determine if both are variable. + U_ASSERT(p != 0 && q != 0); + for(int32_t i = 0;; ++i) { // will terminate + uint32_t lastPrimary = lastSpecialPrimaries[i]; + if(p <= lastPrimary) { + return q <= lastPrimary; + } else if(q <= lastPrimary) { + return false; + } + } +} + +void +CollationFastLatinBuilder::resetCEs() { + contractionCEs.removeAllElements(); + uniqueCEs.removeAllElements(); + shortPrimaryOverflow = false; + result.truncate(headerLength); +} + +void +CollationFastLatinBuilder::getCEs(const CollationData &data, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t i = 0; + for(char16_t c = 0;; ++i, ++c) { + if(c == CollationFastLatin::LATIN_LIMIT) { + c = CollationFastLatin::PUNCT_START; + } else if(c == CollationFastLatin::PUNCT_LIMIT) { + break; + } + const CollationData *d; + uint32_t ce32 = data.getCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + d = data.base; + ce32 = d->getCE32(c); + } else { + d = &data; + } + if(getCEsFromCE32(*d, c, ce32, errorCode)) { + charCEs[i][0] = ce0; + charCEs[i][1] = ce1; + addUniqueCE(ce0, errorCode); + addUniqueCE(ce1, errorCode); + } else { + // bail out for c + charCEs[i][0] = ce0 = Collation::NO_CE; + charCEs[i][1] = ce1 = 0; + } + if(c == 0 && !isContractionCharCE(ce0)) { + // Always map U+0000 to a contraction. + // Write a contraction list with only a default value if there is no real contraction. + U_ASSERT(contractionCEs.isEmpty()); + addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); + charCEs[0][0] = ((int64_t)Collation::NO_CE_PRIMARY << 32) | CONTRACTION_FLAG; + charCEs[0][1] = 0; + } + } + // Terminate the last contraction list. + contractionCEs.addElement(CollationFastLatin::CONTR_CHAR_MASK, errorCode); +} + +UBool +CollationFastLatinBuilder::getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + ce32 = data.getFinalCE32(ce32); + ce1 = 0; + if(Collation::isSimpleOrLongCE32(ce32)) { + ce0 = Collation::ceFromCE32(ce32); + } else { + switch(Collation::tagFromCE32(ce32)) { + case Collation::LATIN_EXPANSION_TAG: + ce0 = Collation::latinCE0FromCE32(ce32); + ce1 = Collation::latinCE1FromCE32(ce32); + break; + case Collation::EXPANSION32_TAG: { + const uint32_t *ce32s = data.ce32s + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + if(length <= 2) { + ce0 = Collation::ceFromCE32(ce32s[0]); + if(length == 2) { + ce1 = Collation::ceFromCE32(ce32s[1]); + } + break; + } else { + return false; + } + } + case Collation::EXPANSION_TAG: { + const int64_t *ces = data.ces + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + if(length <= 2) { + ce0 = ces[0]; + if(length == 2) { + ce1 = ces[1]; + } + break; + } else { + return false; + } + } + // Note: We could support PREFIX_TAG (assert c>=0) + // by recursing on its default CE32 and checking that none of the prefixes starts + // with a fast Latin character. + // However, currently (2013) there are only the L-before-middle-dot + // prefix mappings in the Latin range, and those would be rejected anyway. + case Collation::CONTRACTION_TAG: + U_ASSERT(c >= 0); + return getCEsFromContractionCE32(data, ce32, errorCode); + case Collation::OFFSET_TAG: + U_ASSERT(c >= 0); + ce0 = data.getCEFromOffsetCE32(c, ce32); + break; + default: + return false; + } + } + // A mapping can be completely ignorable. + if(ce0 == 0) { return ce1 == 0; } + // We do not support an ignorable ce0 unless it is completely ignorable. + uint32_t p0 = (uint32_t)(ce0 >> 32); + if(p0 == 0) { return false; } + // We only support primaries up to the Latin script. + if(p0 > lastLatinPrimary) { return false; } + // We support non-common secondary and case weights only together with short primaries. + uint32_t lower32_0 = (uint32_t)ce0; + if(p0 < firstShortPrimary) { + uint32_t sc0 = lower32_0 & Collation::SECONDARY_AND_CASE_MASK; + if(sc0 != Collation::COMMON_SECONDARY_CE) { return false; } + } + // No below-common tertiary weights. + if((lower32_0 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return false; } + if(ce1 != 0) { + // Both primaries must be in the same group, + // or both must get short mini primaries, + // or a short-primary CE is followed by a secondary CE. + // This is so that we can test the first primary and use the same mask for both, + // and determine for both whether they are variable. + uint32_t p1 = (uint32_t)(ce1 >> 32); + if(p1 == 0 ? p0 < firstShortPrimary : !inSameGroup(p0, p1)) { return false; } + uint32_t lower32_1 = (uint32_t)ce1; + // No tertiary CEs. + if((lower32_1 >> 16) == 0) { return false; } + // We support non-common secondary and case weights + // only for secondary CEs or together with short primaries. + if(p1 != 0 && p1 < firstShortPrimary) { + uint32_t sc1 = lower32_1 & Collation::SECONDARY_AND_CASE_MASK; + if(sc1 != Collation::COMMON_SECONDARY_CE) { return false; } + } + // No below-common tertiary weights. + if((lower32_1 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return false; } + } + // No quaternary weights. + if(((ce0 | ce1) & Collation::QUATERNARY_MASK) != 0) { return false; } + return true; +} + +UBool +CollationFastLatinBuilder::getCEsFromContractionCE32(const CollationData &data, uint32_t ce32, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + const char16_t *p = data.contexts + Collation::indexFromCE32(ce32); + ce32 = CollationData::readCE32(p); // Default if no suffix match. + // Since the original ce32 is not a prefix mapping, + // the default ce32 must not be another contraction. + U_ASSERT(!Collation::isContractionCE32(ce32)); + int32_t contractionIndex = contractionCEs.size(); + if(getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { + addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); + } else { + // Bail out for c-without-contraction. + addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, Collation::NO_CE, 0, errorCode); + } + // Handle an encodable contraction unless the next contraction is too long + // and starts with the same character. + int32_t prevX = -1; + UBool addContraction = false; + UCharsTrie::Iterator suffixes(p + 2, 0, errorCode); + while(suffixes.next(errorCode)) { + const UnicodeString &suffix = suffixes.getString(); + int32_t x = CollationFastLatin::getCharIndex(suffix.charAt(0)); + if(x < 0) { continue; } // ignore anything but fast Latin text + if(x == prevX) { + if(addContraction) { + // Bail out for all contractions starting with this character. + addContractionEntry(x, Collation::NO_CE, 0, errorCode); + addContraction = false; + } + continue; + } + if(addContraction) { + addContractionEntry(prevX, ce0, ce1, errorCode); + } + ce32 = (uint32_t)suffixes.getValue(); + if(suffix.length() == 1 && getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { + addContraction = true; + } else { + addContractionEntry(x, Collation::NO_CE, 0, errorCode); + addContraction = false; + } + prevX = x; + } + if(addContraction) { + addContractionEntry(prevX, ce0, ce1, errorCode); + } + if(U_FAILURE(errorCode)) { return false; } + // Note: There might not be any fast Latin contractions, but + // we need to enter contraction handling anyway so that we can bail out + // when there is a non-fast-Latin character following. + // For example: Danish &Y<> 32) == Collation::NO_CE_PRIMARY) { return; } + ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits + int32_t i = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); + if(i < 0) { + uniqueCEs.insertElementAt(ce, ~i, errorCode); + } +} + +uint32_t +CollationFastLatinBuilder::getMiniCE(int64_t ce) const { + ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits + int32_t index = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); + U_ASSERT(index >= 0); + return miniCEs[index]; +} + +UBool +CollationFastLatinBuilder::encodeUniqueCEs(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + uprv_free(miniCEs); + miniCEs = (uint16_t *)uprv_malloc(uniqueCEs.size() * 2); + if(miniCEs == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return false; + } + int32_t group = 0; + uint32_t lastGroupPrimary = lastSpecialPrimaries[group]; + // The lowest unique CE must be at least a secondary CE. + U_ASSERT(((uint32_t)uniqueCEs.elementAti(0) >> 16) != 0); + uint32_t prevPrimary = 0; + uint32_t prevSecondary = 0; + uint32_t pri = 0; + uint32_t sec = 0; + uint32_t ter = CollationFastLatin::COMMON_TER; + for(int32_t i = 0; i < uniqueCEs.size(); ++i) { + int64_t ce = uniqueCEs.elementAti(i); + // Note: At least one of the p/s/t weights changes from one unique CE to the next. + // (uniqueCEs does not store case bits.) + uint32_t p = (uint32_t)(ce >> 32); + if(p != prevPrimary) { + while(p > lastGroupPrimary) { + U_ASSERT(pri <= CollationFastLatin::MAX_LONG); + // Set the group's header entry to the + // last "long primary" in or before the group. + result.setCharAt(1 + group, (char16_t)pri); + if(++group < NUM_SPECIAL_GROUPS) { + lastGroupPrimary = lastSpecialPrimaries[group]; + } else { + lastGroupPrimary = 0xffffffff; + break; + } + } + if(p < firstShortPrimary) { + if(pri == 0) { + pri = CollationFastLatin::MIN_LONG; + } else if(pri < CollationFastLatin::MAX_LONG) { + pri += CollationFastLatin::LONG_INC; + } else { +#if DEBUG_COLLATION_FAST_LATIN_BUILDER + printf("long-primary overflow for %08x\n", p); +#endif + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + } else { + if(pri < CollationFastLatin::MIN_SHORT) { + pri = CollationFastLatin::MIN_SHORT; + } else if(pri < (CollationFastLatin::MAX_SHORT - CollationFastLatin::SHORT_INC)) { + // Reserve the highest primary weight for U+FFFF. + pri += CollationFastLatin::SHORT_INC; + } else { +#if DEBUG_COLLATION_FAST_LATIN_BUILDER + printf("short-primary overflow for %08x\n", p); +#endif + shortPrimaryOverflow = true; + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + } + prevPrimary = p; + prevSecondary = Collation::COMMON_WEIGHT16; + sec = CollationFastLatin::COMMON_SEC; + ter = CollationFastLatin::COMMON_TER; + } + uint32_t lower32 = (uint32_t)ce; + uint32_t s = lower32 >> 16; + if(s != prevSecondary) { + if(pri == 0) { + if(sec == 0) { + sec = CollationFastLatin::MIN_SEC_HIGH; + } else if(sec < CollationFastLatin::MAX_SEC_HIGH) { + sec += CollationFastLatin::SEC_INC; + } else { + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + prevSecondary = s; + ter = CollationFastLatin::COMMON_TER; + } else if(s < Collation::COMMON_WEIGHT16) { + if(sec == CollationFastLatin::COMMON_SEC) { + sec = CollationFastLatin::MIN_SEC_BEFORE; + } else if(sec < CollationFastLatin::MAX_SEC_BEFORE) { + sec += CollationFastLatin::SEC_INC; + } else { + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + } else if(s == Collation::COMMON_WEIGHT16) { + sec = CollationFastLatin::COMMON_SEC; + } else { + if(sec < CollationFastLatin::MIN_SEC_AFTER) { + sec = CollationFastLatin::MIN_SEC_AFTER; + } else if(sec < CollationFastLatin::MAX_SEC_AFTER) { + sec += CollationFastLatin::SEC_INC; + } else { + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + } + prevSecondary = s; + ter = CollationFastLatin::COMMON_TER; + } + U_ASSERT((lower32 & Collation::CASE_MASK) == 0); // blanked out in uniqueCEs + uint32_t t = lower32 & Collation::ONLY_TERTIARY_MASK; + if(t > Collation::COMMON_WEIGHT16) { + if(ter < CollationFastLatin::MAX_TER_AFTER) { + ++ter; + } else { + miniCEs[i] = CollationFastLatin::BAIL_OUT; + continue; + } + } + if(CollationFastLatin::MIN_LONG <= pri && pri <= CollationFastLatin::MAX_LONG) { + U_ASSERT(sec == CollationFastLatin::COMMON_SEC); + miniCEs[i] = (uint16_t)(pri | ter); + } else { + miniCEs[i] = (uint16_t)(pri | sec | ter); + } + } +#if DEBUG_COLLATION_FAST_LATIN_BUILDER + printf("last mini primary: %04x\n", pri); +#endif +#if DEBUG_COLLATION_FAST_LATIN_BUILDER >= 2 + for(int32_t i = 0; i < uniqueCEs.size(); ++i) { + int64_t ce = uniqueCEs.elementAti(i); + printf("unique CE 0x%016lx -> 0x%04x\n", ce, miniCEs[i]); + } +#endif + return U_SUCCESS(errorCode); +} + +UBool +CollationFastLatinBuilder::encodeCharCEs(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + int32_t miniCEsStart = result.length(); + for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { + result.append((char16_t)0); // initialize to completely ignorable + } + int32_t indexBase = result.length(); + for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { + int64_t ce = charCEs[i][0]; + if(isContractionCharCE(ce)) { continue; } // defer contraction + uint32_t miniCE = encodeTwoCEs(ce, charCEs[i][1]); + if(miniCE > 0xffff) { + // Note: There is a chance that this new expansion is the same as a previous one, + // and if so, then we could reuse the other expansion. + // However, that seems unlikely. + int32_t expansionIndex = result.length() - indexBase; + if(expansionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { + miniCE = CollationFastLatin::BAIL_OUT; + } else { + result.append((char16_t)(miniCE >> 16)).append((char16_t)miniCE); + miniCE = CollationFastLatin::EXPANSION | expansionIndex; + } + } + result.setCharAt(miniCEsStart + i, (char16_t)miniCE); + } + return U_SUCCESS(errorCode); +} + +UBool +CollationFastLatinBuilder::encodeContractions(UErrorCode &errorCode) { + // We encode all contraction lists so that the first word of a list + // terminates the previous list, and we only need one additional terminator at the end. + if(U_FAILURE(errorCode)) { return false; } + int32_t indexBase = headerLength + CollationFastLatin::NUM_FAST_CHARS; + int32_t firstContractionIndex = result.length(); + for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { + int64_t ce = charCEs[i][0]; + if(!isContractionCharCE(ce)) { continue; } + int32_t contractionIndex = result.length() - indexBase; + if(contractionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { + result.setCharAt(headerLength + i, CollationFastLatin::BAIL_OUT); + continue; + } + UBool firstTriple = true; + for(int32_t index = (int32_t)ce & 0x7fffffff;; index += 3) { + int32_t x = static_cast(contractionCEs.elementAti(index)); + if((uint32_t)x == CollationFastLatin::CONTR_CHAR_MASK && !firstTriple) { break; } + int64_t cce0 = contractionCEs.elementAti(index + 1); + int64_t cce1 = contractionCEs.elementAti(index + 2); + uint32_t miniCE = encodeTwoCEs(cce0, cce1); + if(miniCE == CollationFastLatin::BAIL_OUT) { + result.append((char16_t)(x | (1 << CollationFastLatin::CONTR_LENGTH_SHIFT))); + } else if(miniCE <= 0xffff) { + result.append((char16_t)(x | (2 << CollationFastLatin::CONTR_LENGTH_SHIFT))); + result.append((char16_t)miniCE); + } else { + result.append((char16_t)(x | (3 << CollationFastLatin::CONTR_LENGTH_SHIFT))); + result.append((char16_t)(miniCE >> 16)).append((char16_t)miniCE); + } + firstTriple = false; + } + // Note: There is a chance that this new contraction list is the same as a previous one, + // and if so, then we could truncate the result and reuse the other list. + // However, that seems unlikely. + result.setCharAt(headerLength + i, + (char16_t)(CollationFastLatin::CONTRACTION | contractionIndex)); + } + if(result.length() > firstContractionIndex) { + // Terminate the last contraction list. + result.append((char16_t)CollationFastLatin::CONTR_CHAR_MASK); + } + if(result.isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return false; + } +#if DEBUG_COLLATION_FAST_LATIN_BUILDER + printf("** fast Latin %d * 2 = %d bytes\n", result.length(), result.length() * 2); + puts(" header & below-digit groups map"); + int32_t i = 0; + for(; i < headerLength; ++i) { + printf(" %04x", result[i]); + } + printf("\n char mini CEs"); + U_ASSERT(CollationFastLatin::NUM_FAST_CHARS % 16 == 0); + for(; i < indexBase; i += 16) { + UChar32 c = i - headerLength; + if(c >= CollationFastLatin::LATIN_LIMIT) { + c = CollationFastLatin::PUNCT_START + c - CollationFastLatin::LATIN_LIMIT; + } + printf("\n %04x:", c); + for(int32_t j = 0; j < 16; ++j) { + printf(" %04x", result[i + j]); + } + } + printf("\n expansions & contractions"); + for(; i < result.length(); ++i) { + if((i - indexBase) % 16 == 0) { puts(""); } + printf(" %04x", result[i]); + } + puts(""); +#endif + return true; +} + +uint32_t +CollationFastLatinBuilder::encodeTwoCEs(int64_t first, int64_t second) const { + if(first == 0) { + return 0; // completely ignorable + } + if(first == Collation::NO_CE) { + return CollationFastLatin::BAIL_OUT; + } + U_ASSERT((uint32_t)(first >> 32) != Collation::NO_CE_PRIMARY); + + uint32_t miniCE = getMiniCE(first); + if(miniCE == CollationFastLatin::BAIL_OUT) { return miniCE; } + if(miniCE >= CollationFastLatin::MIN_SHORT) { + // Extract & copy the case bits. + // Shift them from normal CE bits 15..14 to mini CE bits 4..3. + uint32_t c = (((uint32_t)first & Collation::CASE_MASK) >> (14 - 3)); + // Only in mini CEs: Ignorable case bits = 0, lowercase = 1. + c += CollationFastLatin::LOWER_CASE; + miniCE |= c; + } + if(second == 0) { return miniCE; } + + uint32_t miniCE1 = getMiniCE(second); + if(miniCE1 == CollationFastLatin::BAIL_OUT) { return miniCE1; } + + uint32_t case1 = (uint32_t)second & Collation::CASE_MASK; + if(miniCE >= CollationFastLatin::MIN_SHORT && + (miniCE & CollationFastLatin::SECONDARY_MASK) == CollationFastLatin::COMMON_SEC) { + // Try to combine the two mini CEs into one. + uint32_t sec1 = miniCE1 & CollationFastLatin::SECONDARY_MASK; + uint32_t ter1 = miniCE1 & CollationFastLatin::TERTIARY_MASK; + if(sec1 >= CollationFastLatin::MIN_SEC_HIGH && case1 == 0 && + ter1 == CollationFastLatin::COMMON_TER) { + // sec1>=sec_high implies pri1==0. + return (miniCE & ~CollationFastLatin::SECONDARY_MASK) | sec1; + } + } + + if(miniCE1 <= CollationFastLatin::SECONDARY_MASK || CollationFastLatin::MIN_SHORT <= miniCE1) { + // Secondary CE, or a CE with a short primary, copy the case bits. + case1 = (case1 >> (14 - 3)) + CollationFastLatin::LOWER_CASE; + miniCE1 |= case1; + } + return (miniCE << 16) | miniCE1; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationfastlatinbuilder.h b/intl/icu/source/i18n/collationfastlatinbuilder.h new file mode 100644 index 0000000000..8b63b86815 --- /dev/null +++ b/intl/icu/source/i18n/collationfastlatinbuilder.h @@ -0,0 +1,100 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2016, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationfastlatinbuilder.h +* +* created on: 2013aug09 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONFASTLATINBUILDER_H__ +#define __COLLATIONFASTLATINBUILDER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/unistr.h" +#include "unicode/uobject.h" +#include "collation.h" +#include "collationfastlatin.h" +#include "uvectr64.h" + +U_NAMESPACE_BEGIN + +struct CollationData; + +class U_I18N_API CollationFastLatinBuilder : public UObject { +public: + CollationFastLatinBuilder(UErrorCode &errorCode); + ~CollationFastLatinBuilder(); + + UBool forData(const CollationData &data, UErrorCode &errorCode); + + const uint16_t *getTable() const { + return reinterpret_cast(result.getBuffer()); + } + int32_t lengthOfTable() const { return result.length(); } + +private: + // space, punct, symbol, currency (not digit) + enum { NUM_SPECIAL_GROUPS = UCOL_REORDER_CODE_CURRENCY - UCOL_REORDER_CODE_FIRST + 1 }; + + UBool loadGroups(const CollationData &data, UErrorCode &errorCode); + UBool inSameGroup(uint32_t p, uint32_t q) const; + + void resetCEs(); + void getCEs(const CollationData &data, UErrorCode &errorCode); + UBool getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32, + UErrorCode &errorCode); + UBool getCEsFromContractionCE32(const CollationData &data, uint32_t ce32, + UErrorCode &errorCode); + void addContractionEntry(int32_t x, int64_t cce0, int64_t cce1, UErrorCode &errorCode); + void addUniqueCE(int64_t ce, UErrorCode &errorCode); + uint32_t getMiniCE(int64_t ce) const; + UBool encodeUniqueCEs(UErrorCode &errorCode); + UBool encodeCharCEs(UErrorCode &errorCode); + UBool encodeContractions(UErrorCode &errorCode); + uint32_t encodeTwoCEs(int64_t first, int64_t second) const; + + static UBool isContractionCharCE(int64_t ce) { + return (uint32_t)(ce >> 32) == Collation::NO_CE_PRIMARY && ce != Collation::NO_CE; + } + + static const uint32_t CONTRACTION_FLAG = 0x80000000; + + // temporary "buffer" + int64_t ce0, ce1; + + int64_t charCEs[CollationFastLatin::NUM_FAST_CHARS][2]; + + UVector64 contractionCEs; + UVector64 uniqueCEs; + + /** One 16-bit mini CE per unique CE. */ + uint16_t *miniCEs; + + // These are constant for a given root collator. + uint32_t lastSpecialPrimaries[NUM_SPECIAL_GROUPS]; + uint32_t firstDigitPrimary; + uint32_t firstLatinPrimary; + uint32_t lastLatinPrimary; + // This determines the first normal primary weight which is mapped to + // a short mini primary. It must be >=firstDigitPrimary. + uint32_t firstShortPrimary; + + UBool shortPrimaryOverflow; + + UnicodeString result; + int32_t headerLength; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONFASTLATINBUILDER_H__ diff --git a/intl/icu/source/i18n/collationfcd.cpp b/intl/icu/source/i18n/collationfcd.cpp new file mode 100644 index 0000000000..e1f1d0330c --- /dev/null +++ b/intl/icu/source/i18n/collationfcd.cpp @@ -0,0 +1,301 @@ +// Copyright (C) 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// Copyright (C) 1999-2016, International Business Machines +// Corporation and others. All Rights Reserved. +// +// file name: collationfcd.cpp +// +// machine-generated by: icu/tools/unicode/c/genuca/genuca.cpp + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "collationfcd.h" + +U_NAMESPACE_BEGIN + +const uint8_t CollationFCD::lcccIndex[2048]={ +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,1,1,2,3,0,0,0,0, +0,0,0,0,4,0,0,0,0,0,0,0,5,6,7,0, +8,0,9,0xa,0,0,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0x10, +0x11,0x12,0x13,0,0x14,0,0x15,0x16,0,0x17,0x18,0,0,0x17,0x19,0x1a, +0,0x17,0x19,0,0,0x17,0x19,0,0,0x17,0x19,0,0,0,0x19,0, +0,0x17,0x1b,0,0,0x17,0x19,0,0,0x1c,0x19,0,0,0,0x1d,0, +0,0x1e,0x1f,0,0,0x1e,0x1f,0,0x20,0x21,0,0x22,0x23,0,0x24,0, +0,0x25,0,0,0x19,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x26,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x27,0x28,0,0,0,0,0x29,0, +0,0,0,0,0,0x2a,0,0,0,0x13,0,0,0,0,0,0, +0x2b,0,0,0x2c,0,0x2d,0x2e,0,0,0x28,0x2f,0x30,0,0x31,0,0x32, +0,0x33,0,0,0,0,0x34,0x35,0,0,0,0,0,0,1,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x36,0x37,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x38,0,0,0,0x39,0,0,0,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x3a,0,0,0x3b,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x3c,0x3d,0,0,0x3e,0,0,0,0,0,0,0,0, +0x24,0x3f,0,0,0,0,0x2f,0x40,0,0x41,0x42,0,0,0x42,0x43,0, +0,0,0,0,0,0x44,0x45,0x46,0,0,0,0,0,0,0,0x19, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x47,0x48,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x1a,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +const uint32_t CollationFCD::lcccBits[73]={ +0,0xffffffff,0xffff7fff,0xffff,0xf8,0xfffe0000,0xbfffffff,0xb6,0x7ff0000,0xfffff800,0x10000,0x9fc00000,0x3d9f,0x20000,0xffff0000,0x7ff, +0x200ff800,0xfbc00000,0x3eef,0xe000000,0xff000000,0xfffffc00,0xfffffffb,0x10000000,0x1e2000,0x2000,0x40000000,0x602000,0x18000000,0x400,0x7000000,0xf00, +0x3000000,0x2a00000,0x3c3e0000,0xdf,0x40,0x6800000,0xe0000000,0x300000,0x100000,0x20040000,0x200,0x1800000,0x9fe00001,0xbfff0000,0x7fff,0x10, +0xff800,0xc00,0xc0040,0x800000,0xfff70000,0x31021fd,0x1fff0000,0x1ffe2,0x38000,0x80000000,0xfc00,0x6000000,0x3ff08000,0xc0000000,0x30000,0x1000, +0x3ffff,0x3800,0x80000,1,0xc19d0000,2,0x400000,0xc0000fd,0x7108000 +}; + +const uint8_t CollationFCD::tcccIndex[2048]={ +0,0,0,0,0,0,2,3,4,5,6,7,0,8,9,0xa, +0xb,0xc,0,0,0,0,0,0,1,1,0xd,0xe,0xf,0x10,0x11,0, +0x12,0x13,0x14,0x15,0x16,0,0x17,0x18,0,0,0,0,0x19,0x1a,0x1b,0, +0x1c,0x1d,0x1e,0x1f,0,0,0x20,0x21,0x22,0x23,0x24,0,0,0,0,0x25, +0x26,0x27,0x28,0,0x29,0,0x2a,0x2b,0,0x2c,0x2d,0,0,0x2e,0x2f,0x30, +0,0x31,0x32,0,0,0x2e,0x33,0,0,0x2e,0x34,0,0,0,0x33,0, +0,0x2e,0x35,0,0,0x2e,0x33,0,0,0x36,0x33,0,0,0,0x37,0, +0,0x38,0x39,0,0,0x38,0x39,0,0x3a,0x3b,0,0x3c,0x3d,0,0x3e,0, +0,0x3f,0,0,0x33,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0x40,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x41,0x42,0,0,0,0,0x43,0, +0,0,0,0,0,0x44,0,0,0,0x28,0,0,0,0,0,0, +0x45,0,0,0x46,0,0x47,0x48,0,0,0x42,0x49,0x4a,0,0x4b,0,0x4c, +0,0x4d,0,0,0,0,0x4e,0x4f,0,0,0,0,0,0,1,1, +1,1,1,1,0x50,1,1,0x51,0x52,1,0x53,0x54,1,0x55,0x56,0x57, +0,0,0,0,0,0,0x58,0x59,0,0x5a,0,0,0x5b,0x5c,0x5d,0, +0x5e,0x5f,0x60,0x61,0x62,0x63,0,0x64,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0x2e,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0x65,0,0,0,0x66,0,0,0,1, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0x67,0x68,0x69,0x6a,0x68,0x69,0x6b,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0x6c,0x6d,0,0,0x6e,0,0,0,0,0,0,0,0, +0x3e,0x6f,0,0,0,0,0x49,0x70,0,0x71,0x72,0,0,0x72,0x73,0, +0,0,0,0,0,0x74,0x75,0x76,0,0,0,0,0,0,0,0x33, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0x77,0x78,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0x40,0x79,0x7a,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0xe,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +const uint32_t CollationFCD::tcccBits[123]={ +0,0xffffffff,0x3e7effbf,0xbe7effbf,0xfffcffff,0x7ef1ff3f,0xfff3f1f8,0x7fffff3f,0x18003,0xdfffe000,0xff31ffcf,0xcfffffff,0xfffc0,0xffff7fff,0xffff,0x1d760, +0x1fc00,0x187c00,0x200708b,0x2000000,0x708b0000,0xc00000,0xf8,0xfccf0006,0x33ffcfc,0xfffe0000,0xbfffffff,0xb6,0x7ff0000,0x7c,0xfffff800,0x10000, +0x9fc80005,0x3d9f,0x20000,0xffff0000,0x7ff,0x200ff800,0xfbc00000,0x3eef,0xe000000,0xff000000,0xfffffc00,0xfffffffb,0x10120200,0xff1e2000,0x10000000,0xb0002000, +0x40000000,0x10480000,0x4e002000,0x2000,0x30002000,0x602100,0x18000000,0x24000400,0x7000000,0xf00,0x3000000,0x2a00000,0x3d7e0000,0xdf,0x40,0x6800000, +0xe0000000,0x300000,0x100000,0x20040000,0x200,0x1800000,0x9fe00001,0xbfff0000,0x7fff,0x10,0xff800,0xc00,0xc0040,0x800000,0xfff70000,0x31021fd, +0xbffffff,0x3ffffff,0x3f3fffff,0xaaff3f3f,0x3fffffff,0x1fdfffff,0xefcfffde,0x1fdc7fff,0x1fff0000,0x1ffe2,0x800,0xc000000,0x4000,0xe000,0x1210,0x50, +0x292,0x333e005,0x333,0xf000,0x3c0f,0x38000,0x80000000,0xfc00,0x55555000,0x36db02a5,0x46100000,0x47900000,0x3ff08000,0xc0000000,0x30000,0x1000, +0x3ffff,0x3800,0x80000,1,0xc19d0000,2,0x400000,0xc0000fd,0x7108000,0x5f7ffc00,0x7fdb +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationfcd.h b/intl/icu/source/i18n/collationfcd.h new file mode 100644 index 0000000000..9620452b97 --- /dev/null +++ b/intl/icu/source/i18n/collationfcd.h @@ -0,0 +1,137 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationfcd.h +* +* created on: 2012aug18 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONFCD_H__ +#define __COLLATIONFCD_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/utf16.h" + +U_NAMESPACE_BEGIN + +/** + * Data and functions for the FCD check fast path. + * + * The fast path looks at a pair of 16-bit code units and checks + * whether there is an FCD boundary between them; + * there is if the first unit has a trailing ccc=0 (!hasTccc(first)) + * or the second unit has a leading ccc=0 (!hasLccc(second)), + * or both. + * When the fast path finds a possible non-boundary, + * then the FCD check slow path looks at the actual sequence of FCD values. + * + * This is a pure optimization. + * The fast path must at least find all possible non-boundaries. + * If the fast path is too pessimistic, it costs performance. + * + * For a pair of BMP characters, the fast path tests are precise (1 bit per character). + * + * For a supplementary code point, the two units are its lead and trail surrogates. + * We set hasTccc(lead)=true if any of its 1024 associated supplementary code points + * has lccc!=0 or tccc!=0. + * We set hasLccc(trail)=true for all trail surrogates. + * As a result, we leave the fast path if the lead surrogate might start a + * supplementary code point that is not FCD-inert. + * (So the fast path need not detect that there is a surrogate pair, + * nor look ahead to the next full code point.) + * + * hasLccc(lead)=true if any of its 1024 associated supplementary code points + * has lccc!=0, for fast boundary checking between BMP & supplementary. + * + * hasTccc(trail)=false: + * It should only be tested for unpaired trail surrogates which are FCD-inert. + */ +class U_I18N_API CollationFCD { +public: + static inline UBool hasLccc(UChar32 c) { + // assert c <= 0xffff + // c can be negative, e.g., U_SENTINEL from UCharIterator; + // that is handled in the first test. + int32_t i; + return + // U+0300 is the first character with lccc!=0. + c >= 0x300 && + (i = lcccIndex[c >> 5]) != 0 && + (lcccBits[i] & ((uint32_t)1 << (c & 0x1f))) != 0; + } + + static inline UBool hasTccc(UChar32 c) { + // assert c <= 0xffff + // c can be negative, e.g., U_SENTINEL from UCharIterator; + // that is handled in the first test. + int32_t i; + return + // U+00C0 is the first character with tccc!=0. + c >= 0xc0 && + (i = tcccIndex[c >> 5]) != 0 && + (tcccBits[i] & ((uint32_t)1 << (c & 0x1f))) != 0; + } + + static inline UBool mayHaveLccc(UChar32 c) { + // Handles all of Unicode 0..10FFFF. + // c can be negative, e.g., U_SENTINEL. + // U+0300 is the first character with lccc!=0. + if(c < 0x300) { return false; } + if(c > 0xffff) { c = U16_LEAD(c); } + int32_t i; + return + (i = lcccIndex[c >> 5]) != 0 && + (lcccBits[i] & ((uint32_t)1 << (c & 0x1f))) != 0; + } + + /** + * Tibetan composite vowel signs (U+0F73, U+0F75, U+0F81) + * must be decomposed before reaching the core collation code, + * or else some sequences including them, even ones passing the FCD check, + * do not yield canonically equivalent results. + * + * This is a fast and imprecise test. + * + * @param c a code point + * @return true if c is U+0F73, U+0F75 or U+0F81 or one of several other Tibetan characters + */ + static inline UBool maybeTibetanCompositeVowel(UChar32 c) { + return (c & 0x1fff01) == 0xf01; + } + + /** + * Tibetan composite vowel signs (U+0F73, U+0F75, U+0F81) + * must be decomposed before reaching the core collation code, + * or else some sequences including them, even ones passing the FCD check, + * do not yield canonically equivalent results. + * + * They have distinct lccc/tccc combinations: 129/130 or 129/132. + * + * @param fcd16 the FCD value (lccc/tccc combination) of a code point + * @return true if fcd16 is from U+0F73, U+0F75 or U+0F81 + */ + static inline UBool isFCD16OfTibetanCompositeVowel(uint16_t fcd16) { + return fcd16 == 0x8182 || fcd16 == 0x8184; + } + +private: + CollationFCD() = delete; // No instantiation. + + static const uint8_t lcccIndex[2048]; + static const uint8_t tcccIndex[2048]; + static const uint32_t lcccBits[]; + static const uint32_t tcccBits[]; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONFCD_H__ diff --git a/intl/icu/source/i18n/collationiterator.cpp b/intl/icu/source/i18n/collationiterator.cpp new file mode 100644 index 0000000000..f8c6da22b2 --- /dev/null +++ b/intl/icu/source/i18n/collationiterator.cpp @@ -0,0 +1,955 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationiterator.cpp +* +* created on: 2010oct27 +* created by: Markus W. Scherer +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucharstrie.h" +#include "unicode/ustringtrie.h" +#include "charstr.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationfcd.h" +#include "collationiterator.h" +#include "normalizer2impl.h" +#include "uassert.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +CollationIterator::CEBuffer::~CEBuffer() {} + +UBool +CollationIterator::CEBuffer::ensureAppendCapacity(int32_t appCap, UErrorCode &errorCode) { + int32_t capacity = buffer.getCapacity(); + if((length + appCap) <= capacity) { return true; } + if(U_FAILURE(errorCode)) { return false; } + do { + if(capacity < 1000) { + capacity *= 4; + } else { + capacity *= 2; + } + } while(capacity < (length + appCap)); + int64_t *p = buffer.resize(capacity, length); + if(p == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return false; + } + return true; +} + +// State of combining marks skipped in discontiguous contraction. +// We create a state object on first use and keep it around deactivated between uses. +class SkippedState : public UMemory { +public: + // Born active but empty. + SkippedState() : pos(0), skipLengthAtMatch(0) {} + void clear() { + oldBuffer.remove(); + pos = 0; + // The newBuffer is reset by setFirstSkipped(). + } + + UBool isEmpty() const { return oldBuffer.isEmpty(); } + + UBool hasNext() const { return pos < oldBuffer.length(); } + + // Requires hasNext(). + UChar32 next() { + UChar32 c = oldBuffer.char32At(pos); + pos += U16_LENGTH(c); + return c; + } + + // Accounts for one more input code point read beyond the end of the marks buffer. + void incBeyond() { + U_ASSERT(!hasNext()); + ++pos; + } + + // Goes backward through the skipped-marks buffer. + // Returns the number of code points read beyond the skipped marks + // that need to be backtracked through normal input. + int32_t backwardNumCodePoints(int32_t n) { + int32_t length = oldBuffer.length(); + int32_t beyond = pos - length; + if(beyond > 0) { + if(beyond >= n) { + // Not back far enough to re-enter the oldBuffer. + pos -= n; + return n; + } else { + // Back out all beyond-oldBuffer code points and re-enter the buffer. + pos = oldBuffer.moveIndex32(length, beyond - n); + return beyond; + } + } else { + // Go backwards from inside the oldBuffer. + pos = oldBuffer.moveIndex32(pos, -n); + return 0; + } + } + + void setFirstSkipped(UChar32 c) { + skipLengthAtMatch = 0; + newBuffer.setTo(c); + } + + void skip(UChar32 c) { + newBuffer.append(c); + } + + void recordMatch() { skipLengthAtMatch = newBuffer.length(); } + + // Replaces the characters we consumed with the newly skipped ones. + void replaceMatch() { + // Note: UnicodeString.replace() pins pos to at most length(). + oldBuffer.replace(0, pos, newBuffer, 0, skipLengthAtMatch); + pos = 0; + } + + void saveTrieState(const UCharsTrie &trie) { trie.saveState(state); } + void resetToTrieState(UCharsTrie &trie) const { trie.resetToState(state); } + +private: + // Combining marks skipped in previous discontiguous-contraction matching. + // After that discontiguous contraction was completed, we start reading them from here. + UnicodeString oldBuffer; + // Combining marks newly skipped in current discontiguous-contraction matching. + // These might have been read from the normal text or from the oldBuffer. + UnicodeString newBuffer; + // Reading index in oldBuffer, + // or counter for how many code points have been read beyond oldBuffer (pos-oldBuffer.length()). + int32_t pos; + // newBuffer.length() at the time of the last matching character. + // When a partial match fails, we back out skipped and partial-matching input characters. + int32_t skipLengthAtMatch; + // We save the trie state before we attempt to match a character, + // so that we can skip it and try the next one. + UCharsTrie::State state; +}; + +CollationIterator::CollationIterator(const CollationIterator &other) + : UObject(other), + trie(other.trie), + data(other.data), + cesIndex(other.cesIndex), + skipped(nullptr), + numCpFwd(other.numCpFwd), + isNumeric(other.isNumeric) { + UErrorCode errorCode = U_ZERO_ERROR; + int32_t length = other.ceBuffer.length; + if(length > 0 && ceBuffer.ensureAppendCapacity(length, errorCode)) { + for(int32_t i = 0; i < length; ++i) { + ceBuffer.set(i, other.ceBuffer.get(i)); + } + ceBuffer.length = length; + } else { + cesIndex = 0; + } +} + +CollationIterator::~CollationIterator() { + delete skipped; +} + +bool +CollationIterator::operator==(const CollationIterator &other) const { + // Subclasses: Call this method and then add more specific checks. + // Compare the iterator state but not the collation data (trie & data fields): + // Assume that the caller compares the data. + // Ignore skipped since that should be unused between calls to nextCE(). + // (It only stays around to avoid another memory allocation.) + if(!(typeid(*this) == typeid(other) && + ceBuffer.length == other.ceBuffer.length && + cesIndex == other.cesIndex && + numCpFwd == other.numCpFwd && + isNumeric == other.isNumeric)) { + return false; + } + for(int32_t i = 0; i < ceBuffer.length; ++i) { + if(ceBuffer.get(i) != other.ceBuffer.get(i)) { return false; } + } + return true; +} + +void +CollationIterator::reset() { + cesIndex = ceBuffer.length = 0; + if(skipped != nullptr) { skipped->clear(); } +} + +int32_t +CollationIterator::fetchCEs(UErrorCode &errorCode) { + while(U_SUCCESS(errorCode) && nextCE(errorCode) != Collation::NO_CE) { + // No need to loop for each expansion CE. + cesIndex = ceBuffer.length; + } + return ceBuffer.length; +} + +uint32_t +CollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) { + c = nextCodePoint(errorCode); + return (c < 0) ? Collation::FALLBACK_CE32 : data->getCE32(c); +} + +char16_t +CollationIterator::handleGetTrailSurrogate() { + return 0; +} + +UBool +CollationIterator::foundNULTerminator() { + return false; +} + +UBool +CollationIterator::forbidSurrogateCodePoints() const { + return false; +} + +uint32_t +CollationIterator::getDataCE32(UChar32 c) const { + return data->getCE32(c); +} + +uint32_t +CollationIterator::getCE32FromBuilderData(uint32_t /*ce32*/, UErrorCode &errorCode) { + if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; } + return 0; +} + +int64_t +CollationIterator::nextCEFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, + UErrorCode &errorCode) { + --ceBuffer.length; // Undo ceBuffer.incLength(). + appendCEsFromCE32(d, c, ce32, true, errorCode); + if(U_SUCCESS(errorCode)) { + return ceBuffer.get(cesIndex++); + } else { + return Collation::NO_CE_PRIMARY; + } +} + +void +CollationIterator::appendCEsFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, + UBool forward, UErrorCode &errorCode) { + while(Collation::isSpecialCE32(ce32)) { + switch(Collation::tagFromCE32(ce32)) { + case Collation::FALLBACK_TAG: + case Collation::RESERVED_TAG_3: + if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; } + return; + case Collation::LONG_PRIMARY_TAG: + ceBuffer.append(Collation::ceFromLongPrimaryCE32(ce32), errorCode); + return; + case Collation::LONG_SECONDARY_TAG: + ceBuffer.append(Collation::ceFromLongSecondaryCE32(ce32), errorCode); + return; + case Collation::LATIN_EXPANSION_TAG: + if(ceBuffer.ensureAppendCapacity(2, errorCode)) { + ceBuffer.set(ceBuffer.length, Collation::latinCE0FromCE32(ce32)); + ceBuffer.set(ceBuffer.length + 1, Collation::latinCE1FromCE32(ce32)); + ceBuffer.length += 2; + } + return; + case Collation::EXPANSION32_TAG: { + const uint32_t *ce32s = d->ce32s + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + if(ceBuffer.ensureAppendCapacity(length, errorCode)) { + do { + ceBuffer.appendUnsafe(Collation::ceFromCE32(*ce32s++)); + } while(--length > 0); + } + return; + } + case Collation::EXPANSION_TAG: { + const int64_t *ces = d->ces + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + if(ceBuffer.ensureAppendCapacity(length, errorCode)) { + do { + ceBuffer.appendUnsafe(*ces++); + } while(--length > 0); + } + return; + } + case Collation::BUILDER_DATA_TAG: + ce32 = getCE32FromBuilderData(ce32, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(ce32 == Collation::FALLBACK_CE32) { + d = data->base; + ce32 = d->getCE32(c); + } + break; + case Collation::PREFIX_TAG: + if(forward) { backwardNumCodePoints(1, errorCode); } + ce32 = getCE32FromPrefix(d, ce32, errorCode); + if(forward) { forwardNumCodePoints(1, errorCode); } + break; + case Collation::CONTRACTION_TAG: { + const char16_t *p = d->contexts + Collation::indexFromCE32(ce32); + uint32_t defaultCE32 = CollationData::readCE32(p); // Default if no suffix match. + if(!forward) { + // Backward contractions are handled by previousCEUnsafe(). + // c has contractions but they were not found. + ce32 = defaultCE32; + break; + } + UChar32 nextCp; + if(skipped == nullptr && numCpFwd < 0) { + // Some portion of nextCE32FromContraction() pulled out here as an ASCII fast path, + // avoiding the function call and the nextSkippedCodePoint() overhead. + nextCp = nextCodePoint(errorCode); + if(nextCp < 0) { + // No more text. + ce32 = defaultCE32; + break; + } else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 && + !CollationFCD::mayHaveLccc(nextCp)) { + // All contraction suffixes start with characters with lccc!=0 + // but the next code point has lccc==0. + backwardNumCodePoints(1, errorCode); + ce32 = defaultCE32; + break; + } + } else { + nextCp = nextSkippedCodePoint(errorCode); + if(nextCp < 0) { + // No more text. + ce32 = defaultCE32; + break; + } else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 && + !CollationFCD::mayHaveLccc(nextCp)) { + // All contraction suffixes start with characters with lccc!=0 + // but the next code point has lccc==0. + backwardNumSkipped(1, errorCode); + ce32 = defaultCE32; + break; + } + } + ce32 = nextCE32FromContraction(d, ce32, p + 2, defaultCE32, nextCp, errorCode); + if(ce32 == Collation::NO_CE32) { + // CEs from a discontiguous contraction plus the skipped combining marks + // have been appended already. + return; + } + break; + } + case Collation::DIGIT_TAG: + if(isNumeric) { + appendNumericCEs(ce32, forward, errorCode); + return; + } else { + // Fetch the non-numeric-collation CE32 and continue. + ce32 = d->ce32s[Collation::indexFromCE32(ce32)]; + break; + } + case Collation::U0000_TAG: + U_ASSERT(c == 0); + if(forward && foundNULTerminator()) { + // Handle NUL-termination. (Not needed in Java.) + ceBuffer.append(Collation::NO_CE, errorCode); + return; + } else { + // Fetch the normal ce32 for U+0000 and continue. + ce32 = d->ce32s[0]; + break; + } + case Collation::HANGUL_TAG: { + const uint32_t *jamoCE32s = d->jamoCE32s; + c -= Hangul::HANGUL_BASE; + UChar32 t = c % Hangul::JAMO_T_COUNT; + c /= Hangul::JAMO_T_COUNT; + UChar32 v = c % Hangul::JAMO_V_COUNT; + c /= Hangul::JAMO_V_COUNT; + if((ce32 & Collation::HANGUL_NO_SPECIAL_JAMO) != 0) { + // None of the Jamo CE32s are isSpecialCE32(). + // Avoid recursive function calls and per-Jamo tests. + if(ceBuffer.ensureAppendCapacity(t == 0 ? 2 : 3, errorCode)) { + ceBuffer.set(ceBuffer.length, Collation::ceFromCE32(jamoCE32s[c])); + ceBuffer.set(ceBuffer.length + 1, Collation::ceFromCE32(jamoCE32s[19 + v])); + ceBuffer.length += 2; + if(t != 0) { + ceBuffer.appendUnsafe(Collation::ceFromCE32(jamoCE32s[39 + t])); + } + } + return; + } else { + // We should not need to compute each Jamo code point. + // In particular, there should be no offset or implicit ce32. + appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[c], forward, errorCode); + appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[19 + v], forward, errorCode); + if(t == 0) { return; } + // offset 39 = 19 + 21 - 1: + // 19 = JAMO_L_COUNT + // 21 = JAMO_T_COUNT + // -1 = omit t==0 + ce32 = jamoCE32s[39 + t]; + c = U_SENTINEL; + break; + } + } + case Collation::LEAD_SURROGATE_TAG: { + U_ASSERT(forward); // Backward iteration should never see lead surrogate code _unit_ data. + U_ASSERT(U16_IS_LEAD(c)); + char16_t trail; + if(U16_IS_TRAIL(trail = handleGetTrailSurrogate())) { + c = U16_GET_SUPPLEMENTARY(c, trail); + ce32 &= Collation::LEAD_TYPE_MASK; + if(ce32 == Collation::LEAD_ALL_UNASSIGNED) { + ce32 = Collation::UNASSIGNED_CE32; // unassigned-implicit + } else if(ce32 == Collation::LEAD_ALL_FALLBACK || + (ce32 = d->getCE32FromSupplementary(c)) == Collation::FALLBACK_CE32) { + // fall back to the base data + d = d->base; + ce32 = d->getCE32FromSupplementary(c); + } + } else { + // c is an unpaired surrogate. + ce32 = Collation::UNASSIGNED_CE32; + } + break; + } + case Collation::OFFSET_TAG: + U_ASSERT(c >= 0); + ceBuffer.append(d->getCEFromOffsetCE32(c, ce32), errorCode); + return; + case Collation::IMPLICIT_TAG: + U_ASSERT(c >= 0); + if(U_IS_SURROGATE(c) && forbidSurrogateCodePoints()) { + ce32 = Collation::FFFD_CE32; + break; + } else { + ceBuffer.append(Collation::unassignedCEFromCodePoint(c), errorCode); + return; + } + } + } + ceBuffer.append(Collation::ceFromSimpleCE32(ce32), errorCode); +} + +uint32_t +CollationIterator::getCE32FromPrefix(const CollationData *d, uint32_t ce32, + UErrorCode &errorCode) { + const char16_t *p = d->contexts + Collation::indexFromCE32(ce32); + ce32 = CollationData::readCE32(p); // Default if no prefix match. + p += 2; + // Number of code points read before the original code point. + int32_t lookBehind = 0; + UCharsTrie prefixes(p); + for(;;) { + UChar32 c = previousCodePoint(errorCode); + if(c < 0) { break; } + ++lookBehind; + UStringTrieResult match = prefixes.nextForCodePoint(c); + if(USTRINGTRIE_HAS_VALUE(match)) { + ce32 = (uint32_t)prefixes.getValue(); + } + if(!USTRINGTRIE_HAS_NEXT(match)) { break; } + } + forwardNumCodePoints(lookBehind, errorCode); + return ce32; +} + +UChar32 +CollationIterator::nextSkippedCodePoint(UErrorCode &errorCode) { + if(skipped != nullptr && skipped->hasNext()) { return skipped->next(); } + if(numCpFwd == 0) { return U_SENTINEL; } + UChar32 c = nextCodePoint(errorCode); + if(skipped != nullptr && !skipped->isEmpty() && c >= 0) { skipped->incBeyond(); } + if(numCpFwd > 0 && c >= 0) { --numCpFwd; } + return c; +} + +void +CollationIterator::backwardNumSkipped(int32_t n, UErrorCode &errorCode) { + if(skipped != nullptr && !skipped->isEmpty()) { + n = skipped->backwardNumCodePoints(n); + } + backwardNumCodePoints(n, errorCode); + if(numCpFwd >= 0) { numCpFwd += n; } +} + +uint32_t +CollationIterator::nextCE32FromContraction(const CollationData *d, uint32_t contractionCE32, + const char16_t *p, uint32_t ce32, UChar32 c, + UErrorCode &errorCode) { + // c: next code point after the original one + + // Number of code points read beyond the original code point. + // Needed for discontiguous contraction matching. + int32_t lookAhead = 1; + // Number of code points read since the last match (initially only c). + int32_t sinceMatch = 1; + // Normally we only need a contiguous match, + // and therefore need not remember the suffixes state from before a mismatch for retrying. + // If we are already processing skipped combining marks, then we do track the state. + UCharsTrie suffixes(p); + if(skipped != nullptr && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); } + UStringTrieResult match = suffixes.firstForCodePoint(c); + for(;;) { + UChar32 nextCp; + if(USTRINGTRIE_HAS_VALUE(match)) { + ce32 = (uint32_t)suffixes.getValue(); + if(!USTRINGTRIE_HAS_NEXT(match) || (c = nextSkippedCodePoint(errorCode)) < 0) { + return ce32; + } + if(skipped != nullptr && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); } + sinceMatch = 1; + } else if(match == USTRINGTRIE_NO_MATCH || (nextCp = nextSkippedCodePoint(errorCode)) < 0) { + // No match for c, or partial match (USTRINGTRIE_NO_VALUE) and no further text. + // Back up if necessary, and try a discontiguous contraction. + if((contractionCE32 & Collation::CONTRACT_TRAILING_CCC) != 0 && + // Discontiguous contraction matching extends an existing match. + // If there is no match yet, then there is nothing to do. + ((contractionCE32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) == 0 || + sinceMatch < lookAhead)) { + // The last character of at least one suffix has lccc!=0, + // allowing for discontiguous contractions. + // UCA S2.1.1 only processes non-starters immediately following + // "a match in the table" (sinceMatch=1). + if(sinceMatch > 1) { + // Return to the state after the last match. + // (Return to sinceMatch=0 and re-fetch the first partially-matched character.) + backwardNumSkipped(sinceMatch, errorCode); + c = nextSkippedCodePoint(errorCode); + lookAhead -= sinceMatch - 1; + sinceMatch = 1; + } + if(d->getFCD16(c) > 0xff) { + return nextCE32FromDiscontiguousContraction( + d, suffixes, ce32, lookAhead, c, errorCode); + } + } + break; + } else { + // Continue after partial match (USTRINGTRIE_NO_VALUE) for c. + // It does not have a result value, therefore it is not itself "a match in the table". + // If a partially-matched c has ccc!=0 then + // it might be skipped in discontiguous contraction. + c = nextCp; + ++sinceMatch; + } + ++lookAhead; + match = suffixes.nextForCodePoint(c); + } + backwardNumSkipped(sinceMatch, errorCode); + return ce32; +} + +uint32_t +CollationIterator::nextCE32FromDiscontiguousContraction( + const CollationData *d, UCharsTrie &suffixes, uint32_t ce32, + int32_t lookAhead, UChar32 c, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + + // UCA section 3.3.2 Contractions: + // Contractions that end with non-starter characters + // are known as discontiguous contractions. + // ... discontiguous contractions must be detected in input text + // whenever the final sequence of non-starter characters could be rearranged + // so as to make a contiguous matching sequence that is canonically equivalent. + + // UCA: http://www.unicode.org/reports/tr10/#S2.1 + // S2.1 Find the longest initial substring S at each point that has a match in the table. + // S2.1.1 If there are any non-starters following S, process each non-starter C. + // S2.1.2 If C is not blocked from S, find if S + C has a match in the table. + // Note: A non-starter in a string is called blocked + // if there is another non-starter of the same canonical combining class or zero + // between it and the last character of canonical combining class 0. + // S2.1.3 If there is a match, replace S by S + C, and remove C. + + // First: Is a discontiguous contraction even possible? + uint16_t fcd16 = d->getFCD16(c); + U_ASSERT(fcd16 > 0xff); // The caller checked this already, as a shortcut. + UChar32 nextCp = nextSkippedCodePoint(errorCode); + if(nextCp < 0) { + // No further text. + backwardNumSkipped(1, errorCode); + return ce32; + } + ++lookAhead; + uint8_t prevCC = (uint8_t)fcd16; + fcd16 = d->getFCD16(nextCp); + if(fcd16 <= 0xff) { + // The next code point after c is a starter (S2.1.1 "process each non-starter"). + backwardNumSkipped(2, errorCode); + return ce32; + } + + // We have read and matched (lookAhead-2) code points, + // read non-matching c and peeked ahead at nextCp. + // Return to the state before the mismatch and continue matching with nextCp. + if(skipped == nullptr || skipped->isEmpty()) { + if(skipped == nullptr) { + skipped = new SkippedState(); + if(skipped == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + } + suffixes.reset(); + if(lookAhead > 2) { + // Replay the partial match so far. + backwardNumCodePoints(lookAhead, errorCode); + suffixes.firstForCodePoint(nextCodePoint(errorCode)); + for(int32_t i = 3; i < lookAhead; ++i) { + suffixes.nextForCodePoint(nextCodePoint(errorCode)); + } + // Skip c (which did not match) and nextCp (which we will try now). + forwardNumCodePoints(2, errorCode); + } + skipped->saveTrieState(suffixes); + } else { + // Reset to the trie state before the failed match of c. + skipped->resetToTrieState(suffixes); + } + + skipped->setFirstSkipped(c); + // Number of code points read since the last match (at this point: c and nextCp). + int32_t sinceMatch = 2; + c = nextCp; + for(;;) { + UStringTrieResult match; + // "If C is not blocked from S, find if S + C has a match in the table." (S2.1.2) + if(prevCC < (fcd16 >> 8) && USTRINGTRIE_HAS_VALUE(match = suffixes.nextForCodePoint(c))) { + // "If there is a match, replace S by S + C, and remove C." (S2.1.3) + // Keep prevCC unchanged. + ce32 = (uint32_t)suffixes.getValue(); + sinceMatch = 0; + skipped->recordMatch(); + if(!USTRINGTRIE_HAS_NEXT(match)) { break; } + skipped->saveTrieState(suffixes); + } else { + // No match for "S + C", skip C. + skipped->skip(c); + skipped->resetToTrieState(suffixes); + prevCC = (uint8_t)fcd16; + } + if((c = nextSkippedCodePoint(errorCode)) < 0) { break; } + ++sinceMatch; + fcd16 = d->getFCD16(c); + if(fcd16 <= 0xff) { + // The next code point after c is a starter (S2.1.1 "process each non-starter"). + break; + } + } + backwardNumSkipped(sinceMatch, errorCode); + UBool isTopDiscontiguous = skipped->isEmpty(); + skipped->replaceMatch(); + if(isTopDiscontiguous && !skipped->isEmpty()) { + // We did get a match after skipping one or more combining marks, + // and we are not in a recursive discontiguous contraction. + // Append CEs from the contraction ce32 + // and then from the combining marks that we skipped before the match. + c = U_SENTINEL; + for(;;) { + appendCEsFromCE32(d, c, ce32, true, errorCode); + // Fetch CE32s for skipped combining marks from the normal data, with fallback, + // rather than from the CollationData where we found the contraction. + if(!skipped->hasNext()) { break; } + c = skipped->next(); + ce32 = getDataCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + d = data->base; + ce32 = d->getCE32(c); + } else { + d = data; + } + // Note: A nested discontiguous-contraction match + // replaces consumed combining marks with newly skipped ones + // and resets the reading position to the beginning. + } + skipped->clear(); + ce32 = Collation::NO_CE32; // Signal to the caller that the result is in the ceBuffer. + } + return ce32; +} + +void +CollationIterator::appendNumericCEs(uint32_t ce32, UBool forward, UErrorCode &errorCode) { + // Collect digits. + CharString digits; + if(forward) { + for(;;) { + char digit = Collation::digitFromCE32(ce32); + digits.append(digit, errorCode); + if(numCpFwd == 0) { break; } + UChar32 c = nextCodePoint(errorCode); + if(c < 0) { break; } + ce32 = data->getCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + ce32 = data->base->getCE32(c); + } + if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) { + backwardNumCodePoints(1, errorCode); + break; + } + if(numCpFwd > 0) { --numCpFwd; } + } + } else { + for(;;) { + char digit = Collation::digitFromCE32(ce32); + digits.append(digit, errorCode); + UChar32 c = previousCodePoint(errorCode); + if(c < 0) { break; } + ce32 = data->getCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + ce32 = data->base->getCE32(c); + } + if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) { + forwardNumCodePoints(1, errorCode); + break; + } + } + // Reverse the digit string. + char *p = digits.data(); + char *q = p + digits.length() - 1; + while(p < q) { + char digit = *p; + *p++ = *q; + *q-- = digit; + } + } + if(U_FAILURE(errorCode)) { return; } + int32_t pos = 0; + do { + // Skip leading zeros. + while(pos < (digits.length() - 1) && digits[pos] == 0) { ++pos; } + // Write a sequence of CEs for at most 254 digits at a time. + int32_t segmentLength = digits.length() - pos; + if(segmentLength > 254) { segmentLength = 254; } + appendNumericSegmentCEs(digits.data() + pos, segmentLength, errorCode); + pos += segmentLength; + } while(U_SUCCESS(errorCode) && pos < digits.length()); +} + +void +CollationIterator::appendNumericSegmentCEs(const char *digits, int32_t length, UErrorCode &errorCode) { + U_ASSERT(1 <= length && length <= 254); + U_ASSERT(length == 1 || digits[0] != 0); + uint32_t numericPrimary = data->numericPrimary; + // Note: We use primary byte values 2..255: digits are not compressible. + if(length <= 7) { + // Very dense encoding for small numbers. + int32_t value = digits[0]; + for(int32_t i = 1; i < length; ++i) { + value = value * 10 + digits[i]; + } + // Primary weight second byte values: + // 74 byte values 2.. 75 for small numbers in two-byte primary weights. + // 40 byte values 76..115 for medium numbers in three-byte primary weights. + // 16 byte values 116..131 for large numbers in four-byte primary weights. + // 124 byte values 132..255 for very large numbers with 4..127 digit pairs. + int32_t firstByte = 2; + int32_t numBytes = 74; + if(value < numBytes) { + // Two-byte primary for 0..73, good for day & month numbers etc. + uint32_t primary = numericPrimary | ((firstByte + value) << 16); + ceBuffer.append(Collation::makeCE(primary), errorCode); + return; + } + value -= numBytes; + firstByte += numBytes; + numBytes = 40; + if(value < numBytes * 254) { + // Three-byte primary for 74..10233=74+40*254-1, good for year numbers and more. + uint32_t primary = numericPrimary | + ((firstByte + value / 254) << 16) | ((2 + value % 254) << 8); + ceBuffer.append(Collation::makeCE(primary), errorCode); + return; + } + value -= numBytes * 254; + firstByte += numBytes; + numBytes = 16; + if(value < numBytes * 254 * 254) { + // Four-byte primary for 10234..1042489=10234+16*254*254-1. + uint32_t primary = numericPrimary | (2 + value % 254); + value /= 254; + primary |= (2 + value % 254) << 8; + value /= 254; + primary |= (firstByte + value % 254) << 16; + ceBuffer.append(Collation::makeCE(primary), errorCode); + return; + } + // original value > 1042489 + } + U_ASSERT(length >= 7); + + // The second primary byte value 132..255 indicates the number of digit pairs (4..127), + // then we generate primary bytes with those pairs. + // Omit trailing 00 pairs. + // Decrement the value for the last pair. + + // Set the exponent. 4 pairs->132, 5 pairs->133, ..., 127 pairs->255. + int32_t numPairs = (length + 1) / 2; + uint32_t primary = numericPrimary | ((132 - 4 + numPairs) << 16); + // Find the length without trailing 00 pairs. + while(digits[length - 1] == 0 && digits[length - 2] == 0) { + length -= 2; + } + // Read the first pair. + uint32_t pair; + int32_t pos; + if(length & 1) { + // Only "half a pair" if we have an odd number of digits. + pair = digits[0]; + pos = 1; + } else { + pair = digits[0] * 10 + digits[1]; + pos = 2; + } + pair = 11 + 2 * pair; + // Add the pairs of digits between pos and length. + int32_t shift = 8; + while(pos < length) { + if(shift == 0) { + // Every three pairs/bytes we need to store a 4-byte-primary CE + // and start with a new CE with the '0' primary lead byte. + primary |= pair; + ceBuffer.append(Collation::makeCE(primary), errorCode); + primary = numericPrimary; + shift = 16; + } else { + primary |= pair << shift; + shift -= 8; + } + pair = 11 + 2 * (digits[pos] * 10 + digits[pos + 1]); + pos += 2; + } + primary |= (pair - 1) << shift; + ceBuffer.append(Collation::makeCE(primary), errorCode); +} + +int64_t +CollationIterator::previousCE(UVector32 &offsets, UErrorCode &errorCode) { + if(ceBuffer.length > 0) { + // Return the previous buffered CE. + return ceBuffer.get(--ceBuffer.length); + } + offsets.removeAllElements(); + int32_t limitOffset = getOffset(); + UChar32 c = previousCodePoint(errorCode); + if(c < 0) { return Collation::NO_CE; } + if(data->isUnsafeBackward(c, isNumeric)) { + return previousCEUnsafe(c, offsets, errorCode); + } + // Simple, safe-backwards iteration: + // Get a CE going backwards, handle prefixes but no contractions. + uint32_t ce32 = data->getCE32(c); + const CollationData *d; + if(ce32 == Collation::FALLBACK_CE32) { + d = data->base; + ce32 = d->getCE32(c); + } else { + d = data; + } + if(Collation::isSimpleOrLongCE32(ce32)) { + return Collation::ceFromCE32(ce32); + } + appendCEsFromCE32(d, c, ce32, false, errorCode); + if(U_SUCCESS(errorCode)) { + if(ceBuffer.length > 1) { + offsets.addElement(getOffset(), errorCode); + // For an expansion, the offset of each non-initial CE is the limit offset, + // consistent with forward iteration. + while(offsets.size() <= ceBuffer.length) { + offsets.addElement(limitOffset, errorCode); + } + } + return ceBuffer.get(--ceBuffer.length); + } else { + return Collation::NO_CE_PRIMARY; + } +} + +int64_t +CollationIterator::previousCEUnsafe(UChar32 c, UVector32 &offsets, UErrorCode &errorCode) { + // We just move through the input counting safe and unsafe code points + // without collecting the unsafe-backward substring into a buffer and + // switching to it. + // This is to keep the logic simple. Otherwise we would have to handle + // prefix matching going before the backward buffer, switching + // to iteration and back, etc. + // In the most important case of iterating over a normal string, + // reading from the string itself is already maximally fast. + // The only drawback there is that after getting the CEs we always + // skip backward to the safe character rather than switching out + // of a backwardBuffer. + // But this should not be the common case for previousCE(), + // and correctness and maintainability are more important than + // complex optimizations. + // Find the first safe character before c. + int32_t numBackward = 1; + while((c = previousCodePoint(errorCode)) >= 0) { + ++numBackward; + if(!data->isUnsafeBackward(c, isNumeric)) { + break; + } + } + // Set the forward iteration limit. + // Note: This counts code points. + // We cannot enforce a limit in the middle of a surrogate pair or similar. + numCpFwd = numBackward; + // Reset the forward iterator. + cesIndex = 0; + U_ASSERT(ceBuffer.length == 0); + // Go forward and collect the CEs. + int32_t offset = getOffset(); + while(numCpFwd > 0) { + // nextCE() normally reads one code point. + // Contraction matching and digit specials read more and check numCpFwd. + --numCpFwd; + // Append one or more CEs to the ceBuffer. + (void)nextCE(errorCode); + U_ASSERT(U_FAILURE(errorCode) || ceBuffer.get(ceBuffer.length - 1) != Collation::NO_CE); + // No need to loop for getting each expansion CE from nextCE(). + cesIndex = ceBuffer.length; + // However, we need to write an offset for each CE. + // This is for CollationElementIterator::getOffset() to return + // intermediate offsets from the unsafe-backwards segment. + U_ASSERT(offsets.size() < ceBuffer.length); + offsets.addElement(offset, errorCode); + // For an expansion, the offset of each non-initial CE is the limit offset, + // consistent with forward iteration. + offset = getOffset(); + while(offsets.size() < ceBuffer.length) { + offsets.addElement(offset, errorCode); + } + } + U_ASSERT(offsets.size() == ceBuffer.length); + // End offset corresponding to just after the unsafe-backwards segment. + offsets.addElement(offset, errorCode); + // Reset the forward iteration limit + // and move backward to before the segment for which we fetched CEs. + numCpFwd = -1; + backwardNumCodePoints(numBackward, errorCode); + // Use the collected CEs and return the last one. + cesIndex = 0; // Avoid cesIndex > ceBuffer.length when that gets decremented. + if(U_SUCCESS(errorCode)) { + return ceBuffer.get(--ceBuffer.length); + } else { + return Collation::NO_CE_PRIMARY; + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationiterator.h b/intl/icu/source/i18n/collationiterator.h new file mode 100644 index 0000000000..07bdf61985 --- /dev/null +++ b/intl/icu/source/i18n/collationiterator.h @@ -0,0 +1,336 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationiterator.h +* +* created on: 2010oct27 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONITERATOR_H__ +#define __COLLATIONITERATOR_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" + +U_NAMESPACE_BEGIN + +class SkippedState; +class UCharsTrie; +class UVector32; + +/* Large enough for CEs of most short strings. */ +#define CEBUFFER_INITIAL_CAPACITY 40 + +// Export an explicit template instantiation of the MaybeStackArray that +// is used as a data member of CEBuffer. +// +// When building DLLs for Windows this is required even though +// no direct access to the MaybeStackArray leaks out of the i18n library. +// +// See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples. +// +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +#endif + +/** + * Collation element iterator and abstract character iterator. + * + * When a method returns a code point value, it must be in 0..10FFFF, + * except it can be negative as a sentinel value. + */ +class U_I18N_API CollationIterator : public UObject { +private: + class U_I18N_API CEBuffer { + private: + /** Large enough for CEs of most short strings. */ + static const int32_t INITIAL_CAPACITY = CEBUFFER_INITIAL_CAPACITY; + public: + CEBuffer() : length(0) {} + ~CEBuffer(); + + inline void append(int64_t ce, UErrorCode &errorCode) { + if(length < INITIAL_CAPACITY || ensureAppendCapacity(1, errorCode)) { + buffer[length++] = ce; + } + } + + inline void appendUnsafe(int64_t ce) { + buffer[length++] = ce; + } + + UBool ensureAppendCapacity(int32_t appCap, UErrorCode &errorCode); + + inline UBool incLength(UErrorCode &errorCode) { + // Use INITIAL_CAPACITY for a very simple fastpath. + // (Rather than buffer.getCapacity().) + if(length < INITIAL_CAPACITY || ensureAppendCapacity(1, errorCode)) { + ++length; + return true; + } else { + return false; + } + } + + inline int64_t set(int32_t i, int64_t ce) { + return buffer[i] = ce; + } + inline int64_t get(int32_t i) const { return buffer[i]; } + + const int64_t *getCEs() const { return buffer.getAlias(); } + + int32_t length; + + private: + CEBuffer(const CEBuffer &) = delete; + void operator=(const CEBuffer &) = delete; + + MaybeStackArray buffer; + }; + +public: + CollationIterator(const CollationData *d, UBool numeric) + : trie(d->trie), + data(d), + cesIndex(0), + skipped(nullptr), + numCpFwd(-1), + isNumeric(numeric) {} + + virtual ~CollationIterator(); + + virtual bool operator==(const CollationIterator &other) const; + inline bool operator!=(const CollationIterator &other) const { + return !operator==(other); + } + + /** + * Resets the iterator state and sets the position to the specified offset. + * Subclasses must implement, and must call the parent class method, + * or CollationIterator::reset(). + */ + virtual void resetToOffset(int32_t newOffset) = 0; + + virtual int32_t getOffset() const = 0; + + /** + * Returns the next collation element. + */ + inline int64_t nextCE(UErrorCode &errorCode) { + if(cesIndex < ceBuffer.length) { + // Return the next buffered CE. + return ceBuffer.get(cesIndex++); + } + // assert cesIndex == ceBuffer.length; + if(!ceBuffer.incLength(errorCode)) { + return Collation::NO_CE; + } + UChar32 c; + uint32_t ce32 = handleNextCE32(c, errorCode); + uint32_t t = ce32 & 0xff; + if(t < Collation::SPECIAL_CE32_LOW_BYTE) { // Forced-inline of isSpecialCE32(ce32). + // Normal CE from the main data. + // Forced-inline of ceFromSimpleCE32(ce32). + return ceBuffer.set(cesIndex++, + ((int64_t)(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (t << 8)); + } + const CollationData *d; + // The compiler should be able to optimize the previous and the following + // comparisons of t with the same constant. + if(t == Collation::SPECIAL_CE32_LOW_BYTE) { + if(c < 0) { + return ceBuffer.set(cesIndex++, Collation::NO_CE); + } + d = data->base; + ce32 = d->getCE32(c); + t = ce32 & 0xff; + if(t < Collation::SPECIAL_CE32_LOW_BYTE) { + // Normal CE from the base data. + return ceBuffer.set(cesIndex++, + ((int64_t)(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (t << 8)); + } + } else { + d = data; + } + if(t == Collation::LONG_PRIMARY_CE32_LOW_BYTE) { + // Forced-inline of ceFromLongPrimaryCE32(ce32). + return ceBuffer.set(cesIndex++, + ((int64_t)(ce32 - t) << 32) | Collation::COMMON_SEC_AND_TER_CE); + } + return nextCEFromCE32(d, c, ce32, errorCode); + } + + /** + * Fetches all CEs. + * @return getCEsLength() + */ + int32_t fetchCEs(UErrorCode &errorCode); + + /** + * Overwrites the current CE (the last one returned by nextCE()). + */ + void setCurrentCE(int64_t ce) { + // assert cesIndex > 0; + ceBuffer.set(cesIndex - 1, ce); + } + + /** + * Returns the previous collation element. + */ + int64_t previousCE(UVector32 &offsets, UErrorCode &errorCode); + + inline int32_t getCEsLength() const { + return ceBuffer.length; + } + + inline int64_t getCE(int32_t i) const { + return ceBuffer.get(i); + } + + const int64_t *getCEs() const { + return ceBuffer.getCEs(); + } + + void clearCEs() { + cesIndex = ceBuffer.length = 0; + } + + void clearCEsIfNoneRemaining() { + if(cesIndex == ceBuffer.length) { clearCEs(); } + } + + /** + * Returns the next code point (with post-increment). + * Public for identical-level comparison and for testing. + */ + virtual UChar32 nextCodePoint(UErrorCode &errorCode) = 0; + + /** + * Returns the previous code point (with pre-decrement). + * Public for identical-level comparison and for testing. + */ + virtual UChar32 previousCodePoint(UErrorCode &errorCode) = 0; + +protected: + CollationIterator(const CollationIterator &other); + + void reset(); + + /** + * Returns the next code point and its local CE32 value. + * Returns Collation::FALLBACK_CE32 at the end of the text (c<0) + * or when c's CE32 value is to be looked up in the base data (fallback). + * + * The code point is used for fallbacks, context and implicit weights. + * It is ignored when the returned CE32 is not special (e.g., FFFD_CE32). + */ + virtual uint32_t handleNextCE32(UChar32 &c, UErrorCode &errorCode); + + /** + * Called when handleNextCE32() returns a LEAD_SURROGATE_TAG for a lead surrogate code unit. + * Returns the trail surrogate in that case and advances past it, + * if a trail surrogate follows the lead surrogate. + * Otherwise returns any other code unit and does not advance. + */ + virtual char16_t handleGetTrailSurrogate(); + + /** + * Called when handleNextCE32() returns with c==0, to see whether it is a NUL terminator. + * (Not needed in Java.) + */ + virtual UBool foundNULTerminator(); + + /** + * @return false if surrogate code points U+D800..U+DFFF + * map to their own implicit primary weights (for UTF-16), + * or true if they map to CE(U+FFFD) (for UTF-8) + */ + virtual UBool forbidSurrogateCodePoints() const; + + virtual void forwardNumCodePoints(int32_t num, UErrorCode &errorCode) = 0; + + virtual void backwardNumCodePoints(int32_t num, UErrorCode &errorCode) = 0; + + /** + * Returns the CE32 from the data trie. + * Normally the same as data->getCE32(), but overridden in the builder. + * Call this only when the faster data->getCE32() cannot be used. + */ + virtual uint32_t getDataCE32(UChar32 c) const; + + virtual uint32_t getCE32FromBuilderData(uint32_t ce32, UErrorCode &errorCode); + + void appendCEsFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, + UBool forward, UErrorCode &errorCode); + + // Main lookup trie of the data object. + const UTrie2 *trie; + const CollationData *data; + +private: + int64_t nextCEFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, + UErrorCode &errorCode); + + uint32_t getCE32FromPrefix(const CollationData *d, uint32_t ce32, + UErrorCode &errorCode); + + UChar32 nextSkippedCodePoint(UErrorCode &errorCode); + + void backwardNumSkipped(int32_t n, UErrorCode &errorCode); + + uint32_t nextCE32FromContraction( + const CollationData *d, uint32_t contractionCE32, + const char16_t *p, uint32_t ce32, UChar32 c, + UErrorCode &errorCode); + + uint32_t nextCE32FromDiscontiguousContraction( + const CollationData *d, UCharsTrie &suffixes, uint32_t ce32, + int32_t lookAhead, UChar32 c, + UErrorCode &errorCode); + + /** + * Returns the previous CE when data->isUnsafeBackward(c, isNumeric). + */ + int64_t previousCEUnsafe(UChar32 c, UVector32 &offsets, UErrorCode &errorCode); + + /** + * Turns a string of digits (bytes 0..9) + * into a sequence of CEs that will sort in numeric order. + * + * Starts from this ce32's digit value and consumes the following/preceding digits. + * The digits string must not be empty and must not have leading zeros. + */ + void appendNumericCEs(uint32_t ce32, UBool forward, UErrorCode &errorCode); + + /** + * Turns 1..254 digits into a sequence of CEs. + * Called by appendNumericCEs() for each segment of at most 254 digits. + */ + void appendNumericSegmentCEs(const char *digits, int32_t length, UErrorCode &errorCode); + + CEBuffer ceBuffer; + int32_t cesIndex; + + SkippedState *skipped; + + // Number of code points to read forward, or -1. + // Used as a forward iteration limit in previousCEUnsafe(). + int32_t numCpFwd; + // Numeric collation (CollationSettings::NUMERIC). + UBool isNumeric; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONITERATOR_H__ diff --git a/intl/icu/source/i18n/collationkeys.cpp b/intl/icu/source/i18n/collationkeys.cpp new file mode 100644 index 0000000000..c429ac3f8f --- /dev/null +++ b/intl/icu/source/i18n/collationkeys.cpp @@ -0,0 +1,673 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationkeys.cpp +* +* created on: 2012sep02 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/bytestream.h" +#include "collation.h" +#include "collationiterator.h" +#include "collationkeys.h" +#include "collationsettings.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +SortKeyByteSink::~SortKeyByteSink() {} + +void +SortKeyByteSink::Append(const char *bytes, int32_t n) { + if (n <= 0 || bytes == nullptr) { + return; + } + if (ignore_ > 0) { + int32_t ignoreRest = ignore_ - n; + if (ignoreRest >= 0) { + ignore_ = ignoreRest; + return; + } else { + bytes += ignore_; + n = -ignoreRest; + ignore_ = 0; + } + } + int32_t length = appended_; + appended_ += n; + if ((buffer_ + length) == bytes) { + return; // the caller used GetAppendBuffer() and wrote the bytes already + } + int32_t available = capacity_ - length; + if (n <= available) { + uprv_memcpy(buffer_ + length, bytes, n); + } else { + AppendBeyondCapacity(bytes, n, length); + } +} + +char * +SortKeyByteSink::GetAppendBuffer(int32_t min_capacity, + int32_t desired_capacity_hint, + char *scratch, + int32_t scratch_capacity, + int32_t *result_capacity) { + if (min_capacity < 1 || scratch_capacity < min_capacity) { + *result_capacity = 0; + return nullptr; + } + if (ignore_ > 0) { + // Do not write ignored bytes right at the end of the buffer. + *result_capacity = scratch_capacity; + return scratch; + } + int32_t available = capacity_ - appended_; + if (available >= min_capacity) { + *result_capacity = available; + return buffer_ + appended_; + } else if (Resize(desired_capacity_hint, appended_)) { + *result_capacity = capacity_ - appended_; + return buffer_ + appended_; + } else { + *result_capacity = scratch_capacity; + return scratch; + } +} + +namespace { + +/** + * uint8_t byte buffer, similar to CharString but simpler. + */ +class SortKeyLevel : public UMemory { +public: + SortKeyLevel() : len(0), ok(true) {} + ~SortKeyLevel() {} + + /** @return false if memory allocation failed */ + UBool isOk() const { return ok; } + UBool isEmpty() const { return len == 0; } + int32_t length() const { return len; } + const uint8_t *data() const { return buffer.getAlias(); } + uint8_t operator[](int32_t index) const { return buffer[index]; } + + uint8_t *data() { return buffer.getAlias(); } + + void appendByte(uint32_t b); + void appendWeight16(uint32_t w); + void appendWeight32(uint32_t w); + void appendReverseWeight16(uint32_t w); + + /** Appends all but the last byte to the sink. The last byte should be the 01 terminator. */ + void appendTo(ByteSink &sink) const { + U_ASSERT(len > 0 && buffer[len - 1] == 1); + sink.Append(reinterpret_cast(buffer.getAlias()), len - 1); + } + +private: + MaybeStackArray buffer; + int32_t len; + UBool ok; + + UBool ensureCapacity(int32_t appendCapacity); + + SortKeyLevel(const SortKeyLevel &other); // forbid copying of this class + SortKeyLevel &operator=(const SortKeyLevel &other); // forbid copying of this class +}; + +void SortKeyLevel::appendByte(uint32_t b) { + if(len < buffer.getCapacity() || ensureCapacity(1)) { + buffer[len++] = (uint8_t)b; + } +} + +void +SortKeyLevel::appendWeight16(uint32_t w) { + U_ASSERT((w & 0xffff) != 0); + uint8_t b0 = (uint8_t)(w >> 8); + uint8_t b1 = (uint8_t)w; + int32_t appendLength = (b1 == 0) ? 1 : 2; + if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) { + buffer[len++] = b0; + if(b1 != 0) { + buffer[len++] = b1; + } + } +} + +void +SortKeyLevel::appendWeight32(uint32_t w) { + U_ASSERT(w != 0); + uint8_t bytes[4] = { (uint8_t)(w >> 24), (uint8_t)(w >> 16), (uint8_t)(w >> 8), (uint8_t)w }; + int32_t appendLength = (bytes[1] == 0) ? 1 : (bytes[2] == 0) ? 2 : (bytes[3] == 0) ? 3 : 4; + if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) { + buffer[len++] = bytes[0]; + if(bytes[1] != 0) { + buffer[len++] = bytes[1]; + if(bytes[2] != 0) { + buffer[len++] = bytes[2]; + if(bytes[3] != 0) { + buffer[len++] = bytes[3]; + } + } + } + } +} + +void +SortKeyLevel::appendReverseWeight16(uint32_t w) { + U_ASSERT((w & 0xffff) != 0); + uint8_t b0 = (uint8_t)(w >> 8); + uint8_t b1 = (uint8_t)w; + int32_t appendLength = (b1 == 0) ? 1 : 2; + if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) { + if(b1 == 0) { + buffer[len++] = b0; + } else { + buffer[len] = b1; + buffer[len + 1] = b0; + len += 2; + } + } +} + +UBool SortKeyLevel::ensureCapacity(int32_t appendCapacity) { + if(!ok) { + return false; + } + int32_t newCapacity = 2 * buffer.getCapacity(); + int32_t altCapacity = len + 2 * appendCapacity; + if (newCapacity < altCapacity) { + newCapacity = altCapacity; + } + if (newCapacity < 200) { + newCapacity = 200; + } + if(buffer.resize(newCapacity, len)==nullptr) { + return ok = false; + } + return true; +} + +} // namespace + +CollationKeys::LevelCallback::~LevelCallback() {} + +UBool +CollationKeys::LevelCallback::needToWrite(Collation::Level /*level*/) { return true; } + +/** + * Map from collation strength (UColAttributeValue) + * to a mask of Collation::Level bits up to that strength, + * excluding the CASE_LEVEL which is independent of the strength, + * and excluding IDENTICAL_LEVEL which this function does not write. + */ +static const uint32_t levelMasks[UCOL_STRENGTH_LIMIT] = { + 2, // UCOL_PRIMARY -> PRIMARY_LEVEL + 6, // UCOL_SECONDARY -> up to SECONDARY_LEVEL + 0x16, // UCOL_TERTIARY -> up to TERTIARY_LEVEL + 0x36, // UCOL_QUATERNARY -> up to QUATERNARY_LEVEL + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, + 0x36 // UCOL_IDENTICAL -> up to QUATERNARY_LEVEL +}; + +void +CollationKeys::writeSortKeyUpToQuaternary(CollationIterator &iter, + const UBool *compressibleBytes, + const CollationSettings &settings, + SortKeyByteSink &sink, + Collation::Level minLevel, LevelCallback &callback, + UBool preflight, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + + int32_t options = settings.options; + // Set of levels to process and write. + uint32_t levels = levelMasks[CollationSettings::getStrength(options)]; + if((options & CollationSettings::CASE_LEVEL) != 0) { + levels |= Collation::CASE_LEVEL_FLAG; + } + // Minus the levels below minLevel. + levels &= ~(((uint32_t)1 << minLevel) - 1); + if(levels == 0) { return; } + + uint32_t variableTop; + if((options & CollationSettings::ALTERNATE_MASK) == 0) { + variableTop = 0; + } else { + // +1 so that we can use "<" and primary ignorables test out early. + variableTop = settings.variableTop + 1; + } + + uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options); + + SortKeyLevel cases; + SortKeyLevel secondaries; + SortKeyLevel tertiaries; + SortKeyLevel quaternaries; + + uint32_t prevReorderedPrimary = 0; // 0==no compression + int32_t commonCases = 0; + int32_t commonSecondaries = 0; + int32_t commonTertiaries = 0; + int32_t commonQuaternaries = 0; + + uint32_t prevSecondary = 0; + int32_t secSegmentStart = 0; + + for(;;) { + // No need to keep all CEs in the buffer when we write a sort key. + iter.clearCEsIfNoneRemaining(); + int64_t ce = iter.nextCE(errorCode); + uint32_t p = (uint32_t)(ce >> 32); + if(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY) { + // Variable CE, shift it to quaternary level. + // Ignore all following primary ignorables, and shift further variable CEs. + if(commonQuaternaries != 0) { + --commonQuaternaries; + while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) { + quaternaries.appendByte(QUAT_COMMON_MIDDLE); + commonQuaternaries -= QUAT_COMMON_MAX_COUNT; + } + // Shifted primary weights are lower than the common weight. + quaternaries.appendByte(QUAT_COMMON_LOW + commonQuaternaries); + commonQuaternaries = 0; + } + do { + if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) { + if(settings.hasReordering()) { + p = settings.reorder(p); + } + if((p >> 24) >= QUAT_SHIFTED_LIMIT_BYTE) { + // Prevent shifted primary lead bytes from + // overlapping with the common compression range. + quaternaries.appendByte(QUAT_SHIFTED_LIMIT_BYTE); + } + quaternaries.appendWeight32(p); + } + do { + ce = iter.nextCE(errorCode); + p = (uint32_t)(ce >> 32); + } while(p == 0); + } while(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY); + } + // ce could be primary ignorable, or NO_CE, or the merge separator, + // or a regular primary CE, but it is not variable. + // If ce==NO_CE, then write nothing for the primary level but + // terminate compression on all levels and then exit the loop. + if(p > Collation::NO_CE_PRIMARY && (levels & Collation::PRIMARY_LEVEL_FLAG) != 0) { + // Test the un-reordered primary for compressibility. + UBool isCompressible = compressibleBytes[p >> 24]; + if(settings.hasReordering()) { + p = settings.reorder(p); + } + uint32_t p1 = p >> 24; + if(!isCompressible || p1 != (prevReorderedPrimary >> 24)) { + if(prevReorderedPrimary != 0) { + if(p < prevReorderedPrimary) { + // No primary compression terminator + // at the end of the level or merged segment. + if(p1 > Collation::MERGE_SEPARATOR_BYTE) { + sink.Append(Collation::PRIMARY_COMPRESSION_LOW_BYTE); + } + } else { + sink.Append(Collation::PRIMARY_COMPRESSION_HIGH_BYTE); + } + } + sink.Append(p1); + if(isCompressible) { + prevReorderedPrimary = p; + } else { + prevReorderedPrimary = 0; + } + } + char p2 = (char)(p >> 16); + if(p2 != 0) { + char buffer[3] = { p2, (char)(p >> 8), (char)p }; + sink.Append(buffer, (buffer[1] == 0) ? 1 : (buffer[2] == 0) ? 2 : 3); + } + // Optimization for internalNextSortKeyPart(): + // When the primary level overflows we can stop because we need not + // calculate (preflight) the whole sort key length. + if(!preflight && sink.Overflowed()) { + if(U_SUCCESS(errorCode) && !sink.IsOk()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } + return; + } + } + + uint32_t lower32 = (uint32_t)ce; + if(lower32 == 0) { continue; } // completely ignorable, no secondary/case/tertiary/quaternary + + if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) { + uint32_t s = lower32 >> 16; + if(s == 0) { + // secondary ignorable + } else if(s == Collation::COMMON_WEIGHT16 && + ((options & CollationSettings::BACKWARD_SECONDARY) == 0 || + p != Collation::MERGE_SEPARATOR_PRIMARY)) { + // s is a common secondary weight, and + // backwards-secondary is off or the ce is not the merge separator. + ++commonSecondaries; + } else if((options & CollationSettings::BACKWARD_SECONDARY) == 0) { + if(commonSecondaries != 0) { + --commonSecondaries; + while(commonSecondaries >= SEC_COMMON_MAX_COUNT) { + secondaries.appendByte(SEC_COMMON_MIDDLE); + commonSecondaries -= SEC_COMMON_MAX_COUNT; + } + uint32_t b; + if(s < Collation::COMMON_WEIGHT16) { + b = SEC_COMMON_LOW + commonSecondaries; + } else { + b = SEC_COMMON_HIGH - commonSecondaries; + } + secondaries.appendByte(b); + commonSecondaries = 0; + } + secondaries.appendWeight16(s); + } else { + if(commonSecondaries != 0) { + --commonSecondaries; + // Append reverse weights. The level will be re-reversed later. + int32_t remainder = commonSecondaries % SEC_COMMON_MAX_COUNT; + uint32_t b; + if(prevSecondary < Collation::COMMON_WEIGHT16) { + b = SEC_COMMON_LOW + remainder; + } else { + b = SEC_COMMON_HIGH - remainder; + } + secondaries.appendByte(b); + commonSecondaries -= remainder; + // commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT. + while(commonSecondaries > 0) { // same as >= SEC_COMMON_MAX_COUNT + secondaries.appendByte(SEC_COMMON_MIDDLE); + commonSecondaries -= SEC_COMMON_MAX_COUNT; + } + // commonSecondaries == 0 + } + if(0 < p && p <= Collation::MERGE_SEPARATOR_PRIMARY) { + // The backwards secondary level compares secondary weights backwards + // within segments separated by the merge separator (U+FFFE). + uint8_t *secs = secondaries.data(); + int32_t last = secondaries.length() - 1; + if(secSegmentStart < last) { + uint8_t *q = secs + secSegmentStart; + uint8_t *r = secs + last; + do { + uint8_t b = *q; + *q++ = *r; + *r-- = b; + } while(q < r); + } + secondaries.appendByte(p == Collation::NO_CE_PRIMARY ? + Collation::LEVEL_SEPARATOR_BYTE : Collation::MERGE_SEPARATOR_BYTE); + prevSecondary = 0; + secSegmentStart = secondaries.length(); + } else { + secondaries.appendReverseWeight16(s); + prevSecondary = s; + } + } + } + + if((levels & Collation::CASE_LEVEL_FLAG) != 0) { + if((CollationSettings::getStrength(options) == UCOL_PRIMARY) ? + p == 0 : lower32 <= 0xffff) { + // Primary+caseLevel: Ignore case level weights of primary ignorables. + // Otherwise: Ignore case level weights of secondary ignorables. + // For details see the comments in the CollationCompare class. + } else { + uint32_t c = (lower32 >> 8) & 0xff; // case bits & tertiary lead byte + U_ASSERT((c & 0xc0) != 0xc0); + if((c & 0xc0) == 0 && c > Collation::LEVEL_SEPARATOR_BYTE) { + ++commonCases; + } else { + if((options & CollationSettings::UPPER_FIRST) == 0) { + // lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14, upper=15. + // If there are only common (=lowest) weights in the whole level, + // then we need not write anything. + // Level length differences are handled already on the next-higher level. + if(commonCases != 0 && + (c > Collation::LEVEL_SEPARATOR_BYTE || !cases.isEmpty())) { + --commonCases; + while(commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT) { + cases.appendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4); + commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT; + } + uint32_t b; + if(c <= Collation::LEVEL_SEPARATOR_BYTE) { + b = CASE_LOWER_FIRST_COMMON_LOW + commonCases; + } else { + b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases; + } + cases.appendByte(b << 4); + commonCases = 0; + } + if(c > Collation::LEVEL_SEPARATOR_BYTE) { + c = (CASE_LOWER_FIRST_COMMON_HIGH + (c >> 6)) << 4; // 14 or 15 + } + } else { + // upperFirst: Compress common weights to nibbles 3..15, mixed=2, upper=1. + // The compressed common case weights only go up from the "low" value + // because with upperFirst the common weight is the highest one. + if(commonCases != 0) { + --commonCases; + while(commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT) { + cases.appendByte(CASE_UPPER_FIRST_COMMON_LOW << 4); + commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT; + } + cases.appendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4); + commonCases = 0; + } + if(c > Collation::LEVEL_SEPARATOR_BYTE) { + c = (CASE_UPPER_FIRST_COMMON_LOW - (c >> 6)) << 4; // 2 or 1 + } + } + // c is a separator byte 01, + // or a left-shifted nibble 0x10, 0x20, ... 0xf0. + cases.appendByte(c); + } + } + } + + if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) { + uint32_t t = lower32 & tertiaryMask; + U_ASSERT((lower32 & 0xc000) != 0xc000); + if(t == Collation::COMMON_WEIGHT16) { + ++commonTertiaries; + } else if((tertiaryMask & 0x8000) == 0) { + // Tertiary weights without case bits. + // Move lead bytes 06..3F to C6..FF for a large common-weight range. + if(commonTertiaries != 0) { + --commonTertiaries; + while(commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT) { + tertiaries.appendByte(TER_ONLY_COMMON_MIDDLE); + commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT; + } + uint32_t b; + if(t < Collation::COMMON_WEIGHT16) { + b = TER_ONLY_COMMON_LOW + commonTertiaries; + } else { + b = TER_ONLY_COMMON_HIGH - commonTertiaries; + } + tertiaries.appendByte(b); + commonTertiaries = 0; + } + if(t > Collation::COMMON_WEIGHT16) { t += 0xc000; } + tertiaries.appendWeight16(t); + } else if((options & CollationSettings::UPPER_FIRST) == 0) { + // Tertiary weights with caseFirst=lowerFirst. + // Move lead bytes 06..BF to 46..FF for the common-weight range. + if(commonTertiaries != 0) { + --commonTertiaries; + while(commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT) { + tertiaries.appendByte(TER_LOWER_FIRST_COMMON_MIDDLE); + commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT; + } + uint32_t b; + if(t < Collation::COMMON_WEIGHT16) { + b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries; + } else { + b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries; + } + tertiaries.appendByte(b); + commonTertiaries = 0; + } + if(t > Collation::COMMON_WEIGHT16) { t += 0x4000; } + tertiaries.appendWeight16(t); + } else { + // Tertiary weights with caseFirst=upperFirst. + // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut), + // to keep tertiary CEs well-formed. + // Their case+tertiary weights must be greater than those of + // primary and secondary CEs. + // + // Separator 01 -> 01 (unchanged) + // Lowercase 02..04 -> 82..84 (includes uncased) + // Common weight 05 -> 85..C5 (common-weight compression range) + // Lowercase 06..3F -> C6..FF + // Mixed case 42..7F -> 42..7F + // Uppercase 82..BF -> 02..3F + // Tertiary CE 86..BF -> C6..FF + if(t <= Collation::NO_CE_WEIGHT16) { + // Keep separators unchanged. + } else if(lower32 > 0xffff) { + // Invert case bits of primary & secondary CEs. + t ^= 0xc000; + if(t < (TER_UPPER_FIRST_COMMON_HIGH << 8)) { + t -= 0x4000; + } + } else { + // Keep uppercase bits of tertiary CEs. + U_ASSERT(0x8600 <= t && t <= 0xbfff); + t += 0x4000; + } + if(commonTertiaries != 0) { + --commonTertiaries; + while(commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT) { + tertiaries.appendByte(TER_UPPER_FIRST_COMMON_MIDDLE); + commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT; + } + uint32_t b; + if(t < (TER_UPPER_FIRST_COMMON_LOW << 8)) { + b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries; + } else { + b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries; + } + tertiaries.appendByte(b); + commonTertiaries = 0; + } + tertiaries.appendWeight16(t); + } + } + + if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) { + uint32_t q = lower32 & 0xffff; + if((q & 0xc0) == 0 && q > Collation::NO_CE_WEIGHT16) { + ++commonQuaternaries; + } else if(q == Collation::NO_CE_WEIGHT16 && + (options & CollationSettings::ALTERNATE_MASK) == 0 && + quaternaries.isEmpty()) { + // If alternate=non-ignorable and there are only common quaternary weights, + // then we need not write anything. + // The only weights greater than the merge separator and less than the common weight + // are shifted primary weights, which are not generated for alternate=non-ignorable. + // There are also exactly as many quaternary weights as tertiary weights, + // so level length differences are handled already on tertiary level. + // Any above-common quaternary weight will compare greater regardless. + quaternaries.appendByte(Collation::LEVEL_SEPARATOR_BYTE); + } else { + if(q == Collation::NO_CE_WEIGHT16) { + q = Collation::LEVEL_SEPARATOR_BYTE; + } else { + q = 0xfc + ((q >> 6) & 3); + } + if(commonQuaternaries != 0) { + --commonQuaternaries; + while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) { + quaternaries.appendByte(QUAT_COMMON_MIDDLE); + commonQuaternaries -= QUAT_COMMON_MAX_COUNT; + } + uint32_t b; + if(q < QUAT_COMMON_LOW) { + b = QUAT_COMMON_LOW + commonQuaternaries; + } else { + b = QUAT_COMMON_HIGH - commonQuaternaries; + } + quaternaries.appendByte(b); + commonQuaternaries = 0; + } + quaternaries.appendByte(q); + } + } + + if((lower32 >> 24) == Collation::LEVEL_SEPARATOR_BYTE) { break; } // ce == NO_CE + } + + if(U_FAILURE(errorCode)) { return; } + + // Append the beyond-primary levels. + UBool ok = true; + if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) { + if(!callback.needToWrite(Collation::SECONDARY_LEVEL)) { return; } + ok &= secondaries.isOk(); + sink.Append(Collation::LEVEL_SEPARATOR_BYTE); + secondaries.appendTo(sink); + } + + if((levels & Collation::CASE_LEVEL_FLAG) != 0) { + if(!callback.needToWrite(Collation::CASE_LEVEL)) { return; } + ok &= cases.isOk(); + sink.Append(Collation::LEVEL_SEPARATOR_BYTE); + // Write pairs of nibbles as bytes, except separator bytes as themselves. + int32_t length = cases.length() - 1; // Ignore the trailing NO_CE. + uint8_t b = 0; + for(int32_t i = 0; i < length; ++i) { + uint8_t c = (uint8_t)cases[i]; + U_ASSERT((c & 0xf) == 0 && c != 0); + if(b == 0) { + b = c; + } else { + sink.Append(b | (c >> 4)); + b = 0; + } + } + if(b != 0) { + sink.Append(b); + } + } + + if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) { + if(!callback.needToWrite(Collation::TERTIARY_LEVEL)) { return; } + ok &= tertiaries.isOk(); + sink.Append(Collation::LEVEL_SEPARATOR_BYTE); + tertiaries.appendTo(sink); + } + + if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) { + if(!callback.needToWrite(Collation::QUATERNARY_LEVEL)) { return; } + ok &= quaternaries.isOk(); + sink.Append(Collation::LEVEL_SEPARATOR_BYTE); + quaternaries.appendTo(sink); + } + + if(!ok || !sink.IsOk()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationkeys.h b/intl/icu/source/i18n/collationkeys.h new file mode 100644 index 0000000000..d133156612 --- /dev/null +++ b/intl/icu/source/i18n/collationkeys.h @@ -0,0 +1,169 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationkeys.h +* +* created on: 2012sep02 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONKEYS_H__ +#define __COLLATIONKEYS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/bytestream.h" +#include "unicode/ucol.h" +#include "charstr.h" +#include "collation.h" + +U_NAMESPACE_BEGIN + +class CollationIterator; +struct CollationDataReader; +struct CollationSettings; + +class SortKeyByteSink : public ByteSink { +public: + SortKeyByteSink(char *dest, int32_t destCapacity) + : buffer_(dest), capacity_(destCapacity), + appended_(0), ignore_(0) {} + virtual ~SortKeyByteSink(); + + void IgnoreBytes(int32_t numIgnore) { ignore_ = numIgnore; } + + virtual void Append(const char *bytes, int32_t n) override; + void Append(uint32_t b) { + if (ignore_ > 0) { + --ignore_; + } else { + if (appended_ < capacity_ || Resize(1, appended_)) { + buffer_[appended_] = (char)b; + } + ++appended_; + } + } + virtual char *GetAppendBuffer(int32_t min_capacity, + int32_t desired_capacity_hint, + char *scratch, int32_t scratch_capacity, + int32_t *result_capacity) override; + int32_t NumberOfBytesAppended() const { return appended_; } + + /** + * @return how many bytes can be appended (including ignored ones) + * without reallocation + */ + int32_t GetRemainingCapacity() const { + // Either ignore_ or appended_ should be 0. + return ignore_ + capacity_ - appended_; + } + + UBool Overflowed() const { return appended_ > capacity_; } + /** @return false if memory allocation failed */ + UBool IsOk() const { return buffer_ != nullptr; } + +protected: + virtual void AppendBeyondCapacity(const char *bytes, int32_t n, int32_t length) = 0; + virtual UBool Resize(int32_t appendCapacity, int32_t length) = 0; + + void SetNotOk() { + buffer_ = nullptr; + capacity_ = 0; + } + + char *buffer_; + int32_t capacity_; + int32_t appended_; + int32_t ignore_; + +private: + SortKeyByteSink(const SortKeyByteSink &); // copy constructor not implemented + SortKeyByteSink &operator=(const SortKeyByteSink &); // assignment operator not implemented +}; + +class U_I18N_API CollationKeys /* not : public UObject because all methods are static */ { +public: + class LevelCallback : public UMemory { + public: + virtual ~LevelCallback(); + /** + * @param level The next level about to be written to the ByteSink. + * @return true if the level is to be written + * (the base class implementation always returns true) + */ + virtual UBool needToWrite(Collation::Level level); + }; + + /** + * Writes the sort key bytes for minLevel up to the iterator data's strength. + * Optionally writes the case level. + * Stops writing levels when callback.needToWrite(level) returns false. + * Separates levels with the LEVEL_SEPARATOR_BYTE + * but does not write a TERMINATOR_BYTE. + */ + static void writeSortKeyUpToQuaternary(CollationIterator &iter, + const UBool *compressibleBytes, + const CollationSettings &settings, + SortKeyByteSink &sink, + Collation::Level minLevel, LevelCallback &callback, + UBool preflight, UErrorCode &errorCode); +private: + friend struct CollationDataReader; + + CollationKeys() = delete; // no instantiation + + // Secondary level: Compress up to 33 common weights as 05..25 or 25..45. + static const uint32_t SEC_COMMON_LOW = Collation::COMMON_BYTE; + static const uint32_t SEC_COMMON_MIDDLE = SEC_COMMON_LOW + 0x20; + static const uint32_t SEC_COMMON_HIGH = SEC_COMMON_LOW + 0x40; + static const int32_t SEC_COMMON_MAX_COUNT = 0x21; + + // Case level, lowerFirst: Compress up to 7 common weights as 1..7 or 7..13. + static const uint32_t CASE_LOWER_FIRST_COMMON_LOW = 1; + static const uint32_t CASE_LOWER_FIRST_COMMON_MIDDLE = 7; + static const uint32_t CASE_LOWER_FIRST_COMMON_HIGH = 13; + static const int32_t CASE_LOWER_FIRST_COMMON_MAX_COUNT = 7; + + // Case level, upperFirst: Compress up to 13 common weights as 3..15. + static const uint32_t CASE_UPPER_FIRST_COMMON_LOW = 3; + static const uint32_t CASE_UPPER_FIRST_COMMON_HIGH = 15; + static const int32_t CASE_UPPER_FIRST_COMMON_MAX_COUNT = 13; + + // Tertiary level only (no case): Compress up to 97 common weights as 05..65 or 65..C5. + static const uint32_t TER_ONLY_COMMON_LOW = Collation::COMMON_BYTE; + static const uint32_t TER_ONLY_COMMON_MIDDLE = TER_ONLY_COMMON_LOW + 0x60; + static const uint32_t TER_ONLY_COMMON_HIGH = TER_ONLY_COMMON_LOW + 0xc0; + static const int32_t TER_ONLY_COMMON_MAX_COUNT = 0x61; + + // Tertiary with case, lowerFirst: Compress up to 33 common weights as 05..25 or 25..45. + static const uint32_t TER_LOWER_FIRST_COMMON_LOW = Collation::COMMON_BYTE; + static const uint32_t TER_LOWER_FIRST_COMMON_MIDDLE = TER_LOWER_FIRST_COMMON_LOW + 0x20; + static const uint32_t TER_LOWER_FIRST_COMMON_HIGH = TER_LOWER_FIRST_COMMON_LOW + 0x40; + static const int32_t TER_LOWER_FIRST_COMMON_MAX_COUNT = 0x21; + + // Tertiary with case, upperFirst: Compress up to 33 common weights as 85..A5 or A5..C5. + static const uint32_t TER_UPPER_FIRST_COMMON_LOW = Collation::COMMON_BYTE + 0x80; + static const uint32_t TER_UPPER_FIRST_COMMON_MIDDLE = TER_UPPER_FIRST_COMMON_LOW + 0x20; + static const uint32_t TER_UPPER_FIRST_COMMON_HIGH = TER_UPPER_FIRST_COMMON_LOW + 0x40; + static const int32_t TER_UPPER_FIRST_COMMON_MAX_COUNT = 0x21; + + // Quaternary level: Compress up to 113 common weights as 1C..8C or 8C..FC. + static const uint32_t QUAT_COMMON_LOW = 0x1c; + static const uint32_t QUAT_COMMON_MIDDLE = QUAT_COMMON_LOW + 0x70; + static const uint32_t QUAT_COMMON_HIGH = QUAT_COMMON_LOW + 0xE0; + static const int32_t QUAT_COMMON_MAX_COUNT = 0x71; + // Primary weights shifted to quaternary level must be encoded with + // a lead byte below the common-weight compression range. + static const uint32_t QUAT_SHIFTED_LIMIT_BYTE = QUAT_COMMON_LOW - 1; // 0x1b +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONKEYS_H__ diff --git a/intl/icu/source/i18n/collationroot.cpp b/intl/icu/source/i18n/collationroot.cpp new file mode 100644 index 0000000000..99686345f9 --- /dev/null +++ b/intl/icu/source/i18n/collationroot.cpp @@ -0,0 +1,140 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationroot.cpp +* +* created on: 2012dec17 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/coll.h" +#include "unicode/udata.h" +#include "collation.h" +#include "collationdata.h" +#include "collationdatareader.h" +#include "collationroot.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "normalizer2impl.h" +#include "ucln_in.h" +#include "udatamem.h" +#include "umutex.h" +#include "umapfile.h" + +U_NAMESPACE_BEGIN + +namespace { + +static const CollationCacheEntry *rootSingleton = nullptr; +static UInitOnce initOnce {}; + +} // namespace + +U_CDECL_BEGIN + +static UBool U_CALLCONV uprv_collation_root_cleanup() { + SharedObject::clearPtr(rootSingleton); + initOnce.reset(); + return true; +} + +U_CDECL_END + +UDataMemory* +CollationRoot::loadFromFile(const char* ucadataPath, UErrorCode &errorCode) { + UDataMemory dataMemory; + UDataMemory *rDataMem = nullptr; + if (U_FAILURE(errorCode)) { + return nullptr; + } + if (uprv_mapFile(&dataMemory, ucadataPath, &errorCode)) { + if (dataMemory.pHeader->dataHeader.magic1 == 0xda && + dataMemory.pHeader->dataHeader.magic2 == 0x27 && + CollationDataReader::isAcceptable(nullptr, "icu", "ucadata", &dataMemory.pHeader->info)) { + rDataMem = UDataMemory_createNewInstance(&errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + rDataMem->pHeader = dataMemory.pHeader; + rDataMem->mapAddr = dataMemory.mapAddr; + rDataMem->map = dataMemory.map; + return rDataMem; + } + errorCode = U_INVALID_FORMAT_ERROR; + return nullptr; + } + errorCode = U_MISSING_RESOURCE_ERROR; + return nullptr; +} + +void U_CALLCONV +CollationRoot::load(const char* ucadataPath, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + LocalPointer t(new CollationTailoring(nullptr)); + if(t.isNull() || t->isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + t->memory = ucadataPath ? CollationRoot::loadFromFile(ucadataPath, errorCode) : + udata_openChoice(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll", + "icu", "ucadata", + CollationDataReader::isAcceptable, + t->version, &errorCode); + if(U_FAILURE(errorCode)) { return; } + const uint8_t *inBytes = static_cast(udata_getMemory(t->memory)); + CollationDataReader::read(nullptr, inBytes, udata_getLength(t->memory), *t, errorCode); + if(U_FAILURE(errorCode)) { return; } + ucln_i18n_registerCleanup(UCLN_I18N_COLLATION_ROOT, uprv_collation_root_cleanup); + CollationCacheEntry *entry = new CollationCacheEntry(Locale::getRoot(), t.getAlias()); + if(entry != nullptr) { + t.orphan(); // The rootSingleton took ownership of the tailoring. + entry->addRef(); + rootSingleton = entry; + } +} + +const CollationCacheEntry * +CollationRoot::getRootCacheEntry(UErrorCode &errorCode) { + umtx_initOnce(initOnce, CollationRoot::load, static_cast(nullptr), errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + return rootSingleton; +} + +const CollationTailoring * +CollationRoot::getRoot(UErrorCode &errorCode) { + umtx_initOnce(initOnce, CollationRoot::load, static_cast(nullptr), errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + return rootSingleton->tailoring; +} + +const CollationData * +CollationRoot::getData(UErrorCode &errorCode) { + const CollationTailoring *root = getRoot(errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + return root->data; +} + +const CollationSettings * +CollationRoot::getSettings(UErrorCode &errorCode) { + const CollationTailoring *root = getRoot(errorCode); + if(U_FAILURE(errorCode)) { return nullptr; } + return root->settings; +} + +void +CollationRoot::forceLoadFromFile(const char* ucadataPath, UErrorCode &errorCode) { + umtx_initOnce(initOnce, CollationRoot::load, ucadataPath, errorCode); +} + + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationroot.h b/intl/icu/source/i18n/collationroot.h new file mode 100644 index 0000000000..b203f612b3 --- /dev/null +++ b/intl/icu/source/i18n/collationroot.h @@ -0,0 +1,48 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2012-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationroot.h +* +* created on: 2012dec17 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONROOT_H__ +#define __COLLATIONROOT_H__ + +#include "unicode/utypes.h" +#include "unicode/udata.h" + +#if !UCONFIG_NO_COLLATION + +U_NAMESPACE_BEGIN + +struct CollationCacheEntry; +struct CollationData; +struct CollationSettings; +struct CollationTailoring; + +/** + * Collation root provider. + */ +class U_I18N_API CollationRoot { // purely static +public: + static const CollationCacheEntry *getRootCacheEntry(UErrorCode &errorCode); + static const CollationTailoring *getRoot(UErrorCode &errorCode); + static const CollationData *getData(UErrorCode &errorCode); + static const CollationSettings *getSettings(UErrorCode &errorCode); + static void U_EXPORT2 forceLoadFromFile(const char* ucadataPath, UErrorCode &errorCode); + +private: + static void U_CALLCONV load(const char* ucadataPath, UErrorCode &errorCode); + static UDataMemory* loadFromFile(const char* ucadataPath, UErrorCode &errorCode); +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONROOT_H__ diff --git a/intl/icu/source/i18n/collationrootelements.cpp b/intl/icu/source/i18n/collationrootelements.cpp new file mode 100644 index 0000000000..9b46d14144 --- /dev/null +++ b/intl/icu/source/i18n/collationrootelements.cpp @@ -0,0 +1,341 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationrootelements.cpp +* +* created on: 2013mar05 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "collation.h" +#include "collationrootelements.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +int64_t +CollationRootElements::lastCEWithPrimaryBefore(uint32_t p) const { + if(p == 0) { return 0; } + U_ASSERT(p > elements[elements[IX_FIRST_PRIMARY_INDEX]]); + int32_t index = findP(p); + uint32_t q = elements[index]; + uint32_t secTer; + if(p == (q & 0xffffff00)) { + // p == elements[index] is a root primary. Find the CE before it. + // We must not be in a primary range. + U_ASSERT((q & PRIMARY_STEP_MASK) == 0); + secTer = elements[index - 1]; + if((secTer & SEC_TER_DELTA_FLAG) == 0) { + // Primary CE just before p. + p = secTer & 0xffffff00; + secTer = Collation::COMMON_SEC_AND_TER_CE; + } else { + // secTer = last secondary & tertiary for the previous primary + index -= 2; + for(;;) { + p = elements[index]; + if((p & SEC_TER_DELTA_FLAG) == 0) { + p &= 0xffffff00; + break; + } + --index; + } + } + } else { + // p > elements[index] which is the previous primary. + // Find the last secondary & tertiary weights for it. + p = q & 0xffffff00; + secTer = Collation::COMMON_SEC_AND_TER_CE; + for(;;) { + q = elements[++index]; + if((q & SEC_TER_DELTA_FLAG) == 0) { + // We must not be in a primary range. + U_ASSERT((q & PRIMARY_STEP_MASK) == 0); + break; + } + secTer = q; + } + } + return ((int64_t)p << 32) | (secTer & ~SEC_TER_DELTA_FLAG); +} + +int64_t +CollationRootElements::firstCEWithPrimaryAtLeast(uint32_t p) const { + if(p == 0) { return 0; } + int32_t index = findP(p); + if(p != (elements[index] & 0xffffff00)) { + for(;;) { + p = elements[++index]; + if((p & SEC_TER_DELTA_FLAG) == 0) { + // First primary after p. We must not be in a primary range. + U_ASSERT((p & PRIMARY_STEP_MASK) == 0); + break; + } + } + } + // The code above guarantees that p has at most 3 bytes: (p & 0xff) == 0. + return ((int64_t)p << 32) | Collation::COMMON_SEC_AND_TER_CE; +} + +uint32_t +CollationRootElements::getPrimaryBefore(uint32_t p, UBool isCompressible) const { + int32_t index = findPrimary(p); + int32_t step; + uint32_t q = elements[index]; + if(p == (q & 0xffffff00)) { + // Found p itself. Return the previous primary. + // See if p is at the end of a previous range. + step = (int32_t)q & PRIMARY_STEP_MASK; + if(step == 0) { + // p is not at the end of a range. Look for the previous primary. + do { + p = elements[--index]; + } while((p & SEC_TER_DELTA_FLAG) != 0); + return p & 0xffffff00; + } + } else { + // p is in a range, and not at the start. + uint32_t nextElement = elements[index + 1]; + U_ASSERT(isEndOfPrimaryRange(nextElement)); + step = (int32_t)nextElement & PRIMARY_STEP_MASK; + } + // Return the previous range primary. + if((p & 0xffff) == 0) { + return Collation::decTwoBytePrimaryByOneStep(p, isCompressible, step); + } else { + return Collation::decThreeBytePrimaryByOneStep(p, isCompressible, step); + } +} + +uint32_t +CollationRootElements::getSecondaryBefore(uint32_t p, uint32_t s) const { + int32_t index; + uint32_t previousSec, sec; + if(p == 0) { + index = (int32_t)elements[IX_FIRST_SECONDARY_INDEX]; + // Gap at the beginning of the secondary CE range. + previousSec = 0; + sec = elements[index] >> 16; + } else { + index = findPrimary(p) + 1; + previousSec = Collation::BEFORE_WEIGHT16; + sec = getFirstSecTerForPrimary(index) >> 16; + } + U_ASSERT(s >= sec); + while(s > sec) { + previousSec = sec; + U_ASSERT((elements[index] & SEC_TER_DELTA_FLAG) != 0); + sec = elements[index++] >> 16; + } + U_ASSERT(sec == s); + return previousSec; +} + +uint32_t +CollationRootElements::getTertiaryBefore(uint32_t p, uint32_t s, uint32_t t) const { + U_ASSERT((t & ~Collation::ONLY_TERTIARY_MASK) == 0); + int32_t index; + uint32_t previousTer, secTer; + if(p == 0) { + if(s == 0) { + index = (int32_t)elements[IX_FIRST_TERTIARY_INDEX]; + // Gap at the beginning of the tertiary CE range. + previousTer = 0; + } else { + index = (int32_t)elements[IX_FIRST_SECONDARY_INDEX]; + previousTer = Collation::BEFORE_WEIGHT16; + } + secTer = elements[index] & ~SEC_TER_DELTA_FLAG; + } else { + index = findPrimary(p) + 1; + previousTer = Collation::BEFORE_WEIGHT16; + secTer = getFirstSecTerForPrimary(index); + } + uint32_t st = (s << 16) | t; + while(st > secTer) { + if((secTer >> 16) == s) { previousTer = secTer; } + U_ASSERT((elements[index] & SEC_TER_DELTA_FLAG) != 0); + secTer = elements[index++] & ~SEC_TER_DELTA_FLAG; + } + U_ASSERT(secTer == st); + return previousTer & 0xffff; +} + +uint32_t +CollationRootElements::getPrimaryAfter(uint32_t p, int32_t index, UBool isCompressible) const { + U_ASSERT(p == (elements[index] & 0xffffff00) || isEndOfPrimaryRange(elements[index + 1])); + uint32_t q = elements[++index]; + int32_t step; + if((q & SEC_TER_DELTA_FLAG) == 0 && (step = (int32_t)q & PRIMARY_STEP_MASK) != 0) { + // Return the next primary in this range. + if((p & 0xffff) == 0) { + return Collation::incTwoBytePrimaryByOffset(p, isCompressible, step); + } else { + return Collation::incThreeBytePrimaryByOffset(p, isCompressible, step); + } + } else { + // Return the next primary in the list. + while((q & SEC_TER_DELTA_FLAG) != 0) { + q = elements[++index]; + } + U_ASSERT((q & PRIMARY_STEP_MASK) == 0); + return q; + } +} + +uint32_t +CollationRootElements::getSecondaryAfter(int32_t index, uint32_t s) const { + uint32_t secTer; + uint32_t secLimit; + if(index == 0) { + // primary = 0 + U_ASSERT(s != 0); + index = (int32_t)elements[IX_FIRST_SECONDARY_INDEX]; + secTer = elements[index]; + // Gap at the end of the secondary CE range. + secLimit = 0x10000; + } else { + U_ASSERT(index >= (int32_t)elements[IX_FIRST_PRIMARY_INDEX]); + secTer = getFirstSecTerForPrimary(index + 1); + // If this is an explicit sec/ter unit, then it will be read once more. + // Gap for secondaries of primary CEs. + secLimit = getSecondaryBoundary(); + } + for(;;) { + uint32_t sec = secTer >> 16; + if(sec > s) { return sec; } + secTer = elements[++index]; + if((secTer & SEC_TER_DELTA_FLAG) == 0) { return secLimit; } + } +} + +uint32_t +CollationRootElements::getTertiaryAfter(int32_t index, uint32_t s, uint32_t t) const { + uint32_t secTer; + uint32_t terLimit; + if(index == 0) { + // primary = 0 + if(s == 0) { + U_ASSERT(t != 0); + index = (int32_t)elements[IX_FIRST_TERTIARY_INDEX]; + // Gap at the end of the tertiary CE range. + terLimit = 0x4000; + } else { + index = (int32_t)elements[IX_FIRST_SECONDARY_INDEX]; + // Gap for tertiaries of primary/secondary CEs. + terLimit = getTertiaryBoundary(); + } + secTer = elements[index] & ~SEC_TER_DELTA_FLAG; + } else { + U_ASSERT(index >= (int32_t)elements[IX_FIRST_PRIMARY_INDEX]); + secTer = getFirstSecTerForPrimary(index + 1); + // If this is an explicit sec/ter unit, then it will be read once more. + terLimit = getTertiaryBoundary(); + } + uint32_t st = (s << 16) | t; + for(;;) { + if(secTer > st) { + U_ASSERT((secTer >> 16) == s); + return secTer & 0xffff; + } + secTer = elements[++index]; + // No tertiary greater than t for this primary+secondary. + if((secTer & SEC_TER_DELTA_FLAG) == 0 || (secTer >> 16) > s) { return terLimit; } + secTer &= ~SEC_TER_DELTA_FLAG; + } +} + +uint32_t +CollationRootElements::getFirstSecTerForPrimary(int32_t index) const { + uint32_t secTer = elements[index]; + if((secTer & SEC_TER_DELTA_FLAG) == 0) { + // No sec/ter delta. + return Collation::COMMON_SEC_AND_TER_CE; + } + secTer &= ~SEC_TER_DELTA_FLAG; + if(secTer > Collation::COMMON_SEC_AND_TER_CE) { + // Implied sec/ter. + return Collation::COMMON_SEC_AND_TER_CE; + } + // Explicit sec/ter below common/common. + return secTer; +} + +int32_t +CollationRootElements::findPrimary(uint32_t p) const { + // Requirement: p must occur as a root primary. + U_ASSERT((p & 0xff) == 0); // at most a 3-byte primary + int32_t index = findP(p); + // If p is in a range, then we just assume that p is an actual primary in this range. + // (Too cumbersome/expensive to check.) + // Otherwise, it must be an exact match. + U_ASSERT(isEndOfPrimaryRange(elements[index + 1]) || p == (elements[index] & 0xffffff00)); + return index; +} + +int32_t +CollationRootElements::findP(uint32_t p) const { + // p need not occur as a root primary. + // For example, it might be a reordering group boundary. + U_ASSERT((p >> 24) != Collation::UNASSIGNED_IMPLICIT_BYTE); + // modified binary search + int32_t start = (int32_t)elements[IX_FIRST_PRIMARY_INDEX]; + U_ASSERT(p >= elements[start]); + int32_t limit = length - 1; + U_ASSERT(elements[limit] >= PRIMARY_SENTINEL); + U_ASSERT(p < elements[limit]); + while((start + 1) < limit) { + // Invariant: elements[start] and elements[limit] are primaries, + // and elements[start]<=p<=elements[limit]. + int32_t i = (start + limit) / 2; + uint32_t q = elements[i]; + if((q & SEC_TER_DELTA_FLAG) != 0) { + // Find the next primary. + int32_t j = i + 1; + for(;;) { + if(j == limit) { break; } + q = elements[j]; + if((q & SEC_TER_DELTA_FLAG) == 0) { + i = j; + break; + } + ++j; + } + if((q & SEC_TER_DELTA_FLAG) != 0) { + // Find the preceding primary. + j = i - 1; + for(;;) { + if(j == start) { break; } + q = elements[j]; + if((q & SEC_TER_DELTA_FLAG) == 0) { + i = j; + break; + } + --j; + } + if((q & SEC_TER_DELTA_FLAG) != 0) { + // No primary between start and limit. + break; + } + } + } + if(p < (q & 0xffffff00)) { // Reset the "step" bits of a range end primary. + limit = i; + } else { + start = i; + } + } + return start; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationrootelements.h b/intl/icu/source/i18n/collationrootelements.h new file mode 100644 index 0000000000..7836d8d83b --- /dev/null +++ b/intl/icu/source/i18n/collationrootelements.h @@ -0,0 +1,276 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationrootelements.h +* +* created on: 2013mar01 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONROOTELEMENTS_H__ +#define __COLLATIONROOTELEMENTS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/uobject.h" +#include "collation.h" + +U_NAMESPACE_BEGIN + +/** + * Container and access methods for collation elements and weights + * that occur in the root collator. + * Needed for finding boundaries for building a tailoring. + * + * This class takes and returns 16-bit secondary and tertiary weights. + */ +class U_I18N_API CollationRootElements : public UMemory { +public: + CollationRootElements(const uint32_t *rootElements, int32_t rootElementsLength) + : elements(rootElements), length(rootElementsLength) {} + + /** + * Higher than any root primary. + */ + static const uint32_t PRIMARY_SENTINEL = 0xffffff00; + + /** + * Flag in a root element, set if the element contains secondary & tertiary weights, + * rather than a primary. + */ + static const uint32_t SEC_TER_DELTA_FLAG = 0x80; + /** + * Mask for getting the primary range step value from a primary-range-end element. + */ + static const uint8_t PRIMARY_STEP_MASK = 0x7f; + + enum { + /** + * Index of the first CE with a non-zero tertiary weight. + * Same as the start of the compact root elements table. + */ + IX_FIRST_TERTIARY_INDEX, + /** + * Index of the first CE with a non-zero secondary weight. + */ + IX_FIRST_SECONDARY_INDEX, + /** + * Index of the first CE with a non-zero primary weight. + */ + IX_FIRST_PRIMARY_INDEX, + /** + * Must match Collation::COMMON_SEC_AND_TER_CE. + */ + IX_COMMON_SEC_AND_TER_CE, + /** + * Secondary & tertiary boundaries. + * Bits 31..24: [fixed last secondary common byte 45] + * Bits 23..16: [fixed first ignorable secondary byte 80] + * Bits 15.. 8: reserved, 0 + * Bits 7.. 0: [fixed first ignorable tertiary byte 3C] + */ + IX_SEC_TER_BOUNDARIES, + /** + * The current number of indexes. + * Currently the same as elements[IX_FIRST_TERTIARY_INDEX]. + */ + IX_COUNT + }; + + /** + * Returns the boundary between tertiary weights of primary/secondary CEs + * and those of tertiary CEs. + * This is the upper limit for tertiaries of primary/secondary CEs. + * This minus one is the lower limit for tertiaries of tertiary CEs. + */ + uint32_t getTertiaryBoundary() const { + return (elements[IX_SEC_TER_BOUNDARIES] << 8) & 0xff00; + } + + /** + * Returns the first assigned tertiary CE. + */ + uint32_t getFirstTertiaryCE() const { + return elements[elements[IX_FIRST_TERTIARY_INDEX]] & ~SEC_TER_DELTA_FLAG; + } + + /** + * Returns the last assigned tertiary CE. + */ + uint32_t getLastTertiaryCE() const { + return elements[elements[IX_FIRST_SECONDARY_INDEX] - 1] & ~SEC_TER_DELTA_FLAG; + } + + /** + * Returns the last common secondary weight. + * This is the lower limit for secondaries of primary CEs. + */ + uint32_t getLastCommonSecondary() const { + return (elements[IX_SEC_TER_BOUNDARIES] >> 16) & 0xff00; + } + + /** + * Returns the boundary between secondary weights of primary CEs + * and those of secondary CEs. + * This is the upper limit for secondaries of primary CEs. + * This minus one is the lower limit for secondaries of secondary CEs. + */ + uint32_t getSecondaryBoundary() const { + return (elements[IX_SEC_TER_BOUNDARIES] >> 8) & 0xff00; + } + + /** + * Returns the first assigned secondary CE. + */ + uint32_t getFirstSecondaryCE() const { + return elements[elements[IX_FIRST_SECONDARY_INDEX]] & ~SEC_TER_DELTA_FLAG; + } + + /** + * Returns the last assigned secondary CE. + */ + uint32_t getLastSecondaryCE() const { + return elements[elements[IX_FIRST_PRIMARY_INDEX] - 1] & ~SEC_TER_DELTA_FLAG; + } + + /** + * Returns the first assigned primary weight. + */ + uint32_t getFirstPrimary() const { + return elements[elements[IX_FIRST_PRIMARY_INDEX]]; // step=0: cannot be a range end + } + + /** + * Returns the first assigned primary CE. + */ + int64_t getFirstPrimaryCE() const { + return Collation::makeCE(getFirstPrimary()); + } + + /** + * Returns the last root CE with a primary weight before p. + * Intended only for reordering group boundaries. + */ + int64_t lastCEWithPrimaryBefore(uint32_t p) const; + + /** + * Returns the first root CE with a primary weight of at least p. + * Intended only for reordering group boundaries. + */ + int64_t firstCEWithPrimaryAtLeast(uint32_t p) const; + + /** + * Returns the primary weight before p. + * p must be greater than the first root primary. + */ + uint32_t getPrimaryBefore(uint32_t p, UBool isCompressible) const; + + /** Returns the secondary weight before [p, s]. */ + uint32_t getSecondaryBefore(uint32_t p, uint32_t s) const; + + /** Returns the tertiary weight before [p, s, t]. */ + uint32_t getTertiaryBefore(uint32_t p, uint32_t s, uint32_t t) const; + + /** + * Finds the index of the input primary. + * p must occur as a root primary, and must not be 0. + */ + int32_t findPrimary(uint32_t p) const; + + /** + * Returns the primary weight after p where index=findPrimary(p). + * p must be at least the first root primary. + */ + uint32_t getPrimaryAfter(uint32_t p, int32_t index, UBool isCompressible) const; + /** + * Returns the secondary weight after [p, s] where index=findPrimary(p) + * except use index=0 for p=0. + * + * Must return a weight for every root [p, s] as well as for every weight + * returned by getSecondaryBefore(). If p!=0 then s can be BEFORE_WEIGHT16. + * + * Exception: [0, 0] is handled by the CollationBuilder: + * Both its lower and upper boundaries are special. + */ + uint32_t getSecondaryAfter(int32_t index, uint32_t s) const; + /** + * Returns the tertiary weight after [p, s, t] where index=findPrimary(p) + * except use index=0 for p=0. + * + * Must return a weight for every root [p, s, t] as well as for every weight + * returned by getTertiaryBefore(). If s!=0 then t can be BEFORE_WEIGHT16. + * + * Exception: [0, 0, 0] is handled by the CollationBuilder: + * Both its lower and upper boundaries are special. + */ + uint32_t getTertiaryAfter(int32_t index, uint32_t s, uint32_t t) const; + +private: + /** + * Returns the first secondary & tertiary weights for p where index=findPrimary(p)+1. + */ + uint32_t getFirstSecTerForPrimary(int32_t index) const; + + /** + * Finds the largest index i where elements[i]<=p. + * Requires first primary<=p<0xffffff00 (PRIMARY_SENTINEL). + * Does not require that p is a root collator primary. + */ + int32_t findP(uint32_t p) const; + + static inline UBool isEndOfPrimaryRange(uint32_t q) { + return (q & SEC_TER_DELTA_FLAG) == 0 && (q & PRIMARY_STEP_MASK) != 0; + } + + /** + * Data structure: + * + * The first few entries are indexes, up to elements[IX_FIRST_TERTIARY_INDEX]. + * See the comments on the IX_ constants. + * + * All other elements are a compact form of the root collator CEs + * in mostly collation order. + * + * A sequence of one or more root CEs with the same primary weight is stored as + * one element with the primary weight, with the SEC_TER_DELTA_FLAG flag not set, + * followed by elements with only the secondary/tertiary weights, + * each with that flag set. + * If the lowest secondary/tertiary combination is Collation::COMMON_SEC_AND_TER_CE, + * then the element for that combination is omitted. + * + * Note: If the first actual secondary/tertiary combination is higher than + * Collation::COMMON_SEC_AND_TER_CE (which is unusual), + * the runtime code will assume anyway that Collation::COMMON_SEC_AND_TER_CE is present. + * + * A range of only-primary CEs with a consistent "step" increment + * from each primary to the next may be stored as a range. + * Only the first and last primary are stored, and the last has the step + * value in the low bits (PRIMARY_STEP_MASK). + * + * An range-end element may also either start a new range or be followed by + * elements with secondary/tertiary deltas. + * + * A primary element that is not a range end has zero step bits. + * + * There is no element for the completely ignorable CE (all weights 0). + * + * Before elements[IX_FIRST_PRIMARY_INDEX], all elements are secondary/tertiary deltas, + * for all of the ignorable root CEs. + * + * There are no elements for unassigned-implicit primary CEs. + * All primaries stored here are at most 3 bytes long. + */ + const uint32_t *elements; + int32_t length; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONROOTELEMENTS_H__ diff --git a/intl/icu/source/i18n/collationruleparser.cpp b/intl/icu/source/i18n/collationruleparser.cpp new file mode 100644 index 0000000000..4cc25a1f5c --- /dev/null +++ b/intl/icu/source/i18n/collationruleparser.cpp @@ -0,0 +1,881 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationruleparser.cpp +* +* (replaced the former ucol_tok.cpp) +* +* created on: 2013apr10 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/normalizer2.h" +#include "unicode/parseerr.h" +#include "unicode/uchar.h" +#include "unicode/ucol.h" +#include "unicode/uloc.h" +#include "unicode/unistr.h" +#include "unicode/utf16.h" +#include "charstr.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationruleparser.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "cstring.h" +#include "patternprops.h" +#include "uassert.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +namespace { + +static const char16_t BEFORE[] = { 0x5b, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0 }; // "[before" +const int32_t BEFORE_LENGTH = 7; + +} // namespace + +CollationRuleParser::Sink::~Sink() {} + +void +CollationRuleParser::Sink::suppressContractions(const UnicodeSet &, const char *&, UErrorCode &) {} + +void +CollationRuleParser::Sink::optimize(const UnicodeSet &, const char *&, UErrorCode &) {} + +CollationRuleParser::Importer::~Importer() {} + +CollationRuleParser::CollationRuleParser(const CollationData *base, UErrorCode &errorCode) + : nfd(*Normalizer2::getNFDInstance(errorCode)), + nfc(*Normalizer2::getNFCInstance(errorCode)), + rules(nullptr), baseData(base), settings(nullptr), + parseError(nullptr), errorReason(nullptr), + sink(nullptr), importer(nullptr), + ruleIndex(0) { +} + +CollationRuleParser::~CollationRuleParser() { +} + +void +CollationRuleParser::parse(const UnicodeString &ruleString, + CollationSettings &outSettings, + UParseError *outParseError, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + settings = &outSettings; + parseError = outParseError; + if(parseError != nullptr) { + parseError->line = 0; + parseError->offset = -1; + parseError->preContext[0] = 0; + parseError->postContext[0] = 0; + } + errorReason = nullptr; + parse(ruleString, errorCode); +} + +void +CollationRuleParser::parse(const UnicodeString &ruleString, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + rules = &ruleString; + ruleIndex = 0; + + while(ruleIndex < rules->length()) { + char16_t c = rules->charAt(ruleIndex); + if(PatternProps::isWhiteSpace(c)) { + ++ruleIndex; + continue; + } + switch(c) { + case 0x26: // '&' + parseRuleChain(errorCode); + break; + case 0x5b: // '[' + parseSetting(errorCode); + break; + case 0x23: // '#' starts a comment, until the end of the line + ruleIndex = skipComment(ruleIndex + 1); + break; + case 0x40: // '@' is equivalent to [backwards 2] + settings->setFlag(CollationSettings::BACKWARD_SECONDARY, + UCOL_ON, 0, errorCode); + ++ruleIndex; + break; + case 0x21: // '!' used to turn on Thai/Lao character reversal + // Accept but ignore. The root collator has contractions + // that are equivalent to the character reversal, where appropriate. + ++ruleIndex; + break; + default: + setParseError("expected a reset or setting or comment", errorCode); + break; + } + if(U_FAILURE(errorCode)) { return; } + } +} + +void +CollationRuleParser::parseRuleChain(UErrorCode &errorCode) { + int32_t resetStrength = parseResetAndPosition(errorCode); + UBool isFirstRelation = true; + for(;;) { + int32_t result = parseRelationOperator(errorCode); + if(U_FAILURE(errorCode)) { return; } + if(result < 0) { + if(ruleIndex < rules->length() && rules->charAt(ruleIndex) == 0x23) { + // '#' starts a comment, until the end of the line + ruleIndex = skipComment(ruleIndex + 1); + continue; + } + if(isFirstRelation) { + setParseError("reset not followed by a relation", errorCode); + } + return; + } + int32_t strength = result & STRENGTH_MASK; + if(resetStrength < UCOL_IDENTICAL) { + // reset-before rule chain + if(isFirstRelation) { + if(strength != resetStrength) { + setParseError("reset-before strength differs from its first relation", errorCode); + return; + } + } else { + if(strength < resetStrength) { + setParseError("reset-before strength followed by a stronger relation", errorCode); + return; + } + } + } + int32_t i = ruleIndex + (result >> OFFSET_SHIFT); // skip over the relation operator + if((result & STARRED_FLAG) == 0) { + parseRelationStrings(strength, i, errorCode); + } else { + parseStarredCharacters(strength, i, errorCode); + } + if(U_FAILURE(errorCode)) { return; } + isFirstRelation = false; + } +} + +int32_t +CollationRuleParser::parseResetAndPosition(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } + int32_t i = skipWhiteSpace(ruleIndex + 1); + int32_t j; + char16_t c; + int32_t resetStrength; + if(rules->compare(i, BEFORE_LENGTH, BEFORE, 0, BEFORE_LENGTH) == 0 && + (j = i + BEFORE_LENGTH) < rules->length() && + PatternProps::isWhiteSpace(rules->charAt(j)) && + ((j = skipWhiteSpace(j + 1)) + 1) < rules->length() && + 0x31 <= (c = rules->charAt(j)) && c <= 0x33 && + rules->charAt(j + 1) == 0x5d) { + // &[before n] with n=1 or 2 or 3 + resetStrength = UCOL_PRIMARY + (c - 0x31); + i = skipWhiteSpace(j + 2); + } else { + resetStrength = UCOL_IDENTICAL; + } + if(i >= rules->length()) { + setParseError("reset without position", errorCode); + return UCOL_DEFAULT; + } + UnicodeString str; + if(rules->charAt(i) == 0x5b) { // '[' + i = parseSpecialPosition(i, str, errorCode); + } else { + i = parseTailoringString(i, str, errorCode); + } + sink->addReset(resetStrength, str, errorReason, errorCode); + if(U_FAILURE(errorCode)) { setErrorContext(); } + ruleIndex = i; + return resetStrength; +} + +int32_t +CollationRuleParser::parseRelationOperator(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } + ruleIndex = skipWhiteSpace(ruleIndex); + if(ruleIndex >= rules->length()) { return UCOL_DEFAULT; } + int32_t strength; + int32_t i = ruleIndex; + char16_t c = rules->charAt(i++); + switch(c) { + case 0x3c: // '<' + if(i < rules->length() && rules->charAt(i) == 0x3c) { // << + ++i; + if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<< + ++i; + if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<<< + ++i; + strength = UCOL_QUATERNARY; + } else { + strength = UCOL_TERTIARY; + } + } else { + strength = UCOL_SECONDARY; + } + } else { + strength = UCOL_PRIMARY; + } + if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' + ++i; + strength |= STARRED_FLAG; + } + break; + case 0x3b: // ';' same as << + strength = UCOL_SECONDARY; + break; + case 0x2c: // ',' same as <<< + strength = UCOL_TERTIARY; + break; + case 0x3d: // '=' + strength = UCOL_IDENTICAL; + if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*' + ++i; + strength |= STARRED_FLAG; + } + break; + default: + return UCOL_DEFAULT; + } + return ((i - ruleIndex) << OFFSET_SHIFT) | strength; +} + +void +CollationRuleParser::parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode) { + // Parse + // prefix | str / extension + // where prefix and extension are optional. + UnicodeString prefix, str, extension; + i = parseTailoringString(i, str, errorCode); + if(U_FAILURE(errorCode)) { return; } + char16_t next = (i < rules->length()) ? rules->charAt(i) : 0; + if(next == 0x7c) { // '|' separates the context prefix from the string. + prefix = str; + i = parseTailoringString(i + 1, str, errorCode); + if(U_FAILURE(errorCode)) { return; } + next = (i < rules->length()) ? rules->charAt(i) : 0; + } + if(next == 0x2f) { // '/' separates the string from the extension. + i = parseTailoringString(i + 1, extension, errorCode); + } + if(!prefix.isEmpty()) { + UChar32 prefix0 = prefix.char32At(0); + UChar32 c = str.char32At(0); + if(!nfc.hasBoundaryBefore(prefix0) || !nfc.hasBoundaryBefore(c)) { + setParseError("in 'prefix|str', prefix and str must each start with an NFC boundary", + errorCode); + return; + } + } + sink->addRelation(strength, prefix, str, extension, errorReason, errorCode); + if(U_FAILURE(errorCode)) { setErrorContext(); } + ruleIndex = i; +} + +void +CollationRuleParser::parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode) { + UnicodeString empty, raw; + i = parseString(skipWhiteSpace(i), raw, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(raw.isEmpty()) { + setParseError("missing starred-relation string", errorCode); + return; + } + UChar32 prev = -1; + int32_t j = 0; + for(;;) { + while(j < raw.length()) { + UChar32 c = raw.char32At(j); + if(!nfd.isInert(c)) { + setParseError("starred-relation string is not all NFD-inert", errorCode); + return; + } + sink->addRelation(strength, empty, UnicodeString(c), empty, errorReason, errorCode); + if(U_FAILURE(errorCode)) { + setErrorContext(); + return; + } + j += U16_LENGTH(c); + prev = c; + } + if(i >= rules->length() || rules->charAt(i) != 0x2d) { // '-' + break; + } + if(prev < 0) { + setParseError("range without start in starred-relation string", errorCode); + return; + } + i = parseString(i + 1, raw, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(raw.isEmpty()) { + setParseError("range without end in starred-relation string", errorCode); + return; + } + UChar32 c = raw.char32At(0); + if(c < prev) { + setParseError("range start greater than end in starred-relation string", errorCode); + return; + } + // range prev-c + UnicodeString s; + while(++prev <= c) { + if(!nfd.isInert(prev)) { + setParseError("starred-relation string range is not all NFD-inert", errorCode); + return; + } + if(U_IS_SURROGATE(prev)) { + setParseError("starred-relation string range contains a surrogate", errorCode); + return; + } + if(0xfffd <= prev && prev <= 0xffff) { + setParseError("starred-relation string range contains U+FFFD, U+FFFE or U+FFFF", errorCode); + return; + } + s.setTo(prev); + sink->addRelation(strength, empty, s, empty, errorReason, errorCode); + if(U_FAILURE(errorCode)) { + setErrorContext(); + return; + } + } + prev = -1; + j = U16_LENGTH(c); + } + ruleIndex = skipWhiteSpace(i); +} + +int32_t +CollationRuleParser::parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { + i = parseString(skipWhiteSpace(i), raw, errorCode); + if(U_SUCCESS(errorCode) && raw.isEmpty()) { + setParseError("missing relation string", errorCode); + } + return skipWhiteSpace(i); +} + +int32_t +CollationRuleParser::parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return i; } + raw.remove(); + while(i < rules->length()) { + UChar32 c = rules->charAt(i++); + if(isSyntaxChar(c)) { + if(c == 0x27) { // apostrophe + if(i < rules->length() && rules->charAt(i) == 0x27) { + // Double apostrophe, encodes a single one. + raw.append((char16_t)0x27); + ++i; + continue; + } + // Quote literal text until the next single apostrophe. + for(;;) { + if(i == rules->length()) { + setParseError("quoted literal text missing terminating apostrophe", errorCode); + return i; + } + c = rules->charAt(i++); + if(c == 0x27) { + if(i < rules->length() && rules->charAt(i) == 0x27) { + // Double apostrophe inside quoted literal text, + // still encodes a single apostrophe. + ++i; + } else { + break; + } + } + raw.append((char16_t)c); + } + } else if(c == 0x5c) { // backslash + if(i == rules->length()) { + setParseError("backslash escape at the end of the rule string", errorCode); + return i; + } + c = rules->char32At(i); + raw.append(c); + i += U16_LENGTH(c); + } else { + // Any other syntax character terminates a string. + --i; + break; + } + } else if(PatternProps::isWhiteSpace(c)) { + // Unquoted white space terminates a string. + --i; + break; + } else { + raw.append((char16_t)c); + } + } + for(int32_t j = 0; j < raw.length();) { + UChar32 c = raw.char32At(j); + if(U_IS_SURROGATE(c)) { + setParseError("string contains an unpaired surrogate", errorCode); + return i; + } + if(0xfffd <= c && c <= 0xffff) { + setParseError("string contains U+FFFD, U+FFFE or U+FFFF", errorCode); + return i; + } + j += U16_LENGTH(c); + } + return i; +} + +namespace { + +static const char *const positions[] = { + "first tertiary ignorable", + "last tertiary ignorable", + "first secondary ignorable", + "last secondary ignorable", + "first primary ignorable", + "last primary ignorable", + "first variable", + "last variable", + "first regular", + "last regular", + "first implicit", + "last implicit", + "first trailing", + "last trailing" +}; + +} // namespace + +int32_t +CollationRuleParser::parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + UnicodeString raw; + int32_t j = readWords(i + 1, raw); + if(j > i && rules->charAt(j) == 0x5d && !raw.isEmpty()) { // words end with ] + ++j; + for(int32_t pos = 0; pos < UPRV_LENGTHOF(positions); ++pos) { + if(raw == UnicodeString(positions[pos], -1, US_INV)) { + str.setTo((char16_t)POS_LEAD).append((char16_t)(POS_BASE + pos)); + return j; + } + } + if(raw == UNICODE_STRING_SIMPLE("top")) { + str.setTo((char16_t)POS_LEAD).append((char16_t)(POS_BASE + LAST_REGULAR)); + return j; + } + if(raw == UNICODE_STRING_SIMPLE("variable top")) { + str.setTo((char16_t)POS_LEAD).append((char16_t)(POS_BASE + LAST_VARIABLE)); + return j; + } + } + setParseError("not a valid special reset position", errorCode); + return i; +} + +void +CollationRuleParser::parseSetting(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + UnicodeString raw; + int32_t i = ruleIndex + 1; + int32_t j = readWords(i, raw); + if(j <= i || raw.isEmpty()) { + setParseError("expected a setting/option at '['", errorCode); + } + if(rules->charAt(j) == 0x5d) { // words end with ] + ++j; + if(raw.startsWith(UNICODE_STRING_SIMPLE("reorder")) && + (raw.length() == 7 || raw.charAt(7) == 0x20)) { + parseReordering(raw, errorCode); + ruleIndex = j; + return; + } + if(raw == UNICODE_STRING_SIMPLE("backwards 2")) { + settings->setFlag(CollationSettings::BACKWARD_SECONDARY, + UCOL_ON, 0, errorCode); + ruleIndex = j; + return; + } + UnicodeString v; + int32_t valueIndex = raw.lastIndexOf((char16_t)0x20); + if(valueIndex >= 0) { + v.setTo(raw, valueIndex + 1); + raw.truncate(valueIndex); + } + if(raw == UNICODE_STRING_SIMPLE("strength") && v.length() == 1) { + int32_t value = UCOL_DEFAULT; + char16_t c = v.charAt(0); + if(0x31 <= c && c <= 0x34) { // 1..4 + value = UCOL_PRIMARY + (c - 0x31); + } else if(c == 0x49) { // 'I' + value = UCOL_IDENTICAL; + } + if(value != UCOL_DEFAULT) { + settings->setStrength(value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("alternate")) { + UColAttributeValue value = UCOL_DEFAULT; + if(v == UNICODE_STRING_SIMPLE("non-ignorable")) { + value = UCOL_NON_IGNORABLE; + } else if(v == UNICODE_STRING_SIMPLE("shifted")) { + value = UCOL_SHIFTED; + } + if(value != UCOL_DEFAULT) { + settings->setAlternateHandling(value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("maxVariable")) { + int32_t value = UCOL_DEFAULT; + if(v == UNICODE_STRING_SIMPLE("space")) { + value = CollationSettings::MAX_VAR_SPACE; + } else if(v == UNICODE_STRING_SIMPLE("punct")) { + value = CollationSettings::MAX_VAR_PUNCT; + } else if(v == UNICODE_STRING_SIMPLE("symbol")) { + value = CollationSettings::MAX_VAR_SYMBOL; + } else if(v == UNICODE_STRING_SIMPLE("currency")) { + value = CollationSettings::MAX_VAR_CURRENCY; + } + if(value != UCOL_DEFAULT) { + settings->setMaxVariable(value, 0, errorCode); + settings->variableTop = baseData->getLastPrimaryForGroup( + UCOL_REORDER_CODE_FIRST + value); + U_ASSERT(settings->variableTop != 0); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("caseFirst")) { + UColAttributeValue value = UCOL_DEFAULT; + if(v == UNICODE_STRING_SIMPLE("off")) { + value = UCOL_OFF; + } else if(v == UNICODE_STRING_SIMPLE("lower")) { + value = UCOL_LOWER_FIRST; + } else if(v == UNICODE_STRING_SIMPLE("upper")) { + value = UCOL_UPPER_FIRST; + } + if(value != UCOL_DEFAULT) { + settings->setCaseFirst(value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("caseLevel")) { + UColAttributeValue value = getOnOffValue(v); + if(value != UCOL_DEFAULT) { + settings->setFlag(CollationSettings::CASE_LEVEL, value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("normalization")) { + UColAttributeValue value = getOnOffValue(v); + if(value != UCOL_DEFAULT) { + settings->setFlag(CollationSettings::CHECK_FCD, value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("numericOrdering")) { + UColAttributeValue value = getOnOffValue(v); + if(value != UCOL_DEFAULT) { + settings->setFlag(CollationSettings::NUMERIC, value, 0, errorCode); + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("hiraganaQ")) { + UColAttributeValue value = getOnOffValue(v); + if(value != UCOL_DEFAULT) { + if(value == UCOL_ON) { + setParseError("[hiraganaQ on] is not supported", errorCode); + } + ruleIndex = j; + return; + } + } else if(raw == UNICODE_STRING_SIMPLE("import")) { + CharString lang; + lang.appendInvariantChars(v, errorCode); + if(errorCode == U_MEMORY_ALLOCATION_ERROR) { return; } + // BCP 47 language tag -> ICU locale ID + char localeID[ULOC_FULLNAME_CAPACITY]; + int32_t parsedLength; + int32_t length = uloc_forLanguageTag(lang.data(), localeID, ULOC_FULLNAME_CAPACITY, + &parsedLength, &errorCode); + if(U_FAILURE(errorCode) || + parsedLength != lang.length() || length >= ULOC_FULLNAME_CAPACITY) { + errorCode = U_ZERO_ERROR; + setParseError("expected language tag in [import langTag]", errorCode); + return; + } + // localeID minus all keywords + char baseID[ULOC_FULLNAME_CAPACITY]; + length = uloc_getBaseName(localeID, baseID, ULOC_FULLNAME_CAPACITY, &errorCode); + if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { + errorCode = U_ZERO_ERROR; + setParseError("expected language tag in [import langTag]", errorCode); + return; + } + if(length == 0) { + uprv_strcpy(baseID, "root"); + } else if(*baseID == '_') { + uprv_memmove(baseID + 3, baseID, length + 1); + uprv_memcpy(baseID, "und", 3); + } + // @collation=type, or length=0 if not specified + char collationType[ULOC_KEYWORDS_CAPACITY]; + length = uloc_getKeywordValue(localeID, "collation", + collationType, ULOC_KEYWORDS_CAPACITY, + &errorCode); + if(U_FAILURE(errorCode) || length >= ULOC_KEYWORDS_CAPACITY) { + errorCode = U_ZERO_ERROR; + setParseError("expected language tag in [import langTag]", errorCode); + return; + } + if(importer == nullptr) { + setParseError("[import langTag] is not supported", errorCode); + } else { + UnicodeString importedRules; + importer->getRules(baseID, length > 0 ? collationType : "standard", + importedRules, errorReason, errorCode); + if(U_FAILURE(errorCode)) { + if(errorReason == nullptr) { + errorReason = "[import langTag] failed"; + } + setErrorContext(); + return; + } + const UnicodeString *outerRules = rules; + int32_t outerRuleIndex = ruleIndex; + parse(importedRules, errorCode); + if(U_FAILURE(errorCode)) { + if(parseError != nullptr) { + parseError->offset = outerRuleIndex; + } + } + rules = outerRules; + ruleIndex = j; + } + return; + } + } else if(rules->charAt(j) == 0x5b) { // words end with [ + UnicodeSet set; + j = parseUnicodeSet(j, set, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(raw == UNICODE_STRING_SIMPLE("optimize")) { + sink->optimize(set, errorReason, errorCode); + if(U_FAILURE(errorCode)) { setErrorContext(); } + ruleIndex = j; + return; + } else if(raw == UNICODE_STRING_SIMPLE("suppressContractions")) { + sink->suppressContractions(set, errorReason, errorCode); + if(U_FAILURE(errorCode)) { setErrorContext(); } + ruleIndex = j; + return; + } + } + setParseError("not a valid setting/option", errorCode); +} + +void +CollationRuleParser::parseReordering(const UnicodeString &raw, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t i = 7; // after "reorder" + if(i == raw.length()) { + // empty [reorder] with no codes + settings->resetReordering(); + return; + } + // Parse the codes in [reorder aa bb cc]. + UVector32 reorderCodes(errorCode); + if(U_FAILURE(errorCode)) { return; } + CharString word; + while(i < raw.length()) { + ++i; // skip the word-separating space + int32_t limit = raw.indexOf((char16_t)0x20, i); + if(limit < 0) { limit = raw.length(); } + word.clear().appendInvariantChars(raw.tempSubStringBetween(i, limit), errorCode); + if(U_FAILURE(errorCode)) { return; } + int32_t code = getReorderCode(word.data()); + if(code < 0) { + setParseError("unknown script or reorder code", errorCode); + return; + } + reorderCodes.addElement(code, errorCode); + if(U_FAILURE(errorCode)) { return; } + i = limit; + } + settings->setReordering(*baseData, reorderCodes.getBuffer(), reorderCodes.size(), errorCode); +} + +static const char *const gSpecialReorderCodes[] = { + "space", "punct", "symbol", "currency", "digit" +}; + +int32_t +CollationRuleParser::getReorderCode(const char *word) { + for(int32_t i = 0; i < UPRV_LENGTHOF(gSpecialReorderCodes); ++i) { + if(uprv_stricmp(word, gSpecialReorderCodes[i]) == 0) { + return UCOL_REORDER_CODE_FIRST + i; + } + } + int32_t script = u_getPropertyValueEnum(UCHAR_SCRIPT, word); + if(script >= 0) { + return script; + } + if(uprv_stricmp(word, "others") == 0) { + return UCOL_REORDER_CODE_OTHERS; // same as Zzzz = USCRIPT_UNKNOWN + } + return -1; +} + +UColAttributeValue +CollationRuleParser::getOnOffValue(const UnicodeString &s) { + if(s == UNICODE_STRING_SIMPLE("on")) { + return UCOL_ON; + } else if(s == UNICODE_STRING_SIMPLE("off")) { + return UCOL_OFF; + } else { + return UCOL_DEFAULT; + } +} + +int32_t +CollationRuleParser::parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode) { + // Collect a UnicodeSet pattern between a balanced pair of [brackets]. + int32_t level = 0; + int32_t j = i; + for(;;) { + if(j == rules->length()) { + setParseError("unbalanced UnicodeSet pattern brackets", errorCode); + return j; + } + char16_t c = rules->charAt(j++); + if(c == 0x5b) { // '[' + ++level; + } else if(c == 0x5d) { // ']' + if(--level == 0) { break; } + } + } + set.applyPattern(rules->tempSubStringBetween(i, j), errorCode); + if(U_FAILURE(errorCode)) { + errorCode = U_ZERO_ERROR; + setParseError("not a valid UnicodeSet pattern", errorCode); + return j; + } + j = skipWhiteSpace(j); + if(j == rules->length() || rules->charAt(j) != 0x5d) { + setParseError("missing option-terminating ']' after UnicodeSet pattern", errorCode); + return j; + } + return ++j; +} + +int32_t +CollationRuleParser::readWords(int32_t i, UnicodeString &raw) const { + static const char16_t sp = 0x20; + raw.remove(); + i = skipWhiteSpace(i); + for(;;) { + if(i >= rules->length()) { return 0; } + char16_t c = rules->charAt(i); + if(isSyntaxChar(c) && c != 0x2d && c != 0x5f) { // syntax except -_ + if(raw.isEmpty()) { return i; } + if(raw.endsWith(&sp, 1)) { // remove trailing space + raw.truncate(raw.length() - 1); + } + return i; + } + if(PatternProps::isWhiteSpace(c)) { + raw.append(sp); + i = skipWhiteSpace(i + 1); + } else { + raw.append(c); + ++i; + } + } +} + +int32_t +CollationRuleParser::skipComment(int32_t i) const { + // skip to past the newline + while(i < rules->length()) { + char16_t c = rules->charAt(i++); + // LF or FF or CR or NEL or LS or PS + if(c == 0xa || c == 0xc || c == 0xd || c == 0x85 || c == 0x2028 || c == 0x2029) { + // Unicode Newline Guidelines: "A readline function should stop at NLF, LS, FF, or PS." + // NLF (new line function) = CR or LF or CR+LF or NEL. + // No need to collect all of CR+LF because a following LF will be ignored anyway. + break; + } + } + return i; +} + +void +CollationRuleParser::setParseError(const char *reason, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + // Error code consistent with the old parser (from ca. 2001), + // rather than U_PARSE_ERROR; + errorCode = U_INVALID_FORMAT_ERROR; + errorReason = reason; + if(parseError != nullptr) { setErrorContext(); } +} + +void +CollationRuleParser::setErrorContext() { + if(parseError == nullptr) { return; } + + // Note: This relies on the calling code maintaining the ruleIndex + // at a position that is useful for debugging. + // For example, at the beginning of a reset or relation etc. + parseError->offset = ruleIndex; + parseError->line = 0; // We are not counting line numbers. + + // before ruleIndex + int32_t start = ruleIndex - (U_PARSE_CONTEXT_LEN - 1); + if(start < 0) { + start = 0; + } else if(start > 0 && U16_IS_TRAIL(rules->charAt(start))) { + ++start; + } + int32_t length = ruleIndex - start; + rules->extract(start, length, parseError->preContext); + parseError->preContext[length] = 0; + + // starting from ruleIndex + length = rules->length() - ruleIndex; + if(length >= U_PARSE_CONTEXT_LEN) { + length = U_PARSE_CONTEXT_LEN - 1; + if(U16_IS_LEAD(rules->charAt(ruleIndex + length - 1))) { + --length; + } + } + rules->extract(ruleIndex, length, parseError->postContext); + parseError->postContext[length] = 0; +} + +UBool +CollationRuleParser::isSyntaxChar(UChar32 c) { + return 0x21 <= c && c <= 0x7e && + (c <= 0x2f || (0x3a <= c && c <= 0x40) || + (0x5b <= c && c <= 0x60) || (0x7b <= c)); +} + +int32_t +CollationRuleParser::skipWhiteSpace(int32_t i) const { + while(i < rules->length() && PatternProps::isWhiteSpace(rules->charAt(i))) { + ++i; + } + return i; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationruleparser.h b/intl/icu/source/i18n/collationruleparser.h new file mode 100644 index 0000000000..aabdf03f57 --- /dev/null +++ b/intl/icu/source/i18n/collationruleparser.h @@ -0,0 +1,197 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationruleparser.h +* +* created on: 2013apr10 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONRULEPARSER_H__ +#define __COLLATIONRULEPARSER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" + +struct UParseError; + +U_NAMESPACE_BEGIN + +struct CollationData; +struct CollationTailoring; + +class Locale; +class Normalizer2; + +struct CollationSettings; + +class U_I18N_API CollationRuleParser : public UMemory { +public: + /** Special reset positions. */ + enum Position { + FIRST_TERTIARY_IGNORABLE, + LAST_TERTIARY_IGNORABLE, + FIRST_SECONDARY_IGNORABLE, + LAST_SECONDARY_IGNORABLE, + FIRST_PRIMARY_IGNORABLE, + LAST_PRIMARY_IGNORABLE, + FIRST_VARIABLE, + LAST_VARIABLE, + FIRST_REGULAR, + LAST_REGULAR, + FIRST_IMPLICIT, + LAST_IMPLICIT, + FIRST_TRAILING, + LAST_TRAILING + }; + + /** + * First character of contractions that encode special reset positions. + * U+FFFE cannot be tailored via rule syntax. + * + * The second contraction character is POS_BASE + Position. + */ + static const char16_t POS_LEAD = 0xfffe; + /** + * Base for the second character of contractions that encode special reset positions. + * Braille characters U+28xx are printable and normalization-inert. + * @see POS_LEAD + */ + static const char16_t POS_BASE = 0x2800; + + class U_I18N_API Sink : public UObject { + public: + virtual ~Sink(); + /** + * Adds a reset. + * strength=UCOL_IDENTICAL for &str. + * strength=UCOL_PRIMARY/UCOL_SECONDARY/UCOL_TERTIARY for &[before n]str where n=1/2/3. + */ + virtual void addReset(int32_t strength, const UnicodeString &str, + const char *&errorReason, UErrorCode &errorCode) = 0; + /** + * Adds a relation with strength and prefix | str / extension. + */ + virtual void addRelation(int32_t strength, const UnicodeString &prefix, + const UnicodeString &str, const UnicodeString &extension, + const char *&errorReason, UErrorCode &errorCode) = 0; + + virtual void suppressContractions(const UnicodeSet &set, const char *&errorReason, + UErrorCode &errorCode); + + virtual void optimize(const UnicodeSet &set, const char *&errorReason, + UErrorCode &errorCode); + }; + + class U_I18N_API Importer : public UObject { + public: + virtual ~Importer(); + virtual void getRules( + const char *localeID, const char *collationType, + UnicodeString &rules, + const char *&errorReason, UErrorCode &errorCode) = 0; + }; + + /** + * Constructor. + * The Sink must be set before parsing. + * The Importer can be set, otherwise [import locale] syntax is not supported. + */ + CollationRuleParser(const CollationData *base, UErrorCode &errorCode); + ~CollationRuleParser(); + + /** + * Sets the pointer to a Sink object. + * The pointer is aliased: Pointer copy without cloning or taking ownership. + */ + void setSink(Sink *sinkAlias) { + sink = sinkAlias; + } + + /** + * Sets the pointer to an Importer object. + * The pointer is aliased: Pointer copy without cloning or taking ownership. + */ + void setImporter(Importer *importerAlias) { + importer = importerAlias; + } + + void parse(const UnicodeString &ruleString, + CollationSettings &outSettings, + UParseError *outParseError, + UErrorCode &errorCode); + + const char *getErrorReason() const { return errorReason; } + + /** + * Gets a script or reorder code from its string representation. + * @return the script/reorder code, or + * -1 if not recognized + */ + static int32_t getReorderCode(const char *word); + +private: + /** UCOL_PRIMARY=0 .. UCOL_IDENTICAL=15 */ + static const int32_t STRENGTH_MASK = 0xf; + static const int32_t STARRED_FLAG = 0x10; + static const int32_t OFFSET_SHIFT = 8; + + void parse(const UnicodeString &ruleString, UErrorCode &errorCode); + void parseRuleChain(UErrorCode &errorCode); + int32_t parseResetAndPosition(UErrorCode &errorCode); + int32_t parseRelationOperator(UErrorCode &errorCode); + void parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode); + void parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode); + int32_t parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode); + int32_t parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode); + + /** + * Sets str to a contraction of U+FFFE and (U+2800 + Position). + * @return rule index after the special reset position + */ + int32_t parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode); + void parseSetting(UErrorCode &errorCode); + void parseReordering(const UnicodeString &raw, UErrorCode &errorCode); + static UColAttributeValue getOnOffValue(const UnicodeString &s); + + int32_t parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode); + int32_t readWords(int32_t i, UnicodeString &raw) const; + int32_t skipComment(int32_t i) const; + + void setParseError(const char *reason, UErrorCode &errorCode); + void setErrorContext(); + + /** + * ASCII [:P:] and [:S:]: + * [\u0021-\u002F \u003A-\u0040 \u005B-\u0060 \u007B-\u007E] + */ + static UBool isSyntaxChar(UChar32 c); + int32_t skipWhiteSpace(int32_t i) const; + + const Normalizer2 &nfd, &nfc; + + const UnicodeString *rules; + const CollationData *const baseData; + CollationSettings *settings; + UParseError *parseError; + const char *errorReason; + + Sink *sink; + Importer *importer; + + int32_t ruleIndex; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONRULEPARSER_H__ diff --git a/intl/icu/source/i18n/collationsets.cpp b/intl/icu/source/i18n/collationsets.cpp new file mode 100644 index 0000000000..62e6a5d180 --- /dev/null +++ b/intl/icu/source/i18n/collationsets.cpp @@ -0,0 +1,612 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationsets.cpp +* +* created on: 2013feb09 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucharstrie.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/ustringtrie.h" +#include "collation.h" +#include "collationdata.h" +#include "collationsets.h" +#include "normalizer2impl.h" +#include "uassert.h" +#include "utf16collationiterator.h" +#include "utrie2.h" + +U_NAMESPACE_BEGIN + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumTailoredRange(const void *context, UChar32 start, UChar32 end, uint32_t ce32) { + if(ce32 == Collation::FALLBACK_CE32) { + return true; // fallback to base, not tailored + } + TailoredSet *ts = (TailoredSet *)context; + return ts->handleCE32(start, end, ce32); +} + +U_CDECL_END + +void +TailoredSet::forData(const CollationData *d, UErrorCode &ec) { + if(U_FAILURE(ec)) { return; } + errorCode = ec; // Preserve info & warning codes. + data = d; + baseData = d->base; + U_ASSERT(baseData != nullptr); + utrie2_enum(data->trie, nullptr, enumTailoredRange, this); + ec = errorCode; +} + +UBool +TailoredSet::handleCE32(UChar32 start, UChar32 end, uint32_t ce32) { + U_ASSERT(ce32 != Collation::FALLBACK_CE32); + if(Collation::isSpecialCE32(ce32)) { + ce32 = data->getIndirectCE32(ce32); + if(ce32 == Collation::FALLBACK_CE32) { + return U_SUCCESS(errorCode); + } + } + do { + uint32_t baseCE32 = baseData->getFinalCE32(baseData->getCE32(start)); + // Do not just continue if ce32 == baseCE32 because + // contractions and expansions in different data objects + // normally differ even if they have the same data offsets. + if(Collation::isSelfContainedCE32(ce32) && Collation::isSelfContainedCE32(baseCE32)) { + // fastpath + if(ce32 != baseCE32) { + tailored->add(start); + } + } else { + compare(start, ce32, baseCE32); + } + } while(++start <= end); + return U_SUCCESS(errorCode); +} + +void +TailoredSet::compare(UChar32 c, uint32_t ce32, uint32_t baseCE32) { + if(Collation::isPrefixCE32(ce32)) { + const char16_t *p = data->contexts + Collation::indexFromCE32(ce32); + ce32 = data->getFinalCE32(CollationData::readCE32(p)); + if(Collation::isPrefixCE32(baseCE32)) { + const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32); + baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q)); + comparePrefixes(c, p + 2, q + 2); + } else { + addPrefixes(data, c, p + 2); + } + } else if(Collation::isPrefixCE32(baseCE32)) { + const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32); + baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q)); + addPrefixes(baseData, c, q + 2); + } + + if(Collation::isContractionCE32(ce32)) { + const char16_t *p = data->contexts + Collation::indexFromCE32(ce32); + if((ce32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) { + ce32 = Collation::NO_CE32; + } else { + ce32 = data->getFinalCE32(CollationData::readCE32(p)); + } + if(Collation::isContractionCE32(baseCE32)) { + const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32); + if((baseCE32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) { + baseCE32 = Collation::NO_CE32; + } else { + baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q)); + } + compareContractions(c, p + 2, q + 2); + } else { + addContractions(c, p + 2); + } + } else if(Collation::isContractionCE32(baseCE32)) { + const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32); + baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q)); + addContractions(c, q + 2); + } + + int32_t tag; + if(Collation::isSpecialCE32(ce32)) { + tag = Collation::tagFromCE32(ce32); + U_ASSERT(tag != Collation::PREFIX_TAG); + U_ASSERT(tag != Collation::CONTRACTION_TAG); + // Currently, the tailoring data builder does not write offset tags. + // They might be useful for saving space, + // but they would complicate the builder, + // and in tailorings we assume that performance of tailored characters is more important. + U_ASSERT(tag != Collation::OFFSET_TAG); + } else { + tag = -1; + } + int32_t baseTag; + if(Collation::isSpecialCE32(baseCE32)) { + baseTag = Collation::tagFromCE32(baseCE32); + U_ASSERT(baseTag != Collation::PREFIX_TAG); + U_ASSERT(baseTag != Collation::CONTRACTION_TAG); + } else { + baseTag = -1; + } + + // Non-contextual mappings, expansions, etc. + if(baseTag == Collation::OFFSET_TAG) { + // We might be comparing a tailoring CE which is a copy of + // a base offset-tag CE, via the [optimize [set]] syntax + // or when a single-character mapping was copied for tailored contractions. + // Offset tags always result in long-primary CEs, + // with common secondary/tertiary weights. + if(!Collation::isLongPrimaryCE32(ce32)) { + add(c); + return; + } + int64_t dataCE = baseData->ces[Collation::indexFromCE32(baseCE32)]; + uint32_t p = Collation::getThreeBytePrimaryForOffsetData(c, dataCE); + if(Collation::primaryFromLongPrimaryCE32(ce32) != p) { + add(c); + return; + } + } + + if(tag != baseTag) { + add(c); + return; + } + + if(tag == Collation::EXPANSION32_TAG) { + const uint32_t *ce32s = data->ce32s + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + + const uint32_t *baseCE32s = baseData->ce32s + Collation::indexFromCE32(baseCE32); + int32_t baseLength = Collation::lengthFromCE32(baseCE32); + + if(length != baseLength) { + add(c); + return; + } + for(int32_t i = 0; i < length; ++i) { + if(ce32s[i] != baseCE32s[i]) { + add(c); + break; + } + } + } else if(tag == Collation::EXPANSION_TAG) { + const int64_t *ces = data->ces + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + + const int64_t *baseCEs = baseData->ces + Collation::indexFromCE32(baseCE32); + int32_t baseLength = Collation::lengthFromCE32(baseCE32); + + if(length != baseLength) { + add(c); + return; + } + for(int32_t i = 0; i < length; ++i) { + if(ces[i] != baseCEs[i]) { + add(c); + break; + } + } + } else if(tag == Collation::HANGUL_TAG) { + char16_t jamos[3]; + int32_t length = Hangul::decompose(c, jamos); + if(tailored->contains(jamos[0]) || tailored->contains(jamos[1]) || + (length == 3 && tailored->contains(jamos[2]))) { + add(c); + } + } else if(ce32 != baseCE32) { + add(c); + } +} + +void +TailoredSet::comparePrefixes(UChar32 c, const char16_t *p, const char16_t *q) { + // Parallel iteration over prefixes of both tables. + UCharsTrie::Iterator prefixes(p, 0, errorCode); + UCharsTrie::Iterator basePrefixes(q, 0, errorCode); + const UnicodeString *tp = nullptr; // Tailoring prefix. + const UnicodeString *bp = nullptr; // Base prefix. + // Use a string with a U+FFFF as the limit sentinel. + // U+FFFF is untailorable and will not occur in prefixes. + UnicodeString none((char16_t)0xffff); + for(;;) { + if(tp == nullptr) { + if(prefixes.next(errorCode)) { + tp = &prefixes.getString(); + } else { + tp = &none; + } + } + if(bp == nullptr) { + if(basePrefixes.next(errorCode)) { + bp = &basePrefixes.getString(); + } else { + bp = &none; + } + } + if(tp == &none && bp == &none) { break; } + int32_t cmp = tp->compare(*bp); + if(cmp < 0) { + // tp occurs in the tailoring but not in the base. + addPrefix(data, *tp, c, (uint32_t)prefixes.getValue()); + tp = nullptr; + } else if(cmp > 0) { + // bp occurs in the base but not in the tailoring. + addPrefix(baseData, *bp, c, (uint32_t)basePrefixes.getValue()); + bp = nullptr; + } else { + setPrefix(*tp); + compare(c, (uint32_t)prefixes.getValue(), (uint32_t)basePrefixes.getValue()); + resetPrefix(); + tp = nullptr; + bp = nullptr; + } + } +} + +void +TailoredSet::compareContractions(UChar32 c, const char16_t *p, const char16_t *q) { + // Parallel iteration over suffixes of both tables. + UCharsTrie::Iterator suffixes(p, 0, errorCode); + UCharsTrie::Iterator baseSuffixes(q, 0, errorCode); + const UnicodeString *ts = nullptr; // Tailoring suffix. + const UnicodeString *bs = nullptr; // Base suffix. + // Use a string with two U+FFFF as the limit sentinel. + // U+FFFF is untailorable and will not occur in contractions except maybe + // as a single suffix character for a root-collator boundary contraction. + UnicodeString none((char16_t)0xffff); + none.append((char16_t)0xffff); + for(;;) { + if(ts == nullptr) { + if(suffixes.next(errorCode)) { + ts = &suffixes.getString(); + } else { + ts = &none; + } + } + if(bs == nullptr) { + if(baseSuffixes.next(errorCode)) { + bs = &baseSuffixes.getString(); + } else { + bs = &none; + } + } + if(ts == &none && bs == &none) { break; } + int32_t cmp = ts->compare(*bs); + if(cmp < 0) { + // ts occurs in the tailoring but not in the base. + addSuffix(c, *ts); + ts = nullptr; + } else if(cmp > 0) { + // bs occurs in the base but not in the tailoring. + addSuffix(c, *bs); + bs = nullptr; + } else { + suffix = ts; + compare(c, (uint32_t)suffixes.getValue(), (uint32_t)baseSuffixes.getValue()); + suffix = nullptr; + ts = nullptr; + bs = nullptr; + } + } +} + +void +TailoredSet::addPrefixes(const CollationData *d, UChar32 c, const char16_t *p) { + UCharsTrie::Iterator prefixes(p, 0, errorCode); + while(prefixes.next(errorCode)) { + addPrefix(d, prefixes.getString(), c, (uint32_t)prefixes.getValue()); + } +} + +void +TailoredSet::addPrefix(const CollationData *d, const UnicodeString &pfx, UChar32 c, uint32_t ce32) { + setPrefix(pfx); + ce32 = d->getFinalCE32(ce32); + if(Collation::isContractionCE32(ce32)) { + const char16_t *p = d->contexts + Collation::indexFromCE32(ce32); + addContractions(c, p + 2); + } + tailored->add(UnicodeString(unreversedPrefix).append(c)); + resetPrefix(); +} + +void +TailoredSet::addContractions(UChar32 c, const char16_t *p) { + UCharsTrie::Iterator suffixes(p, 0, errorCode); + while(suffixes.next(errorCode)) { + addSuffix(c, suffixes.getString()); + } +} + +void +TailoredSet::addSuffix(UChar32 c, const UnicodeString &sfx) { + tailored->add(UnicodeString(unreversedPrefix).append(c).append(sfx)); +} + +void +TailoredSet::add(UChar32 c) { + if(unreversedPrefix.isEmpty() && suffix == nullptr) { + tailored->add(c); + } else { + UnicodeString s(unreversedPrefix); + s.append(c); + if(suffix != nullptr) { + s.append(*suffix); + } + tailored->add(s); + } +} + +ContractionsAndExpansions::CESink::~CESink() {} + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumCnERange(const void *context, UChar32 start, UChar32 end, uint32_t ce32) { + ContractionsAndExpansions *cne = (ContractionsAndExpansions *)context; + if(cne->checkTailored == 0) { + // There is no tailoring. + // No need to collect nor check the tailored set. + } else if(cne->checkTailored < 0) { + // Collect the set of code points with mappings in the tailoring data. + if(ce32 == Collation::FALLBACK_CE32) { + return true; // fallback to base, not tailored + } else { + cne->tailored.add(start, end); + } + // checkTailored > 0: Exclude tailored ranges from the base data enumeration. + } else if(start == end) { + if(cne->tailored.contains(start)) { + return true; + } + } else if(cne->tailored.containsSome(start, end)) { + cne->ranges.set(start, end).removeAll(cne->tailored); + int32_t count = cne->ranges.getRangeCount(); + for(int32_t i = 0; i < count; ++i) { + cne->handleCE32(cne->ranges.getRangeStart(i), cne->ranges.getRangeEnd(i), ce32); + } + return U_SUCCESS(cne->errorCode); + } + cne->handleCE32(start, end, ce32); + return U_SUCCESS(cne->errorCode); +} + +U_CDECL_END + +void +ContractionsAndExpansions::forData(const CollationData *d, UErrorCode &ec) { + if(U_FAILURE(ec)) { return; } + errorCode = ec; // Preserve info & warning codes. + // Add all from the data, can be tailoring or base. + if(d->base != nullptr) { + checkTailored = -1; + } + data = d; + utrie2_enum(data->trie, nullptr, enumCnERange, this); + if(d->base == nullptr || U_FAILURE(errorCode)) { + ec = errorCode; + return; + } + // Add all from the base data but only for un-tailored code points. + tailored.freeze(); + checkTailored = 1; + data = d->base; + utrie2_enum(data->trie, nullptr, enumCnERange, this); + ec = errorCode; +} + +void +ContractionsAndExpansions::forCodePoint(const CollationData *d, UChar32 c, UErrorCode &ec) { + if(U_FAILURE(ec)) { return; } + errorCode = ec; // Preserve info & warning codes. + uint32_t ce32 = d->getCE32(c); + if(ce32 == Collation::FALLBACK_CE32) { + d = d->base; + ce32 = d->getCE32(c); + } + data = d; + handleCE32(c, c, ce32); + ec = errorCode; +} + +void +ContractionsAndExpansions::handleCE32(UChar32 start, UChar32 end, uint32_t ce32) { + for(;;) { + if((ce32 & 0xff) < Collation::SPECIAL_CE32_LOW_BYTE) { + // !isSpecialCE32() + if(sink != nullptr) { + sink->handleCE(Collation::ceFromSimpleCE32(ce32)); + } + return; + } + switch(Collation::tagFromCE32(ce32)) { + case Collation::FALLBACK_TAG: + return; + case Collation::RESERVED_TAG_3: + case Collation::BUILDER_DATA_TAG: + case Collation::LEAD_SURROGATE_TAG: + if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; } + return; + case Collation::LONG_PRIMARY_TAG: + if(sink != nullptr) { + sink->handleCE(Collation::ceFromLongPrimaryCE32(ce32)); + } + return; + case Collation::LONG_SECONDARY_TAG: + if(sink != nullptr) { + sink->handleCE(Collation::ceFromLongSecondaryCE32(ce32)); + } + return; + case Collation::LATIN_EXPANSION_TAG: + if(sink != nullptr) { + ces[0] = Collation::latinCE0FromCE32(ce32); + ces[1] = Collation::latinCE1FromCE32(ce32); + sink->handleExpansion(ces, 2); + } + // Optimization: If we have a prefix, + // then the relevant strings have been added already. + if(unreversedPrefix.isEmpty()) { + addExpansions(start, end); + } + return; + case Collation::EXPANSION32_TAG: + if(sink != nullptr) { + const uint32_t *ce32s = data->ce32s + Collation::indexFromCE32(ce32); + int32_t length = Collation::lengthFromCE32(ce32); + for(int32_t i = 0; i < length; ++i) { + ces[i] = Collation::ceFromCE32(*ce32s++); + } + sink->handleExpansion(ces, length); + } + // Optimization: If we have a prefix, + // then the relevant strings have been added already. + if(unreversedPrefix.isEmpty()) { + addExpansions(start, end); + } + return; + case Collation::EXPANSION_TAG: + if(sink != nullptr) { + int32_t length = Collation::lengthFromCE32(ce32); + sink->handleExpansion(data->ces + Collation::indexFromCE32(ce32), length); + } + // Optimization: If we have a prefix, + // then the relevant strings have been added already. + if(unreversedPrefix.isEmpty()) { + addExpansions(start, end); + } + return; + case Collation::PREFIX_TAG: + handlePrefixes(start, end, ce32); + return; + case Collation::CONTRACTION_TAG: + handleContractions(start, end, ce32); + return; + case Collation::DIGIT_TAG: + // Fetch the non-numeric-collation CE32 and continue. + ce32 = data->ce32s[Collation::indexFromCE32(ce32)]; + break; + case Collation::U0000_TAG: + U_ASSERT(start == 0 && end == 0); + // Fetch the normal ce32 for U+0000 and continue. + ce32 = data->ce32s[0]; + break; + case Collation::HANGUL_TAG: + if(sink != nullptr) { + // TODO: This should be optimized, + // especially if [start..end] is the complete Hangul range. (assert that) + UTF16CollationIterator iter(data, false, nullptr, nullptr, nullptr); + char16_t hangul[1] = { 0 }; + for(UChar32 c = start; c <= end; ++c) { + hangul[0] = (char16_t)c; + iter.setText(hangul, hangul + 1); + int32_t length = iter.fetchCEs(errorCode); + if(U_FAILURE(errorCode)) { return; } + // Ignore the terminating non-CE. + U_ASSERT(length >= 2 && iter.getCE(length - 1) == Collation::NO_CE); + sink->handleExpansion(iter.getCEs(), length - 1); + } + } + // Optimization: If we have a prefix, + // then the relevant strings have been added already. + if(unreversedPrefix.isEmpty()) { + addExpansions(start, end); + } + return; + case Collation::OFFSET_TAG: + // Currently no need to send offset CEs to the sink. + return; + case Collation::IMPLICIT_TAG: + // Currently no need to send implicit CEs to the sink. + return; + } + } +} + +void +ContractionsAndExpansions::handlePrefixes( + UChar32 start, UChar32 end, uint32_t ce32) { + const char16_t *p = data->contexts + Collation::indexFromCE32(ce32); + ce32 = CollationData::readCE32(p); // Default if no prefix match. + handleCE32(start, end, ce32); + if(!addPrefixes) { return; } + UCharsTrie::Iterator prefixes(p + 2, 0, errorCode); + while(prefixes.next(errorCode)) { + setPrefix(prefixes.getString()); + // Prefix/pre-context mappings are special kinds of contractions + // that always yield expansions. + addStrings(start, end, contractions); + addStrings(start, end, expansions); + handleCE32(start, end, (uint32_t)prefixes.getValue()); + } + resetPrefix(); +} + +void +ContractionsAndExpansions::handleContractions( + UChar32 start, UChar32 end, uint32_t ce32) { + const char16_t *p = data->contexts + Collation::indexFromCE32(ce32); + if((ce32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) { + // No match on the single code point. + // We are underneath a prefix, and the default mapping is just + // a fallback to the mappings for a shorter prefix. + U_ASSERT(!unreversedPrefix.isEmpty()); + } else { + ce32 = CollationData::readCE32(p); // Default if no suffix match. + U_ASSERT(!Collation::isContractionCE32(ce32)); + handleCE32(start, end, ce32); + } + UCharsTrie::Iterator suffixes(p + 2, 0, errorCode); + while(suffixes.next(errorCode)) { + suffix = &suffixes.getString(); + addStrings(start, end, contractions); + if(!unreversedPrefix.isEmpty()) { + addStrings(start, end, expansions); + } + handleCE32(start, end, (uint32_t)suffixes.getValue()); + } + suffix = nullptr; +} + +void +ContractionsAndExpansions::addExpansions(UChar32 start, UChar32 end) { + if(unreversedPrefix.isEmpty() && suffix == nullptr) { + if(expansions != nullptr) { + expansions->add(start, end); + } + } else { + addStrings(start, end, expansions); + } +} + +void +ContractionsAndExpansions::addStrings(UChar32 start, UChar32 end, UnicodeSet *set) { + if(set == nullptr) { return; } + UnicodeString s(unreversedPrefix); + do { + s.append(start); + if(suffix != nullptr) { + s.append(*suffix); + } + set->add(s); + s.truncate(unreversedPrefix.length()); + } while(++start <= end); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationsets.h b/intl/icu/source/i18n/collationsets.h new file mode 100644 index 0000000000..99aa194e76 --- /dev/null +++ b/intl/icu/source/i18n/collationsets.h @@ -0,0 +1,144 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationsets.h +* +* created on: 2013feb09 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONSETS_H__ +#define __COLLATIONSETS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/uniset.h" +#include "collation.h" + +U_NAMESPACE_BEGIN + +struct CollationData; + +/** + * Finds the set of characters and strings that sort differently in the tailoring + * from the base data. + * + * Every mapping in the tailoring needs to be compared to the base, + * because some mappings are copied for optimization, and + * all contractions for a character are copied if any contractions for that character + * are added, modified or removed. + * + * It might be simpler to re-parse the rule string, but: + * - That would require duplicating some of the from-rules builder code. + * - That would make the runtime code depend on the builder. + * - That would only work if we have the rule string, and we allow users to + * omit the rule string from data files. + */ +class TailoredSet : public UMemory { +public: + TailoredSet(UnicodeSet *t) + : data(nullptr), baseData(nullptr), + tailored(t), + suffix(nullptr), + errorCode(U_ZERO_ERROR) {} + + void forData(const CollationData *d, UErrorCode &errorCode); + + /** + * @return U_SUCCESS(errorCode) in C++, void in Java + * @internal only public for access by callback + */ + UBool handleCE32(UChar32 start, UChar32 end, uint32_t ce32); + +private: + void compare(UChar32 c, uint32_t ce32, uint32_t baseCE32); + void comparePrefixes(UChar32 c, const char16_t *p, const char16_t *q); + void compareContractions(UChar32 c, const char16_t *p, const char16_t *q); + + void addPrefixes(const CollationData *d, UChar32 c, const char16_t *p); + void addPrefix(const CollationData *d, const UnicodeString &pfx, UChar32 c, uint32_t ce32); + void addContractions(UChar32 c, const char16_t *p); + void addSuffix(UChar32 c, const UnicodeString &sfx); + void add(UChar32 c); + + /** Prefixes are reversed in the data structure. */ + void setPrefix(const UnicodeString &pfx) { + unreversedPrefix = pfx; + unreversedPrefix.reverse(); + } + void resetPrefix() { + unreversedPrefix.remove(); + } + + const CollationData *data; + const CollationData *baseData; + UnicodeSet *tailored; + UnicodeString unreversedPrefix; + const UnicodeString *suffix; + UErrorCode errorCode; +}; + +class ContractionsAndExpansions : public UMemory { +public: + class CESink : public UMemory { + public: + virtual ~CESink(); + virtual void handleCE(int64_t ce) = 0; + virtual void handleExpansion(const int64_t ces[], int32_t length) = 0; + }; + + ContractionsAndExpansions(UnicodeSet *con, UnicodeSet *exp, CESink *s, UBool prefixes) + : data(nullptr), + contractions(con), expansions(exp), + sink(s), + addPrefixes(prefixes), + checkTailored(0), + suffix(nullptr), + errorCode(U_ZERO_ERROR) {} + + void forData(const CollationData *d, UErrorCode &errorCode); + void forCodePoint(const CollationData *d, UChar32 c, UErrorCode &ec); + + // all following: @internal, only public for access by callback + + void handleCE32(UChar32 start, UChar32 end, uint32_t ce32); + + void handlePrefixes(UChar32 start, UChar32 end, uint32_t ce32); + void handleContractions(UChar32 start, UChar32 end, uint32_t ce32); + + void addExpansions(UChar32 start, UChar32 end); + void addStrings(UChar32 start, UChar32 end, UnicodeSet *set); + + /** Prefixes are reversed in the data structure. */ + void setPrefix(const UnicodeString &pfx) { + unreversedPrefix = pfx; + unreversedPrefix.reverse(); + } + void resetPrefix() { + unreversedPrefix.remove(); + } + + const CollationData *data; + UnicodeSet *contractions; + UnicodeSet *expansions; + CESink *sink; + UBool addPrefixes; + int8_t checkTailored; // -1: collected tailored +1: exclude tailored + UnicodeSet tailored; + UnicodeSet ranges; + UnicodeString unreversedPrefix; + const UnicodeString *suffix; + int64_t ces[Collation::MAX_EXPANSION_LENGTH]; + UErrorCode errorCode; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONSETS_H__ diff --git a/intl/icu/source/i18n/collationsettings.cpp b/intl/icu/source/i18n/collationsettings.cpp new file mode 100644 index 0000000000..1533daf38c --- /dev/null +++ b/intl/icu/source/i18n/collationsettings.cpp @@ -0,0 +1,377 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationsettings.cpp +* +* created on: 2013feb07 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "cmemory.h" +#include "collation.h" +#include "collationdata.h" +#include "collationsettings.h" +#include "sharedobject.h" +#include "uassert.h" +#include "umutex.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +CollationSettings::CollationSettings(const CollationSettings &other) + : SharedObject(other), + options(other.options), variableTop(other.variableTop), + reorderTable(nullptr), + minHighNoReorder(other.minHighNoReorder), + reorderRanges(nullptr), reorderRangesLength(0), + reorderCodes(nullptr), reorderCodesLength(0), reorderCodesCapacity(0), + fastLatinOptions(other.fastLatinOptions) { + UErrorCode errorCode = U_ZERO_ERROR; + copyReorderingFrom(other, errorCode); + if(fastLatinOptions >= 0) { + uprv_memcpy(fastLatinPrimaries, other.fastLatinPrimaries, sizeof(fastLatinPrimaries)); + } +} + +CollationSettings::~CollationSettings() { + if(reorderCodesCapacity != 0) { + uprv_free(const_cast(reorderCodes)); + } +} + +bool +CollationSettings::operator==(const CollationSettings &other) const { + if(options != other.options) { return false; } + if((options & ALTERNATE_MASK) != 0 && variableTop != other.variableTop) { return false; } + if(reorderCodesLength != other.reorderCodesLength) { return false; } + for(int32_t i = 0; i < reorderCodesLength; ++i) { + if(reorderCodes[i] != other.reorderCodes[i]) { return false; } + } + return true; +} + +int32_t +CollationSettings::hashCode() const { + int32_t h = options << 8; + if((options & ALTERNATE_MASK) != 0) { h ^= variableTop; } + h ^= reorderCodesLength; + for(int32_t i = 0; i < reorderCodesLength; ++i) { + h ^= (reorderCodes[i] << i); + } + return h; +} + +void +CollationSettings::resetReordering() { + // When we turn off reordering, we want to set a nullptr permutation + // rather than a no-op permutation. + // Keep the memory via reorderCodes and its capacity. + reorderTable = nullptr; + minHighNoReorder = 0; + reorderRangesLength = 0; + reorderCodesLength = 0; +} + +void +CollationSettings::aliasReordering(const CollationData &data, const int32_t *codes, int32_t length, + const uint32_t *ranges, int32_t rangesLength, + const uint8_t *table, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(table != nullptr && + (rangesLength == 0 ? + !reorderTableHasSplitBytes(table) : + rangesLength >= 2 && + // The first offset must be 0. The last offset must not be 0. + (ranges[0] & 0xffff) == 0 && (ranges[rangesLength - 1] & 0xffff) != 0)) { + // We need to release the memory before setting the alias pointer. + if(reorderCodesCapacity != 0) { + uprv_free(const_cast(reorderCodes)); + reorderCodesCapacity = 0; + } + reorderTable = table; + reorderCodes = codes; + reorderCodesLength = length; + // Drop ranges before the first split byte. They are reordered by the table. + // This then speeds up reordering of the remaining ranges. + int32_t firstSplitByteRangeIndex = 0; + while(firstSplitByteRangeIndex < rangesLength && + (ranges[firstSplitByteRangeIndex] & 0xff0000) == 0) { + // The second byte of the primary limit is 0. + ++firstSplitByteRangeIndex; + } + if(firstSplitByteRangeIndex == rangesLength) { + U_ASSERT(!reorderTableHasSplitBytes(table)); + minHighNoReorder = 0; + reorderRanges = nullptr; + reorderRangesLength = 0; + } else { + U_ASSERT(table[ranges[firstSplitByteRangeIndex] >> 24] == 0); + minHighNoReorder = ranges[rangesLength - 1] & 0xffff0000; + reorderRanges = ranges + firstSplitByteRangeIndex; + reorderRangesLength = rangesLength - firstSplitByteRangeIndex; + } + return; + } + // Regenerate missing data. + setReordering(data, codes, length, errorCode); +} + +void +CollationSettings::setReordering(const CollationData &data, + const int32_t *codes, int32_t codesLength, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(codesLength == 0 || (codesLength == 1 && codes[0] == UCOL_REORDER_CODE_NONE)) { + resetReordering(); + return; + } + UVector32 rangesList(errorCode); + data.makeReorderRanges(codes, codesLength, rangesList, errorCode); + if(U_FAILURE(errorCode)) { return; } + int32_t rangesLength = rangesList.size(); + if(rangesLength == 0) { + resetReordering(); + return; + } + const uint32_t *ranges = reinterpret_cast(rangesList.getBuffer()); + // ranges[] contains at least two (limit, offset) pairs. + // The first offset must be 0. The last offset must not be 0. + // Separators (at the low end) and trailing weights (at the high end) + // are never reordered. + U_ASSERT(rangesLength >= 2); + U_ASSERT((ranges[0] & 0xffff) == 0 && (ranges[rangesLength - 1] & 0xffff) != 0); + minHighNoReorder = ranges[rangesLength - 1] & 0xffff0000; + + // Write the lead byte permutation table. + // Set a 0 for each lead byte that has a range boundary in the middle. + uint8_t table[256]; + int32_t b = 0; + int32_t firstSplitByteRangeIndex = -1; + for(int32_t i = 0; i < rangesLength; ++i) { + uint32_t pair = ranges[i]; + int32_t limit1 = (int32_t)(pair >> 24); + while(b < limit1) { + table[b] = (uint8_t)(b + pair); + ++b; + } + // Check the second byte of the limit. + if((pair & 0xff0000) != 0) { + table[limit1] = 0; + b = limit1 + 1; + if(firstSplitByteRangeIndex < 0) { + firstSplitByteRangeIndex = i; + } + } + } + while(b <= 0xff) { + table[b] = (uint8_t)b; + ++b; + } + if(firstSplitByteRangeIndex < 0) { + // The lead byte permutation table alone suffices for reordering. + rangesLength = 0; + } else { + // Remove the ranges below the first split byte. + ranges += firstSplitByteRangeIndex; + rangesLength -= firstSplitByteRangeIndex; + } + setReorderArrays(codes, codesLength, ranges, rangesLength, table, errorCode); +} + +void +CollationSettings::setReorderArrays(const int32_t *codes, int32_t codesLength, + const uint32_t *ranges, int32_t rangesLength, + const uint8_t *table, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t *ownedCodes; + int32_t totalLength = codesLength + rangesLength; + U_ASSERT(totalLength > 0); + if(totalLength <= reorderCodesCapacity) { + ownedCodes = const_cast(reorderCodes); + } else { + // Allocate one memory block for the codes, the ranges, and the 16-aligned table. + int32_t capacity = (totalLength + 3) & ~3; // round up to a multiple of 4 ints + ownedCodes = (int32_t *)uprv_malloc(capacity * 4 + 256); + if(ownedCodes == nullptr) { + resetReordering(); + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + if(reorderCodesCapacity != 0) { + uprv_free(const_cast(reorderCodes)); + } + reorderCodes = ownedCodes; + reorderCodesCapacity = capacity; + } + uprv_memcpy(ownedCodes + reorderCodesCapacity, table, 256); + uprv_memcpy(ownedCodes, codes, codesLength * 4); + uprv_memcpy(ownedCodes + codesLength, ranges, rangesLength * 4); + reorderTable = reinterpret_cast(reorderCodes + reorderCodesCapacity); + reorderCodesLength = codesLength; + reorderRanges = reinterpret_cast(ownedCodes) + codesLength; + reorderRangesLength = rangesLength; +} + +void +CollationSettings::copyReorderingFrom(const CollationSettings &other, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(!other.hasReordering()) { + resetReordering(); + return; + } + minHighNoReorder = other.minHighNoReorder; + if(other.reorderCodesCapacity == 0) { + // The reorder arrays are aliased to memory-mapped data. + reorderTable = other.reorderTable; + reorderRanges = other.reorderRanges; + reorderRangesLength = other.reorderRangesLength; + reorderCodes = other.reorderCodes; + reorderCodesLength = other.reorderCodesLength; + } else { + setReorderArrays(other.reorderCodes, other.reorderCodesLength, + other.reorderRanges, other.reorderRangesLength, + other.reorderTable, errorCode); + } +} + +UBool +CollationSettings::reorderTableHasSplitBytes(const uint8_t table[256]) { + U_ASSERT(table[0] == 0); + for(int32_t i = 1; i < 256; ++i) { + if(table[i] == 0) { + return true; + } + } + return false; +} + +uint32_t +CollationSettings::reorderEx(uint32_t p) const { + if(p >= minHighNoReorder) { return p; } + // Round up p so that its lower 16 bits are >= any offset bits. + // Then compare q directly with (limit, offset) pairs. + uint32_t q = p | 0xffff; + uint32_t r; + const uint32_t *ranges = reorderRanges; + while(q >= (r = *ranges)) { ++ranges; } + return p + (r << 24); +} + +void +CollationSettings::setStrength(int32_t value, int32_t defaultOptions, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t noStrength = options & ~STRENGTH_MASK; + switch(value) { + case UCOL_PRIMARY: + case UCOL_SECONDARY: + case UCOL_TERTIARY: + case UCOL_QUATERNARY: + case UCOL_IDENTICAL: + options = noStrength | (value << STRENGTH_SHIFT); + break; + case UCOL_DEFAULT: + options = noStrength | (defaultOptions & STRENGTH_MASK); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } +} + +void +CollationSettings::setFlag(int32_t bit, UColAttributeValue value, + int32_t defaultOptions, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + switch(value) { + case UCOL_ON: + options |= bit; + break; + case UCOL_OFF: + options &= ~bit; + break; + case UCOL_DEFAULT: + options = (options & ~bit) | (defaultOptions & bit); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } +} + +void +CollationSettings::setCaseFirst(UColAttributeValue value, + int32_t defaultOptions, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t noCaseFirst = options & ~CASE_FIRST_AND_UPPER_MASK; + switch(value) { + case UCOL_OFF: + options = noCaseFirst; + break; + case UCOL_LOWER_FIRST: + options = noCaseFirst | CASE_FIRST; + break; + case UCOL_UPPER_FIRST: + options = noCaseFirst | CASE_FIRST_AND_UPPER_MASK; + break; + case UCOL_DEFAULT: + options = noCaseFirst | (defaultOptions & CASE_FIRST_AND_UPPER_MASK); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } +} + +void +CollationSettings::setAlternateHandling(UColAttributeValue value, + int32_t defaultOptions, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t noAlternate = options & ~ALTERNATE_MASK; + switch(value) { + case UCOL_NON_IGNORABLE: + options = noAlternate; + break; + case UCOL_SHIFTED: + options = noAlternate | SHIFTED; + break; + case UCOL_DEFAULT: + options = noAlternate | (defaultOptions & ALTERNATE_MASK); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } +} + +void +CollationSettings::setMaxVariable(int32_t value, int32_t defaultOptions, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + int32_t noMax = options & ~MAX_VARIABLE_MASK; + switch(value) { + case MAX_VAR_SPACE: + case MAX_VAR_PUNCT: + case MAX_VAR_SYMBOL: + case MAX_VAR_CURRENCY: + options = noMax | (value << MAX_VARIABLE_SHIFT); + break; + case UCOL_DEFAULT: + options = noMax | (defaultOptions & MAX_VARIABLE_MASK); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationsettings.h b/intl/icu/source/i18n/collationsettings.h new file mode 100644 index 0000000000..43a181211c --- /dev/null +++ b/intl/icu/source/i18n/collationsettings.h @@ -0,0 +1,274 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationsettings.h +* +* created on: 2013feb07 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONSETTINGS_H__ +#define __COLLATIONSETTINGS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/ucol.h" +#include "collation.h" +#include "sharedobject.h" +#include "umutex.h" + +U_NAMESPACE_BEGIN + +struct CollationData; + +/** + * Collation settings/options/attributes. + * These are the values that can be changed via API. + */ +struct U_I18N_API CollationSettings : public SharedObject { + /** + * Options bit 0: Perform the FCD check on the input text and deliver normalized text. + */ + static const int32_t CHECK_FCD = 1; + /** + * Options bit 1: Numeric collation. + * Also known as CODAN = COllate Digits As Numbers. + * + * Treat digit sequences as numbers with CE sequences in numeric order, + * rather than returning a normal CE for each digit. + */ + static const int32_t NUMERIC = 2; + /** + * "Shifted" alternate handling, see ALTERNATE_MASK. + */ + static const int32_t SHIFTED = 4; + /** + * Options bits 3..2: Alternate-handling mask. 0 for non-ignorable. + * Reserve values 8 and 0xc for shift-trimmed and blanked. + */ + static const int32_t ALTERNATE_MASK = 0xc; + /** + * Options bits 6..4: The 3-bit maxVariable value bit field is shifted by this value. + */ + static const int32_t MAX_VARIABLE_SHIFT = 4; + /** maxVariable options bit mask before shifting. */ + static const int32_t MAX_VARIABLE_MASK = 0x70; + /** Options bit 7: Reserved/unused/0. */ + /** + * Options bit 8: Sort uppercase first if caseLevel or caseFirst is on. + */ + static const int32_t UPPER_FIRST = 0x100; + /** + * Options bit 9: Keep the case bits in the tertiary weight (they trump other tertiary values) + * unless case level is on (when they are *moved* into the separate case level). + * By default, the case bits are removed from the tertiary weight (ignored). + * + * When CASE_FIRST is off, UPPER_FIRST must be off too, corresponding to + * the tri-value UCOL_CASE_FIRST attribute: UCOL_OFF vs. UCOL_LOWER_FIRST vs. UCOL_UPPER_FIRST. + */ + static const int32_t CASE_FIRST = 0x200; + /** + * Options bit mask for caseFirst and upperFirst, before shifting. + * Same value as caseFirst==upperFirst. + */ + static const int32_t CASE_FIRST_AND_UPPER_MASK = CASE_FIRST | UPPER_FIRST; + /** + * Options bit 10: Insert the case level between the secondary and tertiary levels. + */ + static const int32_t CASE_LEVEL = 0x400; + /** + * Options bit 11: Compare secondary weights backwards. ("French secondary") + */ + static const int32_t BACKWARD_SECONDARY = 0x800; + /** + * Options bits 15..12: The 4-bit strength value bit field is shifted by this value. + * It is the top used bit field in the options. (No need to mask after shifting.) + */ + static const int32_t STRENGTH_SHIFT = 12; + /** Strength options bit mask before shifting. */ + static const int32_t STRENGTH_MASK = 0xf000; + + /** maxVariable values */ + enum MaxVariable { + MAX_VAR_SPACE, + MAX_VAR_PUNCT, + MAX_VAR_SYMBOL, + MAX_VAR_CURRENCY + }; + + CollationSettings() + : options((UCOL_DEFAULT_STRENGTH << STRENGTH_SHIFT) | + (MAX_VAR_PUNCT << MAX_VARIABLE_SHIFT)), + variableTop(0), + reorderTable(nullptr), + minHighNoReorder(0), + reorderRanges(nullptr), reorderRangesLength(0), + reorderCodes(nullptr), reorderCodesLength(0), reorderCodesCapacity(0), + fastLatinOptions(-1) {} + + CollationSettings(const CollationSettings &other); + virtual ~CollationSettings(); + + bool operator==(const CollationSettings &other) const; + + inline bool operator!=(const CollationSettings &other) const { + return !operator==(other); + } + + int32_t hashCode() const; + + void resetReordering(); + void aliasReordering(const CollationData &data, const int32_t *codes, int32_t length, + const uint32_t *ranges, int32_t rangesLength, + const uint8_t *table, UErrorCode &errorCode); + void setReordering(const CollationData &data, const int32_t *codes, int32_t codesLength, + UErrorCode &errorCode); + void copyReorderingFrom(const CollationSettings &other, UErrorCode &errorCode); + + inline UBool hasReordering() const { return reorderTable != nullptr; } + static UBool reorderTableHasSplitBytes(const uint8_t table[256]); + inline uint32_t reorder(uint32_t p) const { + uint8_t b = reorderTable[p >> 24]; + if(b != 0 || p <= Collation::NO_CE_PRIMARY) { + return ((uint32_t)b << 24) | (p & 0xffffff); + } else { + return reorderEx(p); + } + } + + void setStrength(int32_t value, int32_t defaultOptions, UErrorCode &errorCode); + + static int32_t getStrength(int32_t options) { + return options >> STRENGTH_SHIFT; + } + + int32_t getStrength() const { + return getStrength(options); + } + + /** Sets the options bit for an on/off attribute. */ + void setFlag(int32_t bit, UColAttributeValue value, + int32_t defaultOptions, UErrorCode &errorCode); + + UColAttributeValue getFlag(int32_t bit) const { + return ((options & bit) != 0) ? UCOL_ON : UCOL_OFF; + } + + void setCaseFirst(UColAttributeValue value, int32_t defaultOptions, UErrorCode &errorCode); + + UColAttributeValue getCaseFirst() const { + int32_t option = options & CASE_FIRST_AND_UPPER_MASK; + return (option == 0) ? UCOL_OFF : + (option == CASE_FIRST) ? UCOL_LOWER_FIRST : UCOL_UPPER_FIRST; + } + + void setAlternateHandling(UColAttributeValue value, + int32_t defaultOptions, UErrorCode &errorCode); + + UColAttributeValue getAlternateHandling() const { + return ((options & ALTERNATE_MASK) == 0) ? UCOL_NON_IGNORABLE : UCOL_SHIFTED; + } + + void setMaxVariable(int32_t value, int32_t defaultOptions, UErrorCode &errorCode); + + MaxVariable getMaxVariable() const { + return (MaxVariable)((options & MAX_VARIABLE_MASK) >> MAX_VARIABLE_SHIFT); + } + + /** + * Include case bits in the tertiary level if caseLevel=off and caseFirst!=off. + */ + static inline UBool isTertiaryWithCaseBits(int32_t options) { + return (options & (CASE_LEVEL | CASE_FIRST)) == CASE_FIRST; + } + static uint32_t getTertiaryMask(int32_t options) { + // Remove the case bits from the tertiary weight when caseLevel is on or caseFirst is off. + return isTertiaryWithCaseBits(options) ? + Collation::CASE_AND_TERTIARY_MASK : Collation::ONLY_TERTIARY_MASK; + } + + static UBool sortsTertiaryUpperCaseFirst(int32_t options) { + // On tertiary level, consider case bits and sort uppercase first + // if caseLevel is off and caseFirst==upperFirst. + return (options & (CASE_LEVEL | CASE_FIRST_AND_UPPER_MASK)) == CASE_FIRST_AND_UPPER_MASK; + } + + inline UBool dontCheckFCD() const { + return (options & CHECK_FCD) == 0; + } + + inline UBool hasBackwardSecondary() const { + return (options & BACKWARD_SECONDARY) != 0; + } + + inline UBool isNumeric() const { + return (options & NUMERIC) != 0; + } + + /** CHECK_FCD etc. */ + int32_t options; + /** Variable-top primary weight. */ + uint32_t variableTop; + /** + * 256-byte table for reordering permutation of primary lead bytes; nullptr if no reordering. + * A 0 entry at a non-zero index means that the primary lead byte is "split" + * (there are different offsets for primaries that share that lead byte) + * and the reordering offset must be determined via the reorderRanges. + */ + const uint8_t *reorderTable; + /** Limit of last reordered range. 0 if no reordering or no split bytes. */ + uint32_t minHighNoReorder; + /** + * Primary-weight ranges for script reordering, + * to be used by reorder(p) for split-reordered primary lead bytes. + * + * Each entry is a (limit, offset) pair. + * The upper 16 bits of the entry are the upper 16 bits of the + * exclusive primary limit of a range. + * Primaries between the previous limit and this one have their lead bytes + * modified by the signed offset (-0xff..+0xff) stored in the lower 16 bits. + * + * CollationData::makeReorderRanges() writes a full list where the first range + * (at least for terminators and separators) has a 0 offset. + * The last range has a non-zero offset. + * minHighNoReorder is set to the limit of that last range. + * + * In the settings object, the initial ranges before the first split lead byte + * are omitted for efficiency; they are handled by reorder(p) via the reorderTable. + * If there are no split-reordered lead bytes, then no ranges are needed. + */ + const uint32_t *reorderRanges; + int32_t reorderRangesLength; + /** Array of reorder codes; ignored if reorderCodesLength == 0. */ + const int32_t *reorderCodes; + /** Number of reorder codes; 0 if no reordering. */ + int32_t reorderCodesLength; + /** + * Capacity of reorderCodes. + * If 0, then the codes, the ranges, and the table are aliases. + * Otherwise, this object owns the memory via the reorderCodes pointer; + * the codes, the ranges, and the table are in the same memory block, in that order. + */ + int32_t reorderCodesCapacity; + + /** Options for CollationFastLatin. Negative if disabled. */ + int32_t fastLatinOptions; + uint16_t fastLatinPrimaries[0x180]; + +private: + void setReorderArrays(const int32_t *codes, int32_t codesLength, + const uint32_t *ranges, int32_t rangesLength, + const uint8_t *table, UErrorCode &errorCode); + uint32_t reorderEx(uint32_t p) const; +}; + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONSETTINGS_H__ diff --git a/intl/icu/source/i18n/collationtailoring.cpp b/intl/icu/source/i18n/collationtailoring.cpp new file mode 100644 index 0000000000..8d22cf2516 --- /dev/null +++ b/intl/icu/source/i18n/collationtailoring.cpp @@ -0,0 +1,113 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationtailoring.cpp +* +* created on: 2013mar12 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/udata.h" +#include "unicode/unistr.h" +#include "unicode/ures.h" +#include "unicode/uversion.h" +#include "unicode/uvernum.h" +#include "cmemory.h" +#include "collationdata.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "normalizer2impl.h" +#include "uassert.h" +#include "uhash.h" +#include "umutex.h" +#include "utrie2.h" + +U_NAMESPACE_BEGIN + +CollationTailoring::CollationTailoring(const CollationSettings *baseSettings) + : data(nullptr), settings(baseSettings), + actualLocale(""), + ownedData(nullptr), + builder(nullptr), memory(nullptr), bundle(nullptr), + trie(nullptr), unsafeBackwardSet(nullptr), + maxExpansions(nullptr) { + if(baseSettings != nullptr) { + U_ASSERT(baseSettings->reorderCodesLength == 0); + U_ASSERT(baseSettings->reorderTable == nullptr); + U_ASSERT(baseSettings->minHighNoReorder == 0); + } else { + settings = new CollationSettings(); + } + if(settings != nullptr) { + settings->addRef(); + } + rules.getTerminatedBuffer(); // ensure NUL-termination + version[0] = version[1] = version[2] = version[3] = 0; + maxExpansionsInitOnce.reset(); +} + +CollationTailoring::~CollationTailoring() { + SharedObject::clearPtr(settings); + delete ownedData; + delete builder; + udata_close(memory); + ures_close(bundle); + utrie2_close(trie); + delete unsafeBackwardSet; + uhash_close(maxExpansions); + maxExpansionsInitOnce.reset(); +} + +UBool +CollationTailoring::ensureOwnedData(UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return false; } + if(ownedData == nullptr) { + const Normalizer2Impl *nfcImpl = Normalizer2Factory::getNFCImpl(errorCode); + if(U_FAILURE(errorCode)) { return false; } + ownedData = new CollationData(*nfcImpl); + if(ownedData == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return false; + } + } + data = ownedData; + return true; +} + +void +CollationTailoring::makeBaseVersion(const UVersionInfo ucaVersion, UVersionInfo version) { + version[0] = UCOL_BUILDER_VERSION; + version[1] = (ucaVersion[0] << 3) + ucaVersion[1]; + version[2] = ucaVersion[2] << 6; + version[3] = 0; +} + +void +CollationTailoring::setVersion(const UVersionInfo baseVersion, const UVersionInfo rulesVersion) { + version[0] = UCOL_BUILDER_VERSION; + version[1] = baseVersion[1]; + version[2] = (baseVersion[2] & 0xc0) + ((rulesVersion[0] + (rulesVersion[0] >> 6)) & 0x3f); + version[3] = (rulesVersion[1] << 3) + (rulesVersion[1] >> 5) + rulesVersion[2] + + (rulesVersion[3] << 4) + (rulesVersion[3] >> 4); +} + +int32_t +CollationTailoring::getUCAVersion() const { + return ((int32_t)version[1] << 4) | (version[2] >> 6); +} + +CollationCacheEntry::~CollationCacheEntry() { + SharedObject::clearPtr(tailoring); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/collationtailoring.h b/intl/icu/source/i18n/collationtailoring.h new file mode 100644 index 0000000000..ed7e46e3b9 --- /dev/null +++ b/intl/icu/source/i18n/collationtailoring.h @@ -0,0 +1,117 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013-2014, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* collationtailoring.h +* +* created on: 2013mar12 +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONTAILORING_H__ +#define __COLLATIONTAILORING_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/locid.h" +#include "unicode/unistr.h" +#include "unicode/uversion.h" +#include "collationsettings.h" +#include "uhash.h" +#include "umutex.h" +#include "unifiedcache.h" + + +struct UDataMemory; +struct UResourceBundle; +struct UTrie2; + +U_NAMESPACE_BEGIN + +struct CollationData; + +class UnicodeSet; + +/** + * Collation tailoring data & settings. + * This is a container of values for a collation tailoring + * built from rules or deserialized from binary data. + * + * It is logically immutable: Do not modify its values. + * The fields are public for convenience. + * + * It is shared, reference-counted, and auto-deleted; see SharedObject. + */ +struct U_I18N_API CollationTailoring : public SharedObject { + CollationTailoring(const CollationSettings *baseSettings); + virtual ~CollationTailoring(); + + /** + * Returns true if the constructor could not initialize properly. + */ + UBool isBogus() { return settings == nullptr; } + + UBool ensureOwnedData(UErrorCode &errorCode); + + static void makeBaseVersion(const UVersionInfo ucaVersion, UVersionInfo version); + void setVersion(const UVersionInfo baseVersion, const UVersionInfo rulesVersion); + int32_t getUCAVersion() const; + + // data for sorting etc. + const CollationData *data; // == base data or ownedData + const CollationSettings *settings; // reference-counted + UnicodeString rules; + // The locale is bogus when built from rules or constructed from a binary blob. + // It can then be set by the service registration code which is thread-safe. + mutable Locale actualLocale; + // UCA version u.v.w & rules version r.s.t.q: + // version[0]: builder version (runtime version is mixed in at runtime) + // version[1]: bits 7..3=u, bits 2..0=v + // version[2]: bits 7..6=w, bits 5..0=r + // version[3]= (s<<5)+(s>>3)+t+(q<<4)+(q>>4) + UVersionInfo version; + + // owned objects + CollationData *ownedData; + UObject *builder; + UDataMemory *memory; + UResourceBundle *bundle; + UTrie2 *trie; + UnicodeSet *unsafeBackwardSet; + mutable UHashtable *maxExpansions; + mutable UInitOnce maxExpansionsInitOnce; + +private: + /** + * No copy constructor: A CollationTailoring cannot be copied. + * It is immutable, and the data trie cannot be copied either. + */ + CollationTailoring(const CollationTailoring &other) = delete; +}; + +struct U_I18N_API CollationCacheEntry : public SharedObject { + CollationCacheEntry(const Locale &loc, const CollationTailoring *t) + : validLocale(loc), tailoring(t) { + if(t != nullptr) { + t->addRef(); + } + } + ~CollationCacheEntry(); + + Locale validLocale; + const CollationTailoring *tailoring; +}; + +template<> U_I18N_API +const CollationCacheEntry * +LocaleCacheKey::createObject(const void *creationContext, + UErrorCode &errorCode) const; +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION +#endif // __COLLATIONTAILORING_H__ diff --git a/intl/icu/source/i18n/collationweights.cpp b/intl/icu/source/i18n/collationweights.cpp new file mode 100644 index 0000000000..2351484590 --- /dev/null +++ b/intl/icu/source/i18n/collationweights.cpp @@ -0,0 +1,570 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 1999-2015, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: collationweights.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001mar08 as ucol_wgt.cpp +* created by: Markus W. Scherer +* +* This file contains code for allocating n collation element weights +* between two exclusive limits. +* It is used only internally by the collation tailoring builder. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "cmemory.h" +#include "collation.h" +#include "collationweights.h" +#include "uarrsort.h" +#include "uassert.h" + +#ifdef UCOL_DEBUG +# include +#endif + +U_NAMESPACE_BEGIN + +/* collation element weight allocation -------------------------------------- */ + +/* helper functions for CE weights */ + +static inline uint32_t +getWeightTrail(uint32_t weight, int32_t length) { + return (uint32_t)(weight>>(8*(4-length)))&0xff; +} + +static inline uint32_t +setWeightTrail(uint32_t weight, int32_t length, uint32_t trail) { + length=8*(4-length); + return (uint32_t)((weight&(0xffffff00<>idx; + } else { + // Do not use uint32_t>>32 because on some platforms that does not shift at all + // while we need it to become 0. + // PowerPC: 0xffffffff>>32 = 0 (wanted) + // x86: 0xffffffff>>32 = 0xffffffff (not wanted) + // + // ANSI C99 6.5.7 Bitwise shift operators: + // "If the value of the right operand is negative + // or is greater than or equal to the width of the promoted left operand, + // the behavior is undefined." + mask=0; + } + idx=32-idx; + mask|=0xffffff00< 0); + } + } +} + +uint32_t +CollationWeights::incWeightByOffset(uint32_t weight, int32_t length, int32_t offset) const { + for(;;) { + offset += getWeightByte(weight, length); + if((uint32_t)offset <= maxBytes[length]) { + return setWeightByte(weight, length, offset); + } else { + // Split the offset between this byte and the previous one. + offset -= minBytes[length]; + weight = setWeightByte(weight, length, minBytes[length] + offset % countBytes(length)); + offset /= countBytes(length); + --length; + U_ASSERT(length > 0); + } + } +} + +void +CollationWeights::lengthenRange(WeightRange &range) const { + int32_t length=range.length+1; + range.start=setWeightTrail(range.start, length, minBytes[length]); + range.end=setWeightTrail(range.end, length, maxBytes[length]); + range.count*=countBytes(length); + range.length=length; +} + +/* for uprv_sortArray: sort ranges in weight order */ +static int32_t U_CALLCONV +compareRanges(const void * /*context*/, const void *left, const void *right) { + uint32_t l, r; + + l=((const CollationWeights::WeightRange *)left)->start; + r=((const CollationWeights::WeightRange *)right)->start; + if(lr) { + return 1; + } else { + return 0; + } +} + +UBool +CollationWeights::getWeightRanges(uint32_t lowerLimit, uint32_t upperLimit) { + U_ASSERT(lowerLimit != 0); + U_ASSERT(upperLimit != 0); + + /* get the lengths of the limits */ + int32_t lowerLength=lengthOfWeight(lowerLimit); + int32_t upperLength=lengthOfWeight(upperLimit); + +#ifdef UCOL_DEBUG + printf("length of lower limit 0x%08lx is %ld\n", lowerLimit, lowerLength); + printf("length of upper limit 0x%08lx is %ld\n", upperLimit, upperLength); +#endif + U_ASSERT(lowerLength>=middleLength); + // Permit upperLength=upperLimit) { +#ifdef UCOL_DEBUG + printf("error: no space between lower & upper limits\n"); +#endif + return false; + } + + /* check that neither is a prefix of the other */ + if(lowerLength=upperLimit has caught it */ + + WeightRange lower[5], middle, upper[5]; /* [0] and [1] are not used - this simplifies indexing */ + uprv_memset(lower, 0, sizeof(lower)); + uprv_memset(&middle, 0, sizeof(middle)); + uprv_memset(upper, 0, sizeof(upper)); + + /* + * With the limit lengths of 1..4, there are up to 7 ranges for allocation: + * range minimum length + * lower[4] 4 + * lower[3] 3 + * lower[2] 2 + * middle 1 + * upper[2] 2 + * upper[3] 3 + * upper[4] 4 + * + * We are now going to calculate up to 7 ranges. + * Some of them will typically overlap, so we will then have to merge and eliminate ranges. + */ + uint32_t weight=lowerLimit; + for(int32_t length=lowerLength; length>middleLength; --length) { + uint32_t trail=getWeightTrail(weight, length); + if(trailmiddleLength; --length) { + uint32_t trail=getWeightTrail(weight, length); + if(trail>minBytes[length]) { + upper[length].start=setWeightTrail(weight, length, minBytes[length]); + upper[length].end=decWeightTrail(weight, length); + upper[length].length=length; + upper[length].count=trail-minBytes[length]; + } + weight=truncateWeight(weight, length-1); + } + middle.end=decWeightTrail(weight, middleLength); + + /* set the middle range */ + middle.length=middleLength; + if(middle.end>=middle.start) { + middle.count=(int32_t)((middle.end-middle.start)>>(8*(4-middleLength)))+1; + } else { + /* no middle range, eliminate overlaps */ + for(int32_t length=4; length>middleLength; --length) { + if(lower[length].count>0 && upper[length].count>0) { + // Note: The lowerEnd and upperStart weights are versions of + // lowerLimit and upperLimit (which are lowerLimitupperStart) { + // These two lower and upper ranges collide. + // Since lowerLimitupperStart is only possible + // if the leading bytes are equal + // and lastByte(lowerEnd)>lastByte(upperStart). + U_ASSERT(truncateWeight(lowerEnd, length-1)== + truncateWeight(upperStart, length-1)); + // Intersect these two ranges. + lower[length].end=upper[length].end; + lower[length].count= + (int32_t)getWeightTrail(lower[length].end, length)- + (int32_t)getWeightTrail(lower[length].start, length)+1; + // count might be <=0 in which case there is no room, + // and the range-collecting code below will ignore this range. + merged=true; + } else if(lowerEnd==upperStart) { + // Not possible, unless minByte==maxByte which is not allowed. + U_ASSERT(minBytes[length]countBytes + merged=true; + } + } + if(merged) { + // Remove all shorter ranges. + // There was no room available for them between the ranges we just merged. + upper[length].count=0; + while(--length>middleLength) { + lower[length].count=upper[length].count=0; + } + break; + } + } + } + } + +#ifdef UCOL_DEBUG + /* print ranges */ + for(int32_t length=4; length>=2; --length) { + if(lower[length].count>0) { + printf("lower[%ld] .start=0x%08lx .end=0x%08lx .count=%ld\n", length, lower[length].start, lower[length].end, lower[length].count); + } + } + if(middle.count>0) { + printf("middle .start=0x%08lx .end=0x%08lx .count=%ld\n", middle.start, middle.end, middle.count); + } + for(int32_t length=2; length<=4; ++length) { + if(upper[length].count>0) { + printf("upper[%ld] .start=0x%08lx .end=0x%08lx .count=%ld\n", length, upper[length].start, upper[length].end, upper[length].count); + } + } +#endif + + /* copy the ranges, shortest first, into the result array */ + rangeCount=0; + if(middle.count>0) { + uprv_memcpy(ranges, &middle, sizeof(WeightRange)); + rangeCount=1; + } + for(int32_t length=middleLength+1; length<=4; ++length) { + /* copy upper first so that later the middle range is more likely the first one to use */ + if(upper[length].count>0) { + uprv_memcpy(ranges+rangeCount, upper+length, sizeof(WeightRange)); + ++rangeCount; + } + if(lower[length].count>0) { + uprv_memcpy(ranges+rangeCount, lower+length, sizeof(WeightRange)); + ++rangeCount; + } + } + return rangeCount>0; +} + +UBool +CollationWeights::allocWeightsInShortRanges(int32_t n, int32_t minLength) { + // See if the first few minLength and minLength+1 ranges have enough weights. + for(int32_t i = 0; i < rangeCount && ranges[i].length <= (minLength + 1); ++i) { + if(n <= ranges[i].count) { + // Use the first few minLength and minLength+1 ranges. + if(ranges[i].length > minLength) { + // Reduce the number of weights from the last minLength+1 range + // which might sort before some minLength ranges, + // so that we use all weights in the minLength ranges. + ranges[i].count = n; + } + rangeCount = i + 1; +#ifdef UCOL_DEBUG + printf("take first %ld ranges\n", rangeCount); +#endif + + if(rangeCount>1) { + /* sort the ranges by weight values */ + UErrorCode errorCode=U_ZERO_ERROR; + uprv_sortArray(ranges, rangeCount, sizeof(WeightRange), + compareRanges, nullptr, false, &errorCode); + /* ignore error code: we know that the internal sort function will not fail here */ + } + return true; + } + n -= ranges[i].count; // still >0 + } + return false; +} + +UBool +CollationWeights::allocWeightsInMinLengthRanges(int32_t n, int32_t minLength) { + // See if the minLength ranges have enough weights + // when we split one and lengthen the following ones. + int32_t count = 0; + int32_t minLengthRangeCount; + for(minLengthRangeCount = 0; + minLengthRangeCount < rangeCount && + ranges[minLengthRangeCount].length == minLength; + ++minLengthRangeCount) { + count += ranges[minLengthRangeCount].count; + } + + int32_t nextCountBytes = countBytes(minLength + 1); + if(n > count * nextCountBytes) { return false; } + + // Use the minLength ranges. Merge them, and then split again as necessary. + uint32_t start = ranges[0].start; + uint32_t end = ranges[0].end; + for(int32_t i = 1; i < minLengthRangeCount; ++i) { + if(ranges[i].start < start) { start = ranges[i].start; } + if(ranges[i].end > end) { end = ranges[i].end; } + } + + // Calculate how to split the range between minLength (count1) and minLength+1 (count2). + // Goal: + // count1 + count2 * nextCountBytes = n + // count1 + count2 = count + // These turn into + // (count - count2) + count2 * nextCountBytes = n + // and then into the following count1 & count2 computations. + int32_t count2 = (n - count) / (nextCountBytes - 1); // number of weights to be lengthened + int32_t count1 = count - count2; // number of minLength weights + if(count2 == 0 || (count1 + count2 * nextCountBytes) < n) { + // round up + ++count2; + --count1; + U_ASSERT((count1 + count2 * nextCountBytes) >= n); + } + + ranges[0].start = start; + + if(count1 == 0) { + // Make one long range. + ranges[0].end = end; + ranges[0].count = count; + lengthenRange(ranges[0]); + rangeCount = 1; + } else { + // Split the range, lengthen the second part. +#ifdef UCOL_DEBUG + printf("split the range number %ld (out of %ld minLength ranges) by %ld:%ld\n", + splitRange, rangeCount, count1, count2); +#endif + + // Next start = start + count1. First end = 1 before that. + ranges[0].end = incWeightByOffset(start, minLength, count1 - 1); + ranges[0].count = count1; + + ranges[1].start = incWeight(ranges[0].end, minLength); + ranges[1].end = end; + ranges[1].length = minLength; // +1 when lengthened + ranges[1].count = count2; // *countBytes when lengthened + lengthenRange(ranges[1]); + rangeCount = 2; + } + return true; +} + +/* + * call getWeightRanges and then determine heuristically + * which ranges to use for a given number of weights between (excluding) + * two limits + */ +UBool +CollationWeights::allocWeights(uint32_t lowerLimit, uint32_t upperLimit, int32_t n) { +#ifdef UCOL_DEBUG + puts(""); +#endif + + if(!getWeightRanges(lowerLimit, upperLimit)) { +#ifdef UCOL_DEBUG + printf("error: unable to get Weight ranges\n"); +#endif + return false; + } + + /* try until we find suitably large ranges */ + for(;;) { + /* get the smallest number of bytes in a range */ + int32_t minLength=ranges[0].length; + + if(allocWeightsInShortRanges(n, minLength)) { break; } + + if(minLength == 4) { +#ifdef UCOL_DEBUG + printf("error: the maximum number of %ld weights is insufficient for n=%ld\n", + minLengthCount, n); +#endif + return false; + } + + if(allocWeightsInMinLengthRanges(n, minLength)) { break; } + + /* no good match, lengthen all minLength ranges and iterate */ +#ifdef UCOL_DEBUG + printf("lengthen the short ranges from %ld bytes to %ld and iterate\n", minLength, minLength+1); +#endif + for(int32_t i=0; i= rangeCount) { + return 0xffffffff; + } else { + /* get the next weight */ + WeightRange &range = ranges[rangeIndex]; + uint32_t weight = range.start; + if(--range.count == 0) { + /* this range is finished */ + ++rangeIndex; + } else { + /* increment the weight for the next value */ + range.start = incWeight(weight, range.length); + U_ASSERT(range.start <= range.end); + } + + return weight; + } +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_COLLATION */ diff --git a/intl/icu/source/i18n/collationweights.h b/intl/icu/source/i18n/collationweights.h new file mode 100644 index 0000000000..0d20b927b2 --- /dev/null +++ b/intl/icu/source/i18n/collationweights.h @@ -0,0 +1,113 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 1999-2014, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: collationweights.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001mar08 as ucol_wgt.h +* created by: Markus W. Scherer +*/ + +#ifndef __COLLATIONWEIGHTS_H__ +#define __COLLATIONWEIGHTS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/uobject.h" + +U_NAMESPACE_BEGIN + +/** + * Allocates n collation element weights between two exclusive limits. + * Used only internally by the collation tailoring builder. + */ +class U_I18N_API CollationWeights : public UMemory { +public: + CollationWeights(); + + static inline int32_t lengthOfWeight(uint32_t weight) { + if((weight&0xffffff)==0) { + return 1; + } else if((weight&0xffff)==0) { + return 2; + } else if((weight&0xff)==0) { + return 3; + } else { + return 4; + } + } + + void initForPrimary(UBool compressible); + void initForSecondary(); + void initForTertiary(); + + /** + * Determine heuristically + * what ranges to use for a given number of weights between (excluding) + * two limits. + * + * @param lowerLimit A collation element weight; the ranges will be filled to cover + * weights greater than this one. + * @param upperLimit A collation element weight; the ranges will be filled to cover + * weights less than this one. + * @param n The number of collation element weights w necessary such that + * lowerLimitproperties.compactStyle = style; + fields->properties.groupingSize = -2; // do not forward grouping information + fields->properties.minimumGroupingDigits = 2; + touch(status); +} + +CompactDecimalFormat::CompactDecimalFormat(const CompactDecimalFormat& source) = default; + +CompactDecimalFormat::~CompactDecimalFormat() = default; + +CompactDecimalFormat& CompactDecimalFormat::operator=(const CompactDecimalFormat& rhs) { + DecimalFormat::operator=(rhs); + return *this; +} + +CompactDecimalFormat* CompactDecimalFormat::clone() const { + return new CompactDecimalFormat(*this); +} + +void +CompactDecimalFormat::parse( + const UnicodeString& /* text */, + Formattable& /* result */, + ParsePosition& /* parsePosition */) const { +} + +void +CompactDecimalFormat::parse( + const UnicodeString& /* text */, + Formattable& /* result */, + UErrorCode& status) const { + status = U_UNSUPPORTED_ERROR; +} + +CurrencyAmount* +CompactDecimalFormat::parseCurrency( + const UnicodeString& /* text */, + ParsePosition& /* pos */) const { + return nullptr; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/coptccal.cpp b/intl/icu/source/i18n/coptccal.cpp new file mode 100644 index 0000000000..a957f8f2c5 --- /dev/null +++ b/intl/icu/source/i18n/coptccal.cpp @@ -0,0 +1,182 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2013, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "umutex.h" +#include "coptccal.h" +#include "cecal.h" +#include + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CopticCalendar) + +static const int32_t COPTIC_JD_EPOCH_OFFSET = 1824665; + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +CopticCalendar::CopticCalendar(const Locale& aLocale, UErrorCode& success) +: CECalendar(aLocale, success) +{ +} + +CopticCalendar::CopticCalendar (const CopticCalendar& other) +: CECalendar(other) +{ +} + +CopticCalendar::~CopticCalendar() +{ +} + +CopticCalendar* +CopticCalendar::clone() const +{ + return new CopticCalendar(*this); +} + +const char* +CopticCalendar::getType() const +{ + return "coptic"; +} + +//------------------------------------------------------------------------- +// Calendar framework +//------------------------------------------------------------------------- + +int32_t +CopticCalendar::handleGetExtendedYear() +{ + int32_t eyear; + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + eyear = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + // The year defaults to the epoch start, the era to CE + int32_t era = internalGet(UCAL_ERA, CE); + if (era == BCE) { + eyear = 1 - internalGet(UCAL_YEAR, 1); // Convert to extended year + } else { + eyear = internalGet(UCAL_YEAR, 1); // Default to year 1 + } + } + return eyear; +} + +void +CopticCalendar::handleComputeFields(int32_t julianDay, UErrorCode &/*status*/) +{ + int32_t eyear, month, day, era, year; + jdToCE(julianDay, getJDEpochOffset(), eyear, month, day); + + if (eyear <= 0) { + era = BCE; + year = 1 - eyear; + } else { + era = CE; + year = eyear; + } + + internalSet(UCAL_EXTENDED_YEAR, eyear); + internalSet(UCAL_ERA, era); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DATE, day); + internalSet(UCAL_DAY_OF_YEAR, (30 * month) + day); +} + +constexpr uint32_t kCopticRelatedYearDiff = 284; + +int32_t CopticCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kCopticRelatedYearDiff; +} + +void CopticCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kCopticRelatedYearDiff); +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + + +static void U_CALLCONV initializeSystemDefaultCentury() { + UErrorCode status = U_ZERO_ERROR; + CopticCalendar calendar(Locale("@calendar=coptic"), status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate +CopticCalendar::defaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t +CopticCalendar::defaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + + +int32_t +CopticCalendar::getJDEpochOffset() const +{ + return COPTIC_JD_EPOCH_OFFSET; +} + + +#if 0 +// We do not want to introduce this API in ICU4C. +// It was accidentally introduced in ICU4J as a public API. + +//------------------------------------------------------------------------- +// Calendar system Conversion methods... +//------------------------------------------------------------------------- + +int32_t +CopticCalendar::copticToJD(int32_t year, int32_t month, int32_t day) +{ + return CECalendar::ceToJD(year, month, day, COPTIC_JD_EPOCH_OFFSET); +} +#endif + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +//eof diff --git a/intl/icu/source/i18n/coptccal.h b/intl/icu/source/i18n/coptccal.h new file mode 100644 index 0000000000..396127e8ad --- /dev/null +++ b/intl/icu/source/i18n/coptccal.h @@ -0,0 +1,258 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2013, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef COPTCCAL_H +#define COPTCCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "cecal.h" + +U_NAMESPACE_BEGIN + +/** + * Implement the Coptic calendar system. + * @internal + */ +class CopticCalendar : public CECalendar { + +public: + /** + * Useful constants for CopticCalendar. + * @internal + */ + enum EMonths { + /** + * Constant for ωογτ/تﻮﺗ, + * the 1st month of the Coptic year. + */ + TOUT, + + /** + * Constant for Παοπι/ﻪﺑﺎﺑ, + * the 2nd month of the Coptic year. + */ + BABA, + + /** + * Constant for Αθορ/رﻮﺗﺎﻫ, + * the 3rd month of the Coptic year. + */ + HATOR, + + /** + * Constant for Χοιακ/ﻚﻬﻴﻛ, + * the 4th month of the Coptic year. + */ + KIAHK, + + /** + * Constant for Τωβι/طﻮﺒﻫ, + * the 5th month of the Coptic year. + */ + TOBA, + + /** + * Constant for Μεϣιρ/ﺮﻴﺸﻣأ, + * the 6th month of the Coptic year. + */ + AMSHIR, + + /** + * Constant for Παρεμϩατ/تﺎﻬﻣﺮﺑ, + * the 7th month of the Coptic year. + */ + BARAMHAT, + + /** + * Constant for Φαρμοθι/هدﻮﻣﺮﺑ, + * the 8th month of the Coptic year. + */ + BARAMOUDA, + + /** + * Constant for Παϣαν/ﺲﻨﺸﺑ, + * the 9th month of the Coptic year. + */ + BASHANS, + + /** + * Constant for Παωνι/ﻪﻧؤﻮﺑ, + * the 10th month of the Coptic year. + */ + PAONA, + + /** + * Constant for Επηπ/ﺐﻴﺑأ, + * the 11th month of the Coptic year. + */ + EPEP, + + /** + * Constant for Μεϲωρη/ىﺮﺴﻣ, + * the 12th month of the Coptic year. + */ + MESRA, + + /** + * Constant for Πικογϫι + * μαβοτ/ﺮﻴﻐﺼﻟا + * ﺮﻬﺸﻟا, + * the 13th month of the Coptic year. + */ + NASIE + }; + + enum EEras { + BCE, // Before the epoch + CE // After the epoch + }; + + /** + * Constructs a CopticCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of CopticCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + CopticCalendar(const Locale& aLocale, UErrorCode& success); + + /** + * Copy Constructor + * @internal + */ + CopticCalendar (const CopticCalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~CopticCalendar(); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual CopticCalendar* clone() const override; + + /** + * return the calendar type, "coptic" + * @return calendar type + * @internal + */ + const char * getType() const override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + +protected: + //------------------------------------------------------------------------- + // Calendar framework + //------------------------------------------------------------------------- + + /** + * Return the extended year defined by the current fields. + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Compute fields from the JD + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + + /** + * Return the date offset from Julian + * @internal + */ + virtual int32_t getJDEpochOffset() const override; + + +public: + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + +#if 0 + // We do not want to introduce this API in ICU4C. + // It was accidentally introduced in ICU4J as a public API. +public: + //------------------------------------------------------------------------- + // Calendar system Conversion methods... + //------------------------------------------------------------------------- + /** + * Convert an Coptic year, month, and day to a Julian day. + * + * @param year the extended year + * @param month the month + * @param day the day + * @return Julian day + * @internal + */ + static int32_t copticToJD(int32_t year, int32_t month, int32_t day); +#endif +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif /* COPTCCAL_H */ +//eof diff --git a/intl/icu/source/i18n/cpdtrans.cpp b/intl/icu/source/i18n/cpdtrans.cpp new file mode 100644 index 0000000000..9b10364689 --- /dev/null +++ b/intl/icu/source/i18n/cpdtrans.cpp @@ -0,0 +1,617 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unifilt.h" +#include "unicode/uniset.h" +#include "cpdtrans.h" +#include "uvector.h" +#include "tridpars.h" +#include "cmemory.h" + +// keep in sync with Transliterator +//static const char16_t ID_SEP = 0x002D; /*-*/ +static const char16_t ID_DELIM = 0x003B; /*;*/ +static const char16_t NEWLINE = 10; + +static const char16_t COLON_COLON[] = {0x3A, 0x3A, 0}; //"::" + +U_NAMESPACE_BEGIN + +const char16_t CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass" + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator) + +/** + * Constructs a new compound transliterator given an array of + * transliterators. The array of transliterators may be of any + * length, including zero or one, however, useful compound + * transliterators have at least two components. + * @param transliterators array of Transliterator + * objects + * @param transliteratorCount The number of + * Transliterator objects in transliterators. + * @param filter the filter. Any character for which + * filter.contains() returns false will not be + * altered by this transliterator. If filter is + * null then no filtering is applied. + */ +CompoundTransliterator::CompoundTransliterator( + Transliterator* const transliterators[], + int32_t transliteratorCount, + UnicodeFilter* adoptedFilter) : + Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter), + trans(0), count(0), numAnonymousRBTs(0) { + setTransliterators(transliterators, transliteratorCount); +} + +/** + * Splits an ID of the form "ID;ID;..." into a compound using each + * of the IDs. + * @param id of above form + * @param forward if false, does the list in reverse order, and + * takes the inverse of each ID. + */ +CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, + UTransDirection direction, + UnicodeFilter* adoptedFilter, + UParseError& /*parseError*/, + UErrorCode& status) : + Transliterator(id, adoptedFilter), + trans(0), numAnonymousRBTs(0) { + // TODO add code for parseError...currently unused, but + // later may be used by parsing code... + init(id, direction, true, status); +} + +CompoundTransliterator::CompoundTransliterator(const UnicodeString& id, + UParseError& /*parseError*/, + UErrorCode& status) : + Transliterator(id, 0), // set filter to 0 here! + trans(0), numAnonymousRBTs(0) { + // TODO add code for parseError...currently unused, but + // later may be used by parsing code... + init(id, UTRANS_FORWARD, true, status); +} + + +/** + * Private constructor for use of TransliteratorAlias + */ +CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID, + UVector& list, + UnicodeFilter* adoptedFilter, + int32_t anonymousRBTs, + UParseError& /*parseError*/, + UErrorCode& status) : + Transliterator(newID, adoptedFilter), + trans(0), numAnonymousRBTs(anonymousRBTs) +{ + init(list, UTRANS_FORWARD, false, status); +} + +/** + * Private constructor for Transliterator from a vector of + * transliterators. The caller is responsible for fixing up the + * ID. + */ +CompoundTransliterator::CompoundTransliterator(UVector& list, + UParseError& /*parseError*/, + UErrorCode& status) : + Transliterator(UnicodeString(), nullptr), + trans(0), numAnonymousRBTs(0) +{ + // TODO add code for parseError...currently unused, but + // later may be used by parsing code... + init(list, UTRANS_FORWARD, false, status); + // assume caller will fixup ID +} + +CompoundTransliterator::CompoundTransliterator(UVector& list, + int32_t anonymousRBTs, + UParseError& /*parseError*/, + UErrorCode& status) : + Transliterator(UnicodeString(), nullptr), + trans(0), numAnonymousRBTs(anonymousRBTs) +{ + init(list, UTRANS_FORWARD, false, status); +} + +/** + * Finish constructing a transliterator: only to be called by + * constructors. Before calling init(), set trans and filter to nullptr. + * @param id the id containing ';'-separated entries + * @param direction either FORWARD or REVERSE + * @param idSplitPoint the index into id at which the + * adoptedSplitTransliterator should be inserted, if there is one, or + * -1 if there is none. + * @param adoptedSplitTransliterator a transliterator to be inserted + * before the entry at offset idSplitPoint in the id string. May be + * nullptr to insert no entry. + * @param fixReverseID if true, then reconstruct the ID of reverse + * entries by calling getID() of component entries. Some constructors + * do not require this because they apply a facade ID anyway. + * @param status the error code indicating success or failure + */ +void CompoundTransliterator::init(const UnicodeString& id, + UTransDirection direction, + UBool fixReverseID, + UErrorCode& status) { + // assert(trans == 0); + + if (U_FAILURE(status)) { + return; + } + + UVector list(status); + UnicodeSet* compoundFilter = nullptr; + UnicodeString regenID; + if (!TransliteratorIDParser::parseCompoundID(id, direction, + regenID, list, compoundFilter)) { + status = U_INVALID_ID; + delete compoundFilter; + return; + } + + TransliteratorIDParser::instantiateList(list, status); + + init(list, direction, fixReverseID, status); + + if (compoundFilter != nullptr) { + adoptFilter(compoundFilter); + } +} + +/** + * Finish constructing a transliterator: only to be called by + * constructors. Before calling init(), set trans and filter to nullptr. + * @param list a vector of transliterator objects to be adopted. It + * should NOT be empty. The list should be in declared order. That + * is, it should be in the FORWARD order; if direction is REVERSE then + * the list order will be reversed. + * @param direction either FORWARD or REVERSE + * @param fixReverseID if true, then reconstruct the ID of reverse + * entries by calling getID() of component entries. Some constructors + * do not require this because they apply a facade ID anyway. + * @param status the error code indicating success or failure + */ +void CompoundTransliterator::init(UVector& list, + UTransDirection direction, + UBool fixReverseID, + UErrorCode& status) { + // assert(trans == 0); + + // Allocate array + if (U_SUCCESS(status)) { + count = list.size(); + trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *)); + /* test for nullptr */ + if (trans == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + if (U_FAILURE(status) || trans == 0) { + // assert(trans == 0); + return; + } + + // Move the transliterators from the vector into an array. + // Reverse the order if necessary. + int32_t i; + for (i=0; i 0) { + newID.append(ID_DELIM); + } + newID.append(trans[i]->getID()); + } + setID(newID); + } + + computeMaximumContextLength(); +} + +/** + * Return the IDs of the given list of transliterators, concatenated + * with ID_DELIM delimiting them. Equivalent to the perlish expression + * join(ID_DELIM, map($_.getID(), transliterators). + */ +UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[], + int32_t transCount) { + UnicodeString id; + for (int32_t i=0; i 0) { + id.append(ID_DELIM); + } + id.append(transliterators[i]->getID()); + } + return id; // Return temporary +} + +/** + * Copy constructor. + */ +CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) : + Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) { + *this = t; +} + +/** + * Destructor + */ +CompoundTransliterator::~CompoundTransliterator() { + freeTransliterators(); +} + +void CompoundTransliterator::freeTransliterators() { + if (trans != 0) { + for (int32_t i=0; i count) { + if (trans != nullptr) { + uprv_free(trans); + } + trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *)); + } + count = t.count; + if (trans != nullptr) { + for (i=0; iclone(); + if (trans[i] == nullptr) { + failed = true; + break; + } + } + } + + // if memory allocation failed delete backwards trans array + if (failed && i > 0) { + int32_t n; + for (n = i-1; n >= 0; n--) { + uprv_free(trans[n]); + trans[n] = nullptr; + } + } + numAnonymousRBTs = t.numAnonymousRBTs; + return *this; +} + +/** + * Transliterator API. + */ +CompoundTransliterator* CompoundTransliterator::clone() const { + return new CompoundTransliterator(*this); +} + +/** + * Returns the number of transliterators in this chain. + * @return number of transliterators in this chain. + */ +int32_t CompoundTransliterator::getCount() const { + return count; +} + +/** + * Returns the transliterator at the given index in this chain. + * @param index index into chain, from 0 to getCount() - 1 + * @return transliterator at the given index + */ +const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const { + return *trans[index]; +} + +void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[], + int32_t transCount) { + Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *)); + if (a == nullptr) { + return; + } + int32_t i = 0; + UBool failed = false; + for (i=0; iclone(); + if (a[i] == nullptr) { + failed = true; + break; + } + } + if (failed && i > 0) { + int32_t n; + for (n = i-1; n >= 0; n--) { + uprv_free(a[n]); + a[n] = nullptr; + } + return; + } + adoptTransliterators(a, transCount); +} + +void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[], + int32_t transCount) { + // First free trans[] and set count to zero. Once this is done, + // orphan the filter. Set up the new trans[]. + freeTransliterators(); + trans = adoptedTransliterators; + count = transCount; + computeMaximumContextLength(); + setID(joinIDs(trans, count)); +} + +/** + * Append c to buf, unless buf is empty or buf already ends in c. + */ +static void _smartAppend(UnicodeString& buf, char16_t c) { + if (buf.length() != 0 && + buf.charAt(buf.length() - 1) != c) { + buf.append(c); + } +} + +UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource, + UBool escapeUnprintable) const { + // We do NOT call toRules() on our component transliterators, in + // general. If we have several rule-based transliterators, this + // yields a concatenation of the rules -- not what we want. We do + // handle compound RBT transliterators specially -- those for which + // compoundRBTIndex >= 0. For the transliterator at compoundRBTIndex, + // we do call toRules() recursively. + rulesSource.truncate(0); + if (numAnonymousRBTs >= 1 && getFilter() != nullptr) { + // If we are a compound RBT and if we have a global + // filter, then emit it at the top. + UnicodeString pat; + rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM); + } + for (int32_t i=0; igetID().startsWith(PASS_STRING, 5)) { + trans[i]->toRules(rule, escapeUnprintable); + if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5)) + rule = UNICODE_STRING_SIMPLE("::Null;") + rule; + + // we also use toRules() on CompoundTransliterators (which we + // check for by looking for a semicolon in the ID)-- this gets + // the list of their child transliterators output in the right + // format + } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) { + trans[i]->toRules(rule, escapeUnprintable); + + // for everything else, use Transliterator::toRules() + } else { + trans[i]->Transliterator::toRules(rule, escapeUnprintable); + } + _smartAppend(rulesSource, NEWLINE); + rulesSource.append(rule); + _smartAppend(rulesSource, ID_DELIM); + } + return rulesSource; +} + +/** + * Implement Transliterator framework + */ +void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const { + UnicodeSet set; + result.clear(); + for (int32_t i=0; igetSourceSet(set)); + // Take the example of Hiragana-Latin. This is really + // Hiragana-Katakana; Katakana-Latin. The source set of + // these two is roughly [:Hiragana:] and [:Katakana:]. + // But the source set for the entire transliterator is + // actually [:Hiragana:] ONLY -- that is, the first + // non-empty source set. + + // This is a heuristic, and not 100% reliable. + if (!result.isEmpty()) { + break; + } + } +} + +/** + * Override Transliterator framework + */ +UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const { + UnicodeSet set; + result.clear(); + for (int32_t i=0; igetTargetSet(set)); + } + return result; +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index, + UBool incremental) const { + /* Call each transliterator with the same contextStart and + * start, but with the limit as modified + * by preceding transliterators. The start index must be + * reset for each transliterator to give each a chance to + * transliterate the text. The initial contextStart index is known + * to still point to the same place after each transliterator + * is called because each transliterator will not change the + * text between contextStart and the initial start index. + * + * IMPORTANT: After the first transliterator, each subsequent + * transliterator only gets to transliterate text committed by + * preceding transliterators; that is, the start (output + * value) of transliterator i becomes the limit (input value) + * of transliterator i+1. Finally, the overall limit is fixed + * up before we return. + * + * Assumptions we make here: + * (1) contextStart <= start <= limit <= contextLimit <= text.length() + * (2) start <= start' <= limit' ;cursor doesn't move back + * (3) start <= limit' ;text before cursor unchanged + * - start' is the value of start after calling handleKT + * - limit' is the value of limit after calling handleKT + */ + + /** + * Example: 3 transliterators. This example illustrates the + * mechanics we need to implement. C, S, and L are the contextStart, + * start, and limit. gl is the globalLimit. contextLimit is + * equal to limit throughout. + * + * 1. h-u, changes hex to Unicode + * + * 4 7 a d 0 4 7 a + * abc/u0061/u => abca/u + * C S L C S L gl=f->a + * + * 2. upup, changes "x" to "XX" + * + * 4 7 a 4 7 a + * abca/u => abcAA/u + * C SL C S + * L gl=a->b + * 3. u-h, changes Unicode to hex + * + * 4 7 a 4 7 a d 0 3 + * abcAA/u => abc/u0041/u0041/u + * C S L C S + * L gl=b->15 + * 4. return + * + * 4 7 a d 0 3 + * abc/u0041/u0041/u + * C S L + */ + + if (count < 1) { + index.start = index.limit; + return; // Short circuit for empty compound transliterators + } + + // compoundLimit is the limit value for the entire compound + // operation. We overwrite index.limit with the previous + // index.start. After each transliteration, we update + // compoundLimit for insertions or deletions that have happened. + int32_t compoundLimit = index.limit; + + // compoundStart is the start for the entire compound + // operation. + int32_t compoundStart = index.start; + + int32_t delta = 0; // delta in length + + // Give each transliterator a crack at the run of characters. + // See comments at the top of the method for more detail. + for (int32_t i=0; ifilteredTransliterate(text, index, incremental); + + // In a properly written transliterator, start == limit after + // handleTransliterate() returns when incremental is false. + // Catch cases where the subclass doesn't do this, and throw + // an exception. (Just pinning start to limit is a bad idea, + // because what's probably happening is that the subclass + // isn't transliterating all the way to the end, and it should + // in non-incremental mode.) + if (!incremental && index.start != index.limit) { + // We can't throw an exception, so just fudge things + index.start = index.limit; + } + + // Cumulative delta for insertions/deletions + delta += index.limit - limit; + + if (incremental) { + // In the incremental case, only allow subsequent + // transliterators to modify what has already been + // completely processed by prior transliterators. In the + // non-incrmental case, allow each transliterator to + // process the entire text. + index.limit = index.start; + } + } + + compoundLimit += delta; + + // Start is good where it is -- where the last transliterator left + // it. Limit needs to be put back where it was, modulo + // adjustments for deletions/insertions. + index.limit = compoundLimit; +} + +/** + * Sets the length of the longest context required by this transliterator. + * This is preceding context. + */ +void CompoundTransliterator::computeMaximumContextLength() { + int32_t max = 0; + for (int32_t i=0; igetMaximumContextLength(); + if (len > max) { + max = len; + } + } + setMaximumContextLength(max); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +/* eof */ diff --git a/intl/icu/source/i18n/cpdtrans.h b/intl/icu/source/i18n/cpdtrans.h new file mode 100644 index 0000000000..a27c617c95 --- /dev/null +++ b/intl/icu/source/i18n/cpdtrans.h @@ -0,0 +1,232 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef CPDTRANS_H +#define CPDTRANS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" + +U_NAMESPACE_BEGIN + +class U_COMMON_API UVector; +class TransliteratorRegistry; + +/** + * A transliterator that is composed of two or more other + * transliterator objects linked together. For example, if one + * transliterator transliterates from script A to script B, and + * another transliterates from script B to script C, the two may be + * combined to form a new transliterator from A to C. + * + *

Composed transliterators may not behave as expected. For + * example, inverses may not combine to form the identity + * transliterator. See the class documentation for {@link + * Transliterator} for details. + * + * @author Alan Liu + */ +class U_I18N_API CompoundTransliterator : public Transliterator { + + Transliterator** trans; + + int32_t count; + + int32_t numAnonymousRBTs; + +public: + + /** + * Constructs a new compound transliterator given an array of + * transliterators. The array of transliterators may be of any + * length, including zero or one, however, useful compound + * transliterators have at least two components. + * @param transliterators array of Transliterator + * objects + * @param transliteratorCount The number of + * Transliterator objects in transliterators. + * @param adoptedFilter the filter. Any character for which + * filter.contains() returns false will not be + * altered by this transliterator. If filter is + * null then no filtering is applied. + */ + CompoundTransliterator(Transliterator* const transliterators[], + int32_t transliteratorCount, + UnicodeFilter* adoptedFilter = 0); + + /** + * Constructs a new compound transliterator. + * @param id compound ID + * @param dir either UTRANS_FORWARD or UTRANS_REVERSE + * @param adoptedFilter a global filter for this compound transliterator + * or nullptr + */ + CompoundTransliterator(const UnicodeString& id, + UTransDirection dir, + UnicodeFilter* adoptedFilter, + UParseError& parseError, + UErrorCode& status); + + /** + * Constructs a new compound transliterator in the FORWARD + * direction with a nullptr filter. + */ + CompoundTransliterator(const UnicodeString& id, + UParseError& parseError, + UErrorCode& status); + /** + * Destructor. + */ + virtual ~CompoundTransliterator(); + + /** + * Copy constructor. + */ + CompoundTransliterator(const CompoundTransliterator&); + + /** + * Transliterator API. + */ + virtual CompoundTransliterator* clone() const override; + + /** + * Returns the number of transliterators in this chain. + * @return number of transliterators in this chain. + */ + virtual int32_t getCount() const; + + /** + * Returns the transliterator at the given index in this chain. + * @param idx index into chain, from 0 to getCount() - 1 + * @return transliterator at the given index + */ + virtual const Transliterator& getTransliterator(int32_t idx) const; + + /** + * Sets the transliterators. + */ + void setTransliterators(Transliterator* const transliterators[], + int32_t count); + + /** + * Adopts the transliterators. + */ + void adoptTransliterators(Transliterator* adoptedTransliterators[], + int32_t count); + + /** + * Override Transliterator: + * Create a rule string that can be passed to createFromRules() + * to recreate this transliterator. + * @param result the string to receive the rules. Previous + * contents will be deleted. + * @param escapeUnprintable if true then convert unprintable + * character to their hex escape representations, \uxxxx or + * \Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + */ + virtual UnicodeString& toRules(UnicodeString& result, + UBool escapeUnprintable) const override; + + protected: + /** + * Implement Transliterator framework + */ + virtual void handleGetSourceSet(UnicodeSet& result) const override; + + public: + /** + * Override Transliterator framework + */ + virtual UnicodeSet& getTargetSet(UnicodeSet& result) const override; + +protected: + /** + * Implements {@link Transliterator#handleTransliterate}. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& idx, + UBool incremental) const override; + +public: + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /* @internal */ + static const char16_t PASS_STRING[]; + +private: + + friend class Transliterator; + friend class TransliteratorAlias; // to access private ct + + /** + * Assignment operator. + */ + CompoundTransliterator& operator=(const CompoundTransliterator&); + + /** + * Private constructor for Transliterator. + */ + CompoundTransliterator(const UnicodeString& ID, + UVector& list, + UnicodeFilter* adoptedFilter, + int32_t numAnonymousRBTs, + UParseError& parseError, + UErrorCode& status); + + CompoundTransliterator(UVector& list, + UParseError& parseError, + UErrorCode& status); + + CompoundTransliterator(UVector& list, + int32_t anonymousRBTs, + UParseError& parseError, + UErrorCode& status); + + void init(const UnicodeString& id, + UTransDirection direction, + UBool fixReverseID, + UErrorCode& status); + + void init(UVector& list, + UTransDirection direction, + UBool fixReverseID, + UErrorCode& status); + + /** + * Return the IDs of the given list of transliterators, concatenated + * with ';' delimiting them. Equivalent to the perlish expression + * join(';', map($_.getID(), transliterators). + */ + UnicodeString joinIDs(Transliterator* const transliterators[], + int32_t transCount); + + void freeTransliterators(); + + void computeMaximumContextLength(); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/csdetect.cpp b/intl/icu/source/i18n/csdetect.cpp new file mode 100644 index 0000000000..16004f9f5d --- /dev/null +++ b/intl/icu/source/i18n/csdetect.cpp @@ -0,0 +1,492 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/ucsdet.h" + +#include "csdetect.h" +#include "csmatch.h" +#include "uenumimp.h" + +#include "cmemory.h" +#include "cstring.h" +#include "umutex.h" +#include "ucln_in.h" +#include "uarrsort.h" +#include "inputext.h" +#include "csrsbcs.h" +#include "csrmbcs.h" +#include "csrutf8.h" +#include "csrucode.h" +#include "csr2022.h" + +#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type)) +#define DELETE_ARRAY(array) uprv_free((void *) (array)) + +U_NAMESPACE_BEGIN + +struct CSRecognizerInfo : public UMemory { + CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled) + : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {} + + ~CSRecognizerInfo() {delete recognizer;} + + CharsetRecognizer *recognizer; + UBool isDefaultEnabled; +}; + +U_NAMESPACE_END + +static icu::CSRecognizerInfo **fCSRecognizers = nullptr; +static icu::UInitOnce gCSRecognizersInitOnce {}; +static int32_t fCSRecognizers_size = 0; + +U_CDECL_BEGIN +static UBool U_CALLCONV csdet_cleanup() +{ + U_NAMESPACE_USE + if (fCSRecognizers != nullptr) { + for(int32_t r = 0; r < fCSRecognizers_size; r += 1) { + delete fCSRecognizers[r]; + fCSRecognizers[r] = nullptr; + } + + DELETE_ARRAY(fCSRecognizers); + fCSRecognizers = nullptr; + fCSRecognizers_size = 0; + } + gCSRecognizersInitOnce.reset(); + + return true; +} + +static int32_t U_CALLCONV +charsetMatchComparator(const void * /*context*/, const void *left, const void *right) +{ + U_NAMESPACE_USE + + const CharsetMatch **csm_l = (const CharsetMatch **) left; + const CharsetMatch **csm_r = (const CharsetMatch **) right; + + // NOTE: compare is backwards to sort from highest to lowest. + return (*csm_r)->getConfidence() - (*csm_l)->getConfidence(); +} + +static void U_CALLCONV initRecognizers(UErrorCode &status) { + U_NAMESPACE_USE + ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup); + CSRecognizerInfo *tempArray[] = { + new CSRecognizerInfo(new CharsetRecog_UTF8(), true), + + new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), true), + new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), true), + new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), true), + new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), true), + + new CSRecognizerInfo(new CharsetRecog_8859_1(), true), + new CSRecognizerInfo(new CharsetRecog_8859_2(), true), + new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), true), + new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), true), + new CSRecognizerInfo(new CharsetRecog_8859_7_el(), true), + new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), true), + new CSRecognizerInfo(new CharsetRecog_8859_8_he(), true), + new CSRecognizerInfo(new CharsetRecog_windows_1251(), true), + new CSRecognizerInfo(new CharsetRecog_windows_1256(), true), + new CSRecognizerInfo(new CharsetRecog_KOI8_R(), true), + new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), true), + new CSRecognizerInfo(new CharsetRecog_sjis(), true), + new CSRecognizerInfo(new CharsetRecog_gb_18030(), true), + new CSRecognizerInfo(new CharsetRecog_euc_jp(), true), + new CSRecognizerInfo(new CharsetRecog_euc_kr(), true), + new CSRecognizerInfo(new CharsetRecog_big5(), true), + + new CSRecognizerInfo(new CharsetRecog_2022JP(), true), +#if !UCONFIG_ONLY_HTML_CONVERSION + new CSRecognizerInfo(new CharsetRecog_2022KR(), true), + new CSRecognizerInfo(new CharsetRecog_2022CN(), true), + + new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), false), + new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), false), + new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), false), + new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), false) +#endif + }; + int32_t rCount = UPRV_LENGTHOF(tempArray); + + fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount); + + if (fCSRecognizers == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + else { + fCSRecognizers_size = rCount; + for (int32_t r = 0; r < rCount; r += 1) { + fCSRecognizers[r] = tempArray[r]; + if (fCSRecognizers[r] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + } +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +void CharsetDetector::setRecognizers(UErrorCode &status) +{ + umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status); +} + +CharsetDetector::CharsetDetector(UErrorCode &status) + : textIn(new InputText(status)), resultArray(nullptr), + resultCount(0), fStripTags(false), fFreshTextSet(false), + fEnabledRecognizers(nullptr) +{ + if (U_FAILURE(status)) { + return; + } + + setRecognizers(status); + + if (U_FAILURE(status)) { + return; + } + + resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size); + + if (resultArray == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + for(int32_t i = 0; i < fCSRecognizers_size; i += 1) { + resultArray[i] = new CharsetMatch(); + + if (resultArray[i] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + } +} + +CharsetDetector::~CharsetDetector() +{ + delete textIn; + + for(int32_t i = 0; i < fCSRecognizers_size; i += 1) { + delete resultArray[i]; + } + + uprv_free(resultArray); + + if (fEnabledRecognizers) { + uprv_free(fEnabledRecognizers); + } +} + +void CharsetDetector::setText(const char *in, int32_t len) +{ + textIn->setText(in, len); + fFreshTextSet = true; +} + +UBool CharsetDetector::setStripTagsFlag(UBool flag) +{ + UBool temp = fStripTags; + fStripTags = flag; + fFreshTextSet = true; + return temp; +} + +UBool CharsetDetector::getStripTagsFlag() const +{ + return fStripTags; +} + +void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const +{ + textIn->setDeclaredEncoding(encoding,len); +} + +int32_t CharsetDetector::getDetectableCount() +{ + UErrorCode status = U_ZERO_ERROR; + + setRecognizers(status); + + return fCSRecognizers_size; +} + +const CharsetMatch *CharsetDetector::detect(UErrorCode &status) +{ + int32_t maxMatchesFound = 0; + + detectAll(maxMatchesFound, status); + + if(maxMatchesFound > 0) { + return resultArray[0]; + } else { + return nullptr; + } +} + +const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status) +{ + if(!textIn->isSet()) { + status = U_MISSING_RESOURCE_ERROR;// TODO: Need to set proper status code for input text not set + + return nullptr; + } else if (fFreshTextSet) { + CharsetRecognizer *csr; + int32_t i; + + textIn->MungeInput(fStripTags); + + // Iterate over all possible charsets, remember all that + // give a match quality > 0. + resultCount = 0; + for (i = 0; i < fCSRecognizers_size; i += 1) { + csr = fCSRecognizers[i]->recognizer; + if (csr->match(textIn, resultArray[resultCount])) { + resultCount++; + } + } + + if (resultCount > 1) { + uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, nullptr, true, &status); + } + fFreshTextSet = false; + } + + maxMatchesFound = resultCount; + + if (maxMatchesFound == 0) { + status = U_INVALID_CHAR_FOUND; + return nullptr; + } + + return resultArray; +} + +void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + + int32_t modIdx = -1; + UBool isDefaultVal = false; + for (int32_t i = 0; i < fCSRecognizers_size; i++) { + CSRecognizerInfo *csrinfo = fCSRecognizers[i]; + if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) { + modIdx = i; + isDefaultVal = (csrinfo->isDefaultEnabled == enabled); + break; + } + } + if (modIdx < 0) { + // No matching encoding found + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if (fEnabledRecognizers == nullptr && !isDefaultVal) { + // Create an array storing the non default setting + fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size); + if (fEnabledRecognizers == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + // Initialize the array with default info + for (int32_t i = 0; i < fCSRecognizers_size; i++) { + fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled; + } + } + + if (fEnabledRecognizers != nullptr) { + fEnabledRecognizers[modIdx] = enabled; + } +} + +/*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const +{ + if( index > fCSRecognizers_size-1 || index < 0) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + + return 0; + } else { + return fCSRecognizers[index]->getName(); + } +}*/ + +U_NAMESPACE_END + +U_CDECL_BEGIN +typedef struct { + int32_t currIndex; + UBool all; + UBool *enabledRecognizers; +} Context; + + + +static void U_CALLCONV +enumClose(UEnumeration *en) { + if(en->context != nullptr) { + DELETE_ARRAY(en->context); + } + + DELETE_ARRAY(en); +} + +static int32_t U_CALLCONV +enumCount(UEnumeration *en, UErrorCode *) { + if (((Context *)en->context)->all) { + // ucsdet_getAllDetectableCharsets, all charset detector names + return fCSRecognizers_size; + } + + // Otherwise, ucsdet_getDetectableCharsets - only enabled ones + int32_t count = 0; + UBool *enabledArray = ((Context *)en->context)->enabledRecognizers; + if (enabledArray != nullptr) { + // custom set + for (int32_t i = 0; i < fCSRecognizers_size; i++) { + if (enabledArray[i]) { + count++; + } + } + } else { + // default set + for (int32_t i = 0; i < fCSRecognizers_size; i++) { + if (fCSRecognizers[i]->isDefaultEnabled) { + count++; + } + } + } + return count; +} + +static const char* U_CALLCONV +enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) { + const char *currName = nullptr; + + if (((Context *)en->context)->currIndex < fCSRecognizers_size) { + if (((Context *)en->context)->all) { + // ucsdet_getAllDetectableCharsets, all charset detector names + currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName(); + ((Context *)en->context)->currIndex++; + } else { + // ucsdet_getDetectableCharsets + UBool *enabledArray = ((Context *)en->context)->enabledRecognizers; + if (enabledArray != nullptr) { + // custom set + while (currName == nullptr && ((Context *)en->context)->currIndex < fCSRecognizers_size) { + if (enabledArray[((Context *)en->context)->currIndex]) { + currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName(); + } + ((Context *)en->context)->currIndex++; + } + } else { + // default set + while (currName == nullptr && ((Context *)en->context)->currIndex < fCSRecognizers_size) { + if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) { + currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName(); + } + ((Context *)en->context)->currIndex++; + } + } + } + } + + if(resultLength != nullptr) { + *resultLength = currName == nullptr ? 0 : (int32_t)uprv_strlen(currName); + } + + return currName; +} + + +static void U_CALLCONV +enumReset(UEnumeration *en, UErrorCode *) { + ((Context *)en->context)->currIndex = 0; +} + +static const UEnumeration gCSDetEnumeration = { + nullptr, + nullptr, + enumClose, + enumCount, + uenum_unextDefault, + enumNext, + enumReset +}; + +U_CDECL_END + +U_NAMESPACE_BEGIN + +UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status) +{ + + /* Initialize recognized charsets. */ + setRecognizers(status); + + if(U_FAILURE(status)) { + return 0; + } + + UEnumeration *en = NEW_ARRAY(UEnumeration, 1); + if (en == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration)); + en->context = (void*)NEW_ARRAY(Context, 1); + if (en->context == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + DELETE_ARRAY(en); + return 0; + } + uprv_memset(en->context, 0, sizeof(Context)); + ((Context*)en->context)->all = true; + return en; +} + +UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const +{ + if(U_FAILURE(status)) { + return 0; + } + + UEnumeration *en = NEW_ARRAY(UEnumeration, 1); + if (en == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration)); + en->context = (void*)NEW_ARRAY(Context, 1); + if (en->context == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + DELETE_ARRAY(en); + return 0; + } + uprv_memset(en->context, 0, sizeof(Context)); + ((Context*)en->context)->all = false; + ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers; + return en; +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/csdetect.h b/intl/icu/source/i18n/csdetect.h new file mode 100644 index 0000000000..d4bfa75eef --- /dev/null +++ b/intl/icu/source/i18n/csdetect.h @@ -0,0 +1,69 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSDETECT_H +#define __CSDETECT_H + +#include "unicode/uobject.h" + +#if !UCONFIG_NO_CONVERSION + +#include "unicode/uenum.h" + +U_NAMESPACE_BEGIN + +class InputText; +class CharsetRecognizer; +class CharsetMatch; + +class CharsetDetector : public UMemory +{ +private: + InputText *textIn; + CharsetMatch **resultArray; + int32_t resultCount; + UBool fStripTags; // If true, setText() will strip tags from input text. + UBool fFreshTextSet; + static void setRecognizers(UErrorCode &status); + + UBool *fEnabledRecognizers; // If not null, active set of charset recognizers had + // been changed from the default. The array index is + // corresponding to fCSRecognizers. See setDetectableCharset(). + +public: + CharsetDetector(UErrorCode &status); + + ~CharsetDetector(); + + void setText(const char *in, int32_t len); + + const CharsetMatch * const *detectAll(int32_t &maxMatchesFound, UErrorCode &status); + + const CharsetMatch *detect(UErrorCode& status); + + void setDeclaredEncoding(const char *encoding, int32_t len) const; + + UBool setStripTagsFlag(UBool flag); + + UBool getStripTagsFlag() const; + +// const char *getCharsetName(int32_t index, UErrorCode& status) const; + + static int32_t getDetectableCount(); + + + static UEnumeration * getAllDetectableCharsets(UErrorCode &status); + UEnumeration * getDetectableCharsets(UErrorCode &status) const; + void setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status); +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSDETECT_H */ diff --git a/intl/icu/source/i18n/csmatch.cpp b/intl/icu/source/i18n/csmatch.cpp new file mode 100644 index 0000000000..4c5f73b31b --- /dev/null +++ b/intl/icu/source/i18n/csmatch.cpp @@ -0,0 +1,73 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION +#include "unicode/unistr.h" +#include "unicode/ucnv.h" + +#include "csmatch.h" + +#include "csrecog.h" +#include "inputext.h" + +U_NAMESPACE_BEGIN + +CharsetMatch::CharsetMatch() + : textIn(nullptr), confidence(0), fCharsetName(nullptr), fLang(nullptr) +{ + // nothing else to do. +} + +void CharsetMatch::set(InputText *input, const CharsetRecognizer *cr, int32_t conf, + const char *csName, const char *lang) +{ + textIn = input; + confidence = conf; + fCharsetName = csName; + fLang = lang; + if (cr != nullptr) { + if (fCharsetName == nullptr) { + fCharsetName = cr->getName(); + } + if (fLang == nullptr) { + fLang = cr->getLanguage(); + } + } +} + +const char* CharsetMatch::getName()const +{ + return fCharsetName; +} + +const char* CharsetMatch::getLanguage()const +{ + return fLang; +} + +int32_t CharsetMatch::getConfidence()const +{ + return confidence; +} + +int32_t CharsetMatch::getUChars(char16_t *buf, int32_t cap, UErrorCode *status) const +{ + UConverter *conv = ucnv_open(getName(), status); + int32_t result = ucnv_toUChars(conv, buf, cap, (const char *) textIn->fRawInput, textIn->fRawLength, status); + + ucnv_close(conv); + + return result; +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/csmatch.h b/intl/icu/source/i18n/csmatch.h new file mode 100644 index 0000000000..c31da81863 --- /dev/null +++ b/intl/icu/source/i18n/csmatch.h @@ -0,0 +1,71 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSMATCH_H +#define __CSMATCH_H + +#include "unicode/uobject.h" + +#if !UCONFIG_NO_CONVERSION + +U_NAMESPACE_BEGIN + +class InputText; +class CharsetRecognizer; + +/* + * CharsetMatch represents the results produced by one Charset Recognizer for one input text + * Any confidence > 0 indicates a possible match, meaning that the input bytes + * are at least legal. + * + * The full results of a detect are represented by an array of these + * CharsetMatch objects, each representing a possible matching charset. + * + * Note that a single charset recognizer may detect multiple closely related + * charsets, and set different names depending on the exact input bytes seen. + */ +class CharsetMatch : public UMemory +{ + private: + InputText *textIn; + int32_t confidence; + const char *fCharsetName; + const char *fLang; + + public: + CharsetMatch(); + + /** + * fully set the state of this CharsetMatch. + * Called by the CharsetRecognizers to record match results. + * Default (nullptr) parameters for names will be filled by calling the + * corresponding getters on the recognizer. + */ + void set(InputText *input, + const CharsetRecognizer *cr, + int32_t conf, + const char *csName=nullptr, + const char *lang=nullptr); + + /** + * Return the name of the charset for this Match + */ + const char *getName() const; + + const char *getLanguage()const; + + int32_t getConfidence()const; + + int32_t getUChars(char16_t *buf, int32_t cap, UErrorCode *status) const; +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSMATCH_H */ diff --git a/intl/icu/source/i18n/csr2022.cpp b/intl/icu/source/i18n/csr2022.cpp new file mode 100644 index 0000000000..e064c426a2 --- /dev/null +++ b/intl/icu/source/i18n/csr2022.cpp @@ -0,0 +1,195 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "cmemory.h" +#include "cstring.h" + +#include "csr2022.h" +#include "csmatch.h" + +U_NAMESPACE_BEGIN + +/** + * Matching function shared among the 2022 detectors JP, CN and KR + * Counts up the number of legal and unrecognized escape sequences in + * the sample of text, and computes a score based on the total number & + * the proportion that fit the encoding. + * + * + * @param text the byte buffer containing text to analyse + * @param textLen the size of the text in the byte. + * @param escapeSequences the byte escape sequences to test for. + * @return match quality, in the range of 0-100. + */ +int32_t CharsetRecog_2022::match_2022(const uint8_t *text, int32_t textLen, const uint8_t escapeSequences[][5], int32_t escapeSequences_length) const +{ + int32_t i, j; + int32_t escN; + int32_t hits = 0; + int32_t misses = 0; + int32_t shifts = 0; + int32_t quality; + + i = 0; + while(i < textLen) { + if(text[i] == 0x1B) { + escN = 0; + while(escN < escapeSequences_length) { + const uint8_t *seq = escapeSequences[escN]; + int32_t seq_length = (int32_t)uprv_strlen((const char *) seq); + + if (textLen-i >= seq_length) { + j = 1; + while(j < seq_length) { + if(seq[j] != text[i+j]) { + goto checkEscapes; + } + + j += 1; + } + + hits += 1; + i += seq_length-1; + goto scanInput; + } + // else we ran out of string to compare this time. +checkEscapes: + escN += 1; + } + + misses += 1; + } + + if( text[i]== 0x0e || text[i] == 0x0f){ + shifts += 1; + } + +scanInput: + i += 1; + } + + if (hits == 0) { + return 0; + } + + // + // Initial quality is based on relative proportion of recognized vs. + // unrecognized escape sequences. + // All good: quality = 100; + // half or less good: quality = 0; + // linear inbetween. + quality = (100*hits - 100*misses) / (hits + misses); + + // Back off quality if there were too few escape sequences seen. + // Include shifts in this computation, so that KR does not get penalized + // for having only a single Escape sequence, but many shifts. + if (hits+shifts < 5) { + quality -= (5-(hits+shifts))*10; + } + + if (quality < 0) { + quality = 0; + } + + return quality; +} + + +static const uint8_t escapeSequences_2022JP[][5] = { + {0x1b, 0x24, 0x28, 0x43, 0x00}, // KS X 1001:1992 + {0x1b, 0x24, 0x28, 0x44, 0x00}, // JIS X 212-1990 + {0x1b, 0x24, 0x40, 0x00, 0x00}, // JIS C 6226-1978 + {0x1b, 0x24, 0x41, 0x00, 0x00}, // GB 2312-80 + {0x1b, 0x24, 0x42, 0x00, 0x00}, // JIS X 208-1983 + {0x1b, 0x26, 0x40, 0x00, 0x00}, // JIS X 208 1990, 1997 + {0x1b, 0x28, 0x42, 0x00, 0x00}, // ASCII + {0x1b, 0x28, 0x48, 0x00, 0x00}, // JIS-Roman + {0x1b, 0x28, 0x49, 0x00, 0x00}, // Half-width katakana + {0x1b, 0x28, 0x4a, 0x00, 0x00}, // JIS-Roman + {0x1b, 0x2e, 0x41, 0x00, 0x00}, // ISO 8859-1 + {0x1b, 0x2e, 0x46, 0x00, 0x00} // ISO 8859-7 +}; + +#if !UCONFIG_ONLY_HTML_CONVERSION +static const uint8_t escapeSequences_2022KR[][5] = { + {0x1b, 0x24, 0x29, 0x43, 0x00} +}; + +static const uint8_t escapeSequences_2022CN[][5] = { + {0x1b, 0x24, 0x29, 0x41, 0x00}, // GB 2312-80 + {0x1b, 0x24, 0x29, 0x47, 0x00}, // CNS 11643-1992 Plane 1 + {0x1b, 0x24, 0x2A, 0x48, 0x00}, // CNS 11643-1992 Plane 2 + {0x1b, 0x24, 0x29, 0x45, 0x00}, // ISO-IR-165 + {0x1b, 0x24, 0x2B, 0x49, 0x00}, // CNS 11643-1992 Plane 3 + {0x1b, 0x24, 0x2B, 0x4A, 0x00}, // CNS 11643-1992 Plane 4 + {0x1b, 0x24, 0x2B, 0x4B, 0x00}, // CNS 11643-1992 Plane 5 + {0x1b, 0x24, 0x2B, 0x4C, 0x00}, // CNS 11643-1992 Plane 6 + {0x1b, 0x24, 0x2B, 0x4D, 0x00}, // CNS 11643-1992 Plane 7 + {0x1b, 0x4e, 0x00, 0x00, 0x00}, // SS2 + {0x1b, 0x4f, 0x00, 0x00, 0x00}, // SS3 +}; +#endif + +CharsetRecog_2022JP::~CharsetRecog_2022JP() {} + +const char *CharsetRecog_2022JP::getName() const { + return "ISO-2022-JP"; +} + +UBool CharsetRecog_2022JP::match(InputText *textIn, CharsetMatch *results) const { + int32_t confidence = match_2022(textIn->fInputBytes, + textIn->fInputLen, + escapeSequences_2022JP, + UPRV_LENGTHOF(escapeSequences_2022JP)); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +#if !UCONFIG_ONLY_HTML_CONVERSION +CharsetRecog_2022KR::~CharsetRecog_2022KR() {} + +const char *CharsetRecog_2022KR::getName() const { + return "ISO-2022-KR"; +} + +UBool CharsetRecog_2022KR::match(InputText *textIn, CharsetMatch *results) const { + int32_t confidence = match_2022(textIn->fInputBytes, + textIn->fInputLen, + escapeSequences_2022KR, + UPRV_LENGTHOF(escapeSequences_2022KR)); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_2022CN::~CharsetRecog_2022CN() {} + +const char *CharsetRecog_2022CN::getName() const { + return "ISO-2022-CN"; +} + +UBool CharsetRecog_2022CN::match(InputText *textIn, CharsetMatch *results) const { + int32_t confidence = match_2022(textIn->fInputBytes, + textIn->fInputLen, + escapeSequences_2022CN, + UPRV_LENGTHOF(escapeSequences_2022CN)); + results->set(textIn, this, confidence); + return (confidence > 0); +} +#endif + +CharsetRecog_2022::~CharsetRecog_2022() { + // nothing to do +} + +U_NAMESPACE_END +#endif diff --git a/intl/icu/source/i18n/csr2022.h b/intl/icu/source/i18n/csr2022.h new file mode 100644 index 0000000000..4418728f0e --- /dev/null +++ b/intl/icu/source/i18n/csr2022.h @@ -0,0 +1,95 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2015, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSR2022_H +#define __CSR2022_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +class CharsetMatch; + +/** + * class CharsetRecog_2022 part of the ICU charset detection implementation. + * This is a superclass for the individual detectors for + * each of the detectable members of the ISO 2022 family + * of encodings. + * + * The separate classes are nested within this class. + * + * @internal + */ +class CharsetRecog_2022 : public CharsetRecognizer +{ + +public: + virtual ~CharsetRecog_2022() = 0; + +protected: + + /** + * Matching function shared among the 2022 detectors JP, CN and KR + * Counts up the number of legal an unrecognized escape sequences in + * the sample of text, and computes a score based on the total number & + * the proportion that fit the encoding. + * + * + * @param text the byte buffer containing text to analyse + * @param textLen the size of the text in the byte. + * @param escapeSequences the byte escape sequences to test for. + * @return match quality, in the range of 0-100. + */ + int32_t match_2022(const uint8_t *text, + int32_t textLen, + const uint8_t escapeSequences[][5], + int32_t escapeSequences_length) const; + +}; + +class CharsetRecog_2022JP :public CharsetRecog_2022 +{ +public: + virtual ~CharsetRecog_2022JP(); + + const char *getName() const override; + + UBool match(InputText *textIn, CharsetMatch *results) const override; +}; + +#if !UCONFIG_ONLY_HTML_CONVERSION +class CharsetRecog_2022KR :public CharsetRecog_2022 { +public: + virtual ~CharsetRecog_2022KR(); + + const char *getName() const override; + + UBool match(InputText *textIn, CharsetMatch *results) const override; + +}; + +class CharsetRecog_2022CN :public CharsetRecog_2022 +{ +public: + virtual ~CharsetRecog_2022CN(); + + const char* getName() const override; + + UBool match(InputText *textIn, CharsetMatch *results) const override; +}; +#endif + +U_NAMESPACE_END + +#endif +#endif /* __CSR2022_H */ diff --git a/intl/icu/source/i18n/csrecog.cpp b/intl/icu/source/i18n/csrecog.cpp new file mode 100644 index 0000000000..31fce5dd01 --- /dev/null +++ b/intl/icu/source/i18n/csrecog.cpp @@ -0,0 +1,30 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2006, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +CharsetRecognizer::~CharsetRecognizer() +{ + // nothing to do. +} + +const char *CharsetRecognizer::getLanguage() const +{ + return ""; +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/csrecog.h b/intl/icu/source/i18n/csrecog.h new file mode 100644 index 0000000000..944a5007fe --- /dev/null +++ b/intl/icu/source/i18n/csrecog.h @@ -0,0 +1,57 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSRECOG_H +#define __CSRECOG_H + +#include "unicode/uobject.h" + +#if !UCONFIG_NO_CONVERSION + +#include "inputext.h" + +U_NAMESPACE_BEGIN + +class CharsetMatch; + +class CharsetRecognizer : public UMemory +{ + public: + /** + * Get the IANA name of this charset. + * Note that some recognizers can recognize more than one charset, but that this API + * assumes just one name per recognizer. + * TODO: need to account for multiple names in public API that enumerates over the + * known detectable charsets. + * @return the charset name. + */ + virtual const char *getName() const = 0; + + /** + * Get the ISO language code for this charset. + * @return the language code, or null if the language cannot be determined. + */ + virtual const char *getLanguage() const; + + /* + * Try the given input text against this Charset, and fill in the results object + * with the quality of the match plus other information related to the match. + * + * Return true if the the input bytes are a potential match, and + * false if the input data is not compatible with, or illegal in this charset. + */ + virtual UBool match(InputText *textIn, CharsetMatch *results) const = 0; + + virtual ~CharsetRecognizer(); +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSRECOG_H */ diff --git a/intl/icu/source/i18n/csrmbcs.cpp b/intl/icu/source/i18n/csrmbcs.cpp new file mode 100644 index 0000000000..ec346b5fb3 --- /dev/null +++ b/intl/icu/source/i18n/csrmbcs.cpp @@ -0,0 +1,527 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "cmemory.h" +#include "csmatch.h" +#include "csrmbcs.h" + +#include + +U_NAMESPACE_BEGIN + +#define min(x,y) (((x)<(y))?(x):(y)) + +static const uint16_t commonChars_sjis [] = { +// TODO: This set of data comes from the character frequency- +// of-occurrence analysis tool. The data needs to be moved +// into a resource and loaded from there. +0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0, +0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5, +0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc, +0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341, +0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, +0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa}; + +static const uint16_t commonChars_euc_jp[] = { +// TODO: This set of data comes from the character frequency- +// of-occurrence analysis tool. The data needs to be moved +// into a resource and loaded from there. +0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2, +0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3, +0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4, +0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, +0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, +0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, +0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, +0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, +0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, +0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1}; + +static const uint16_t commonChars_euc_kr[] = { +// TODO: This set of data comes from the character frequency- +// of-occurrence analysis tool. The data needs to be moved +// into a resource and loaded from there. +0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc, +0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9, +0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce, +0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, +0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, +0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, +0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, +0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, +0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, +0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad}; + +static const uint16_t commonChars_big5[] = { +// TODO: This set of data comes from the character frequency- +// of-occurrence analysis tool. The data needs to be moved +// into a resource and loaded from there. +0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446, +0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3, +0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548, +0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8, +0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, +0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, +0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, +0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, +0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, +0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f}; + +static const uint16_t commonChars_gb_18030[] = { +// TODO: This set of data comes from the character frequency- +// of-occurrence analysis tool. The data needs to be moved +// into a resource and loaded from there. +0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac, +0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4, +0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4, +0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, +0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, +0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, +0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, +0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, +0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, +0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0}; + +static int32_t binarySearch(const uint16_t *array, int32_t len, uint16_t value) +{ + int32_t start = 0, end = len-1; + int32_t mid = (start+end)/2; + + while(start <= end) { + if(array[mid] == value) { + return mid; + } + + if(array[mid] < value){ + start = mid+1; + } else { + end = mid-1; + } + + mid = (start+end)/2; + } + + return -1; +} + +IteratedChar::IteratedChar() : +charValue(0), index(-1), nextIndex(0), error(false), done(false) +{ + // nothing else to do. +} + +/*void IteratedChar::reset() +{ + charValue = 0; + index = -1; + nextIndex = 0; + error = false; + done = false; +}*/ + +int32_t IteratedChar::nextByte(InputText *det) +{ + if (nextIndex >= det->fRawLength) { + done = true; + + return -1; + } + + return det->fRawInput[nextIndex++]; +} + +CharsetRecog_mbcs::~CharsetRecog_mbcs() +{ + // nothing to do. +} + +int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars[], int32_t commonCharsLen) const { + int32_t doubleByteCharCount = 0; + int32_t commonCharCount = 0; + int32_t badCharCount = 0; + int32_t totalCharCount = 0; + int32_t confidence = 0; + IteratedChar iter; + + while (nextChar(&iter, det)) { + totalCharCount++; + + if (iter.error) { + badCharCount++; + } else { + if (iter.charValue > 0xFF) { + doubleByteCharCount++; + + if (commonChars != 0) { + if (binarySearch(commonChars, commonCharsLen, static_cast(iter.charValue)) >= 0){ + commonCharCount += 1; + } + } + } + } + + + if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) { + // Bail out early if the byte data is not matching the encoding scheme. + // break detectBlock; + return confidence; + } + } + + if (doubleByteCharCount <= 10 && badCharCount == 0) { + // Not many multi-byte chars. + if (doubleByteCharCount == 0 && totalCharCount < 10) { + // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes. + // We don't have enough data to have any confidence. + // Statistical analysis of single byte non-ASCII characters would probably help here. + confidence = 0; + } + else { + // ASCII or ISO file? It's probably not our encoding, + // but is not incompatible with our encoding, so don't give it a zero. + confidence = 10; + } + + return confidence; + } + + // + // No match if there are too many characters that don't fit the encoding scheme. + // (should we have zero tolerance for these?) + // + if (doubleByteCharCount < 20*badCharCount) { + confidence = 0; + + return confidence; + } + + if (commonChars == 0) { + // We have no statistics on frequently occurring characters. + // Assess confidence purely on having a reasonable number of + // multi-byte characters (the more the better) + confidence = 30 + doubleByteCharCount - 20*badCharCount; + + if (confidence > 100) { + confidence = 100; + } + } else { + // + // Frequency of occurrence statistics exist. + // + + double maxVal = log((double)doubleByteCharCount / 4); /*(float)?*/ + double scaleFactor = 90.0 / maxVal; + confidence = (int32_t)(log((double)commonCharCount+1) * scaleFactor + 10.0); + + confidence = min(confidence, 100); + } + + if (confidence < 0) { + confidence = 0; + } + + return confidence; +} + +CharsetRecog_sjis::~CharsetRecog_sjis() +{ + // nothing to do +} + +UBool CharsetRecog_sjis::nextChar(IteratedChar* it, InputText* det) const { + it->index = it->nextIndex; + it->error = false; + + int32_t firstByte = it->charValue = it->nextByte(det); + + if (firstByte < 0) { + return false; + } + + if (firstByte <= 0x7F || (firstByte > 0xA0 && firstByte <= 0xDF)) { + return true; + } + + int32_t secondByte = it->nextByte(det); + if (secondByte >= 0) { + it->charValue = (firstByte << 8) | secondByte; + } + // else we'll handle the error later. + + if (! ((secondByte >= 0x40 && secondByte <= 0x7F) || (secondByte >= 0x80 && secondByte <= 0xFE))) { + // Illegal second byte value. + it->error = true; + } + + return true; +} + +UBool CharsetRecog_sjis::match(InputText* det, CharsetMatch *results) const { + int32_t confidence = match_mbcs(det, commonChars_sjis, UPRV_LENGTHOF(commonChars_sjis)); + results->set(det, this, confidence); + return (confidence > 0); +} + +const char *CharsetRecog_sjis::getName() const +{ + return "Shift_JIS"; +} + +const char *CharsetRecog_sjis::getLanguage() const +{ + return "ja"; +} + +CharsetRecog_euc::~CharsetRecog_euc() +{ + // nothing to do +} + +UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) const { + int32_t firstByte = 0; + int32_t secondByte = 0; + int32_t thirdByte = 0; + + it->index = it->nextIndex; + it->error = false; + firstByte = it->charValue = it->nextByte(det); + + if (firstByte < 0) { + // Ran off the end of the input data + return false; + } + + if (firstByte <= 0x8D) { + // single byte char + return true; + } + + secondByte = it->nextByte(det); + if (secondByte >= 0) { + it->charValue = (it->charValue << 8) | secondByte; + } + // else we'll handle the error later. + + if (firstByte >= 0xA1 && firstByte <= 0xFE) { + // Two byte Char + if (secondByte < 0xA1) { + it->error = true; + } + + return true; + } + + if (firstByte == 0x8E) { + // Code Set 2. + // In EUC-JP, total char size is 2 bytes, only one byte of actual char value. + // In EUC-TW, total char size is 4 bytes, three bytes contribute to char value. + // We don't know which we've got. + // Treat it like EUC-JP. If the data really was EUC-TW, the following two + // bytes will look like a well formed 2 byte char. + if (secondByte < 0xA1) { + it->error = true; + } + + return true; + } + + if (firstByte == 0x8F) { + // Code set 3. + // Three byte total char size, two bytes of actual char value. + thirdByte = it->nextByte(det); + it->charValue = (it->charValue << 8) | thirdByte; + + if (thirdByte < 0xa1) { + // Bad second byte or ran off the end of the input data with a non-ASCII first byte. + it->error = true; + } + } + + return true; + +} + +CharsetRecog_euc_jp::~CharsetRecog_euc_jp() +{ + // nothing to do +} + +const char *CharsetRecog_euc_jp::getName() const +{ + return "EUC-JP"; +} + +const char *CharsetRecog_euc_jp::getLanguage() const +{ + return "ja"; +} + +UBool CharsetRecog_euc_jp::match(InputText *det, CharsetMatch *results) const +{ + int32_t confidence = match_mbcs(det, commonChars_euc_jp, UPRV_LENGTHOF(commonChars_euc_jp)); + results->set(det, this, confidence); + return (confidence > 0); +} + +CharsetRecog_euc_kr::~CharsetRecog_euc_kr() +{ + // nothing to do +} + +const char *CharsetRecog_euc_kr::getName() const +{ + return "EUC-KR"; +} + +const char *CharsetRecog_euc_kr::getLanguage() const +{ + return "ko"; +} + +UBool CharsetRecog_euc_kr::match(InputText *det, CharsetMatch *results) const +{ + int32_t confidence = match_mbcs(det, commonChars_euc_kr, UPRV_LENGTHOF(commonChars_euc_kr)); + results->set(det, this, confidence); + return (confidence > 0); +} + +CharsetRecog_big5::~CharsetRecog_big5() +{ + // nothing to do +} + +UBool CharsetRecog_big5::nextChar(IteratedChar* it, InputText* det) const +{ + int32_t firstByte; + + it->index = it->nextIndex; + it->error = false; + firstByte = it->charValue = it->nextByte(det); + + if (firstByte < 0) { + return false; + } + + if (firstByte <= 0x7F || firstByte == 0xFF) { + // single byte character. + return true; + } + + int32_t secondByte = it->nextByte(det); + if (secondByte >= 0) { + it->charValue = (it->charValue << 8) | secondByte; + } + // else we'll handle the error later. + + if (secondByte < 0x40 || secondByte == 0x7F || secondByte == 0xFF) { + it->error = true; + } + + return true; +} + +const char *CharsetRecog_big5::getName() const +{ + return "Big5"; +} + +const char *CharsetRecog_big5::getLanguage() const +{ + return "zh"; +} + +UBool CharsetRecog_big5::match(InputText *det, CharsetMatch *results) const +{ + int32_t confidence = match_mbcs(det, commonChars_big5, UPRV_LENGTHOF(commonChars_big5)); + results->set(det, this, confidence); + return (confidence > 0); +} + +CharsetRecog_gb_18030::~CharsetRecog_gb_18030() +{ + // nothing to do +} + +UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) const { + int32_t firstByte = 0; + int32_t secondByte = 0; + int32_t thirdByte = 0; + int32_t fourthByte = 0; + + it->index = it->nextIndex; + it->error = false; + firstByte = it->charValue = it->nextByte(det); + + if (firstByte < 0) { + // Ran off the end of the input data + return false; + } + + if (firstByte <= 0x80) { + // single byte char + return true; + } + + secondByte = it->nextByte(det); + if (secondByte >= 0) { + it->charValue = (it->charValue << 8) | secondByte; + } + // else we'll handle the error later. + + if (firstByte >= 0x81 && firstByte <= 0xFE) { + // Two byte Char + if ((secondByte >= 0x40 && secondByte <= 0x7E) || (secondByte >=80 && secondByte <= 0xFE)) { + return true; + } + + // Four byte char + if (secondByte >= 0x30 && secondByte <= 0x39) { + thirdByte = it->nextByte(det); + + if (thirdByte >= 0x81 && thirdByte <= 0xFE) { + fourthByte = it->nextByte(det); + + if (fourthByte >= 0x30 && fourthByte <= 0x39) { + it->charValue = (it->charValue << 16) | (thirdByte << 8) | fourthByte; + + return true; + } + } + } + + // Something wasn't valid, or we ran out of data (-1). + it->error = true; + } + + return true; +} + +const char *CharsetRecog_gb_18030::getName() const +{ + return "GB18030"; +} + +const char *CharsetRecog_gb_18030::getLanguage() const +{ + return "zh"; +} + +UBool CharsetRecog_gb_18030::match(InputText *det, CharsetMatch *results) const +{ + int32_t confidence = match_mbcs(det, commonChars_gb_18030, UPRV_LENGTHOF(commonChars_gb_18030)); + results->set(det, this, confidence); + return (confidence > 0); +} + +U_NAMESPACE_END +#endif diff --git a/intl/icu/source/i18n/csrmbcs.h b/intl/icu/source/i18n/csrmbcs.h new file mode 100644 index 0000000000..ff7fc4e2a7 --- /dev/null +++ b/intl/icu/source/i18n/csrmbcs.h @@ -0,0 +1,207 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSRMBCS_H +#define __CSRMBCS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +// "Character" iterated character class. +// Recognizers for specific mbcs encodings make their "characters" available +// by providing a nextChar() function that fills in an instance of IteratedChar +// with the next char from the input. +// The returned characters are not converted to Unicode, but remain as the raw +// bytes (concatenated into an int) from the codepage data. +// +// For Asian charsets, use the raw input rather than the input that has been +// stripped of markup. Detection only considers multi-byte chars, effectively +// stripping markup anyway, and double byte chars do occur in markup too. +// +class IteratedChar : public UMemory +{ +public: + uint32_t charValue; // 1-4 bytes from the raw input data + int32_t index; + int32_t nextIndex; + UBool error; + UBool done; + +public: + IteratedChar(); + //void reset(); + int32_t nextByte(InputText* det); +}; + + +class CharsetRecog_mbcs : public CharsetRecognizer { + +protected: + /** + * Test the match of this charset with the input text data + * which is obtained via the CharsetDetector object. + * + * @param det The CharsetDetector, which contains the input text + * to be checked for being in this charset. + * @return Two values packed into one int (Damn java, anyhow) + *
+ * bits 0-7: the match confidence, ranging from 0-100 + *
+ * bits 8-15: The match reason, an enum-like value. + */ + int32_t match_mbcs(InputText* det, const uint16_t commonChars[], int32_t commonCharsLen) const; + +public: + + virtual ~CharsetRecog_mbcs(); + + /** + * Get the IANA name of this charset. + * @return the charset name. + */ + + const char *getName() const override = 0; + const char *getLanguage() const override = 0; + UBool match(InputText* input, CharsetMatch *results) const override = 0; + + /** + * Get the next character (however many bytes it is) from the input data + * Subclasses for specific charset encodings must implement this function + * to get characters according to the rules of their encoding scheme. + * + * This function is not a method of class IteratedChar only because + * that would require a lot of extra derived classes, which is awkward. + * @param it The IteratedChar "struct" into which the returned char is placed. + * @param det The charset detector, which is needed to get at the input byte data + * being iterated over. + * @return True if a character was returned, false at end of input. + */ + virtual UBool nextChar(IteratedChar *it, InputText *textIn) const = 0; + +}; + + +/** + * Shift-JIS charset recognizer. + * + */ +class CharsetRecog_sjis : public CharsetRecog_mbcs { +public: + virtual ~CharsetRecog_sjis(); + + UBool nextChar(IteratedChar *it, InputText *det) const override; + + UBool match(InputText* input, CharsetMatch *results) const override; + + const char *getName() const override; + const char *getLanguage() const override; + +}; + + +/** + * EUC charset recognizers. One abstract class that provides the common function + * for getting the next character according to the EUC encoding scheme, + * and nested derived classes for EUC_KR, EUC_JP, EUC_CN. + * + */ +class CharsetRecog_euc : public CharsetRecog_mbcs +{ +public: + virtual ~CharsetRecog_euc(); + + const char *getName() const override = 0; + const char *getLanguage() const override = 0; + + UBool match(InputText* input, CharsetMatch *results) const override = 0; + /* + * (non-Javadoc) + * Get the next character value for EUC based encodings. + * Character "value" is simply the raw bytes that make up the character + * packed into an int. + */ + UBool nextChar(IteratedChar *it, InputText *det) const override; +}; + +/** + * The charset recognize for EUC-JP. A singleton instance of this class + * is created and kept by the public CharsetDetector class + */ +class CharsetRecog_euc_jp : public CharsetRecog_euc +{ +public: + virtual ~CharsetRecog_euc_jp(); + + const char *getName() const override; + const char *getLanguage() const override; + + UBool match(InputText* input, CharsetMatch *results) const override; +}; + +/** + * The charset recognize for EUC-KR. A singleton instance of this class + * is created and kept by the public CharsetDetector class + */ +class CharsetRecog_euc_kr : public CharsetRecog_euc +{ +public: + virtual ~CharsetRecog_euc_kr(); + + const char *getName() const override; + const char *getLanguage() const override; + + UBool match(InputText* input, CharsetMatch *results) const override; +}; + +/** + * + * Big5 charset recognizer. + * + */ +class CharsetRecog_big5 : public CharsetRecog_mbcs +{ +public: + virtual ~CharsetRecog_big5(); + + UBool nextChar(IteratedChar* it, InputText* det) const override; + + const char *getName() const override; + const char *getLanguage() const override; + + UBool match(InputText* input, CharsetMatch *results) const override; +}; + + +/** + * + * GB-18030 recognizer. Uses simplified Chinese statistics. + * + */ +class CharsetRecog_gb_18030 : public CharsetRecog_mbcs +{ +public: + virtual ~CharsetRecog_gb_18030(); + + UBool nextChar(IteratedChar* it, InputText* det) const override; + + const char *getName() const override; + const char *getLanguage() const override; + + UBool match(InputText* input, CharsetMatch *results) const override; +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSRMBCS_H */ diff --git a/intl/icu/source/i18n/csrsbcs.cpp b/intl/icu/source/i18n/csrsbcs.cpp new file mode 100644 index 0000000000..92af9b5291 --- /dev/null +++ b/intl/icu/source/i18n/csrsbcs.cpp @@ -0,0 +1,1271 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#include "cmemory.h" + +#if !UCONFIG_NO_CONVERSION +#include "csrsbcs.h" +#include "csmatch.h" + +#define N_GRAM_SIZE 3 +#define N_GRAM_MASK 0xFFFFFF + +U_NAMESPACE_BEGIN + +NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap) + : ngram(0), byteIndex(0) +{ + ngramList = theNgramList; + charMap = theCharMap; + + ngramCount = hitCount = 0; +} + +NGramParser::~NGramParser() +{ +} + +/* + * Binary search for value in table, which must have exactly 64 entries. + */ + +int32_t NGramParser::search(const int32_t *table, int32_t value) +{ + int32_t index = 0; + + if (table[index + 32] <= value) { + index += 32; + } + + if (table[index + 16] <= value) { + index += 16; + } + + if (table[index + 8] <= value) { + index += 8; + } + + if (table[index + 4] <= value) { + index += 4; + } + + if (table[index + 2] <= value) { + index += 2; + } + + if (table[index + 1] <= value) { + index += 1; + } + + if (table[index] > value) { + index -= 1; + } + + if (index < 0 || table[index] != value) { + return -1; + } + + return index; +} + +void NGramParser::lookup(int32_t thisNgram) +{ + ngramCount += 1; + + if (search(ngramList, thisNgram) >= 0) { + hitCount += 1; + } + +} + +void NGramParser::addByte(int32_t b) +{ + ngram = ((ngram << 8) + b) & N_GRAM_MASK; + lookup(ngram); +} + +int32_t NGramParser::nextByte(InputText *det) +{ + if (byteIndex >= det->fInputLen) { + return -1; + } + + return det->fInputBytes[byteIndex++]; +} + +void NGramParser::parseCharacters(InputText *det) +{ + int32_t b; + bool ignoreSpace = false; + + while ((b = nextByte(det)) >= 0) { + uint8_t mb = charMap[b]; + + // TODO: 0x20 might not be a space in all character sets... + if (mb != 0) { + if (!(mb == 0x20 && ignoreSpace)) { + addByte(mb); + } + + ignoreSpace = (mb == 0x20); + } + } +} + +int32_t NGramParser::parse(InputText *det) +{ + parseCharacters(det); + + // TODO: Is this OK? The buffer could have ended in the middle of a word... + addByte(0x20); + + double rawPercent = (double) hitCount / (double) ngramCount; + + // if (rawPercent <= 2.0) { + // return 0; + // } + + // TODO - This is a bit of a hack to take care of a case + // were we were getting a confidence of 135... + if (rawPercent > 0.33) { + return 98; + } + + return (int32_t) (rawPercent * 300.0); +} + +#if !UCONFIG_ONLY_HTML_CONVERSION +static const uint8_t unshapeMap_IBM420[] = { +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 4- */ 0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, +/* 5- */ 0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, +/* 6- */ 0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, +/* 7- */ 0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, +/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F, +/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E, +/* A- */ 0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF, +/* B- */ 0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF, +/* C- */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF, +/* D- */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF, +/* E- */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, +/* F- */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +}; + +NGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap) +{ + alef = 0x00; +} + +NGramParser_IBM420::~NGramParser_IBM420() {} + +int32_t NGramParser_IBM420::isLamAlef(int32_t b) +{ + if(b == 0xB2 || b == 0xB3){ + return 0x47; + }else if(b == 0xB4 || b == 0xB5){ + return 0x49; + }else if(b == 0xB8 || b == 0xB9){ + return 0x56; + }else + return 0x00; +} + +/* +* Arabic shaping needs to be done manually. Cannot call ArabicShaping class +* because CharsetDetector is dealing with bytes not Unicode code points. We could +* convert the bytes to Unicode code points but that would leave us dependent +* on CharsetICU which we try to avoid. IBM420 converter amongst different versions +* of JDK can produce different results and therefore is also avoided. +*/ +int32_t NGramParser_IBM420::nextByte(InputText *det) +{ + + if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) { + return -1; + } + int next; + + alef = isLamAlef(det->fInputBytes[byteIndex]); + if(alef != 0x00) + next = 0xB1 & 0xFF; + else + next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF; + + byteIndex++; + + return next; +} + +void NGramParser_IBM420::parseCharacters(InputText *det) +{ + int32_t b; + bool ignoreSpace = false; + + while ((b = nextByte(det)) >= 0) { + uint8_t mb = charMap[b]; + + // TODO: 0x20 might not be a space in all character sets... + if (mb != 0) { + if (!(mb == 0x20 && ignoreSpace)) { + addByte(mb); + } + ignoreSpace = (mb == 0x20); + } + + if(alef != 0x00){ + mb = charMap[alef & 0xFF]; + + // TODO: 0x20 might not be a space in all character sets... + if (mb != 0) { + if (!(mb == 0x20 && ignoreSpace)) { + addByte(mb); + } + + ignoreSpace = (mb == 0x20); + } + + } + } +} +#endif + +CharsetRecog_sbcs::CharsetRecog_sbcs() +{ + // nothing else to do +} + +CharsetRecog_sbcs::~CharsetRecog_sbcs() +{ + // nothing to do +} + +int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const +{ + NGramParser parser(ngrams, byteMap); + int32_t result; + + result = parser.parse(det); + + return result; +} + +static const uint8_t charMap_8859_1[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +}; + +static const uint8_t charMap_8859_2[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20, + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7, + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, +}; + +static const uint8_t charMap_8859_5[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, +}; + +static const uint8_t charMap_8859_6[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, +}; + +static const uint8_t charMap_8859_7[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20, + 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE, + 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, +}; + +static const uint8_t charMap_8859_8[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20, +}; + +static const uint8_t charMap_8859_9[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +}; + +static const int32_t ngrams_windows_1251[] = { + 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE, + 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED, + 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, + 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520, +}; + +static const uint8_t charMap_windows_1251[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, + 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20, + 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF, + 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20, + 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +}; + +static const int32_t ngrams_windows_1256[] = { + 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8, + 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD, + 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, + 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420, +}; + +static const uint8_t charMap_windows_1256[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, + 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20, + 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF, +}; + +static const int32_t ngrams_KOI8_R[] = { + 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1, + 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE, + 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, + 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF, +}; + +static const uint8_t charMap_KOI8_R[] = { + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, +}; + +#if !UCONFIG_ONLY_HTML_CONVERSION +static const int32_t ngrams_IBM424_he_rtl[] = { + 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, + 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, + 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, + 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069, +}; + +static const int32_t ngrams_IBM424_he_ltr[] = { + 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141, + 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054, + 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, + 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651, +}; + +static const uint8_t charMap_IBM424_he[] = { +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40, +/* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +}; + +static const int32_t ngrams_IBM420_ar_rtl[] = { + 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158, + 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB, + 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, + 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40, +}; + +static const int32_t ngrams_IBM420_ar_ltr[] = { + 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, + 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD, + 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, + 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156 +}; + +static const uint8_t charMap_IBM420_ar[]= { +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, +/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, +/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, +/* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, +/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF, +/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, +/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF, +/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40, +}; +#endif + +//ISO-8859-1,2,5,6,7,8,9 Ngrams + +struct NGramsPlusLang { + const int32_t ngrams[64]; + const char * lang; +}; + +static const NGramsPlusLang ngrams_8859_1[] = { + { + { + 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F, + 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74, + 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420, + 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320, + }, + "en" + }, + { + { + 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620, + 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320, + 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520, + 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572, + }, + "da" + }, + { + { + 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F, + 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220, + 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465, + 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572, + }, + "de" + }, + { + { + 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, + 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C, + 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064, + 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20, + }, + "es" + }, + { + { + 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E, + 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20, + 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420, + 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220, + }, + "fr" + }, + { + { + 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073, + 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220, + 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20, + 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F, + }, + "it" + }, + { + { + 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665, + 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E, + 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F, + 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F, + }, + "nl" + }, + { + { + 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469, + 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474, + 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65, + 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572, + }, + "no" + }, + { + { + 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, + 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20, + 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065, + 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F, + }, + "pt" + }, + { + { + 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469, + 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220, + 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20, + 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220, + }, + "sv" + } +}; + + +static const NGramsPlusLang ngrams_8859_2[] = { + { + { + 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F, + 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465, + 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865, + 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564, + }, + "cs" + }, + { + { + 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69, + 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20, + 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061, + 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320, + }, + "hu" + }, + { + { + 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779, + 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20, + 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769, + 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720, + }, + "pl" + }, + { + { + 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69, + 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070, + 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72, + 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20, + }, + "ro" + } +}; + +static const int32_t ngrams_8859_5_ru[] = { + 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE, + 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD, + 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, + 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520, +}; + +static const int32_t ngrams_8859_6_ar[] = { + 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8, + 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1, + 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, + 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620, +}; + +static const int32_t ngrams_8859_7_el[] = { + 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7, + 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120, + 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, + 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20, +}; + +static const int32_t ngrams_8859_8_I_he[] = { + 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0, + 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4, + 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, + 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9, +}; + +static const int32_t ngrams_8859_8_he[] = { + 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0, + 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC, + 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, + 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9, +}; + +static const int32_t ngrams_8859_9_tr[] = { + 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961, + 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062, + 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, + 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD, +}; + +CharsetRecog_8859_1::~CharsetRecog_8859_1() +{ + // nothing to do +} + +UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const { + const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1"; + uint32_t i; + int32_t bestConfidenceSoFar = -1; + for (i=0; i < UPRV_LENGTHOF(ngrams_8859_1) ; i++) { + const int32_t *ngrams = ngrams_8859_1[i].ngrams; + const char *lang = ngrams_8859_1[i].lang; + int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1); + if (confidence > bestConfidenceSoFar) { + results->set(textIn, this, confidence, name, lang); + bestConfidenceSoFar = confidence; + } + } + return (bestConfidenceSoFar > 0); +} + +const char *CharsetRecog_8859_1::getName() const +{ + return "ISO-8859-1"; +} + + +CharsetRecog_8859_2::~CharsetRecog_8859_2() +{ + // nothing to do +} + +UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const { + const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2"; + uint32_t i; + int32_t bestConfidenceSoFar = -1; + for (i=0; i < UPRV_LENGTHOF(ngrams_8859_2) ; i++) { + const int32_t *ngrams = ngrams_8859_2[i].ngrams; + const char *lang = ngrams_8859_2[i].lang; + int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2); + if (confidence > bestConfidenceSoFar) { + results->set(textIn, this, confidence, name, lang); + bestConfidenceSoFar = confidence; + } + } + return (bestConfidenceSoFar > 0); +} + +const char *CharsetRecog_8859_2::getName() const +{ + return "ISO-8859-2"; +} + + +CharsetRecog_8859_5::~CharsetRecog_8859_5() +{ + // nothing to do +} + +const char *CharsetRecog_8859_5::getName() const +{ + return "ISO-8859-5"; +} + +CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru() +{ + // nothing to do +} + +const char *CharsetRecog_8859_5_ru::getLanguage() const +{ + return "ru"; +} + +UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_8859_6::~CharsetRecog_8859_6() +{ + // nothing to do +} + +const char *CharsetRecog_8859_6::getName() const +{ + return "ISO-8859-6"; +} + +CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar() +{ + // nothing to do +} + +const char *CharsetRecog_8859_6_ar::getLanguage() const +{ + return "ar"; +} + +UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_8859_7::~CharsetRecog_8859_7() +{ + // nothing to do +} + +const char *CharsetRecog_8859_7::getName() const +{ + return "ISO-8859-7"; +} + +CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el() +{ + // nothing to do +} + +const char *CharsetRecog_8859_7_el::getLanguage() const +{ + return "el"; +} + +UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const +{ + const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7"; + int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7); + results->set(textIn, this, confidence, name, "el"); + return (confidence > 0); +} + +CharsetRecog_8859_8::~CharsetRecog_8859_8() +{ + // nothing to do +} + +const char *CharsetRecog_8859_8::getName() const +{ + return "ISO-8859-8"; +} + +CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he () +{ + // nothing to do +} + +const char *CharsetRecog_8859_8_I_he::getName() const +{ + return "ISO-8859-8-I"; +} + +const char *CharsetRecog_8859_8_I_he::getLanguage() const +{ + return "he"; +} + +UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const +{ + const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I"; + int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8); + results->set(textIn, this, confidence, name, "he"); + return (confidence > 0); +} + +CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he() +{ + // od ot gnihton +} + +const char *CharsetRecog_8859_8_he::getLanguage() const +{ + return "he"; +} + +UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const +{ + const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8"; + int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8); + results->set(textIn, this, confidence, name, "he"); + return (confidence > 0); +} + +CharsetRecog_8859_9::~CharsetRecog_8859_9() +{ + // nothing to do +} + +const char *CharsetRecog_8859_9::getName() const +{ + return "ISO-8859-9"; +} + +CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr () +{ + // nothing to do +} + +const char *CharsetRecog_8859_9_tr::getLanguage() const +{ + return "tr"; +} + +UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const +{ + const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9"; + int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9); + results->set(textIn, this, confidence, name, "tr"); + return (confidence > 0); +} + +CharsetRecog_windows_1256::~CharsetRecog_windows_1256() +{ + // nothing to do +} + +const char *CharsetRecog_windows_1256::getName() const +{ + return "windows-1256"; +} + +const char *CharsetRecog_windows_1256::getLanguage() const +{ + return "ar"; +} + +UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_windows_1251::~CharsetRecog_windows_1251() +{ + // nothing to do +} + +const char *CharsetRecog_windows_1251::getName() const +{ + return "windows-1251"; +} + +const char *CharsetRecog_windows_1251::getLanguage() const +{ + return "ru"; +} + +UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R() +{ + // nothing to do +} + +const char *CharsetRecog_KOI8_R::getName() const +{ + return "KOI8-R"; +} + +const char *CharsetRecog_KOI8_R::getLanguage() const +{ + return "ru"; +} + +UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +#if !UCONFIG_ONLY_HTML_CONVERSION +CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he() +{ + // nothing to do +} + +const char *CharsetRecog_IBM424_he::getLanguage() const +{ + return "he"; +} + +CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl() +{ + // nothing to do +} + +const char *CharsetRecog_IBM424_he_rtl::getName() const +{ + return "IBM424_rtl"; +} + +UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr() +{ + // nothing to do +} + +const char *CharsetRecog_IBM424_he_ltr::getName() const +{ + return "IBM424_ltr"; +} + +UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar() +{ + // nothing to do +} + +const char *CharsetRecog_IBM420_ar::getLanguage() const +{ + return "ar"; +} + + +int32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const +{ + NGramParser_IBM420 parser(ngrams, byteMap); + int32_t result; + + result = parser.parse(det); + + return result; +} + +CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl() +{ + // nothing to do +} + +const char *CharsetRecog_IBM420_ar_rtl::getName() const +{ + return "IBM420_rtl"; +} + +UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar); + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr() +{ + // nothing to do +} + +const char *CharsetRecog_IBM420_ar_ltr::getName() const +{ + return "IBM420_ltr"; +} + +UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const +{ + int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar); + results->set(textIn, this, confidence); + return (confidence > 0); +} +#endif + +U_NAMESPACE_END +#endif + diff --git a/intl/icu/source/i18n/csrsbcs.h b/intl/icu/source/i18n/csrsbcs.h new file mode 100644 index 0000000000..96f982c59b --- /dev/null +++ b/intl/icu/source/i18n/csrsbcs.h @@ -0,0 +1,295 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2015, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSRSBCS_H +#define __CSRSBCS_H + +#include "unicode/uobject.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +class NGramParser : public UMemory +{ +private: + int32_t ngram; + const int32_t *ngramList; + + int32_t ngramCount; + int32_t hitCount; + +protected: + int32_t byteIndex; + const uint8_t *charMap; + + void addByte(int32_t b); + +public: + NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap); + virtual ~NGramParser(); + +private: + /* + * Binary search for value in table, which must have exactly 64 entries. + */ + int32_t search(const int32_t *table, int32_t value); + + void lookup(int32_t thisNgram); + + virtual int32_t nextByte(InputText *det); + virtual void parseCharacters(InputText *det); + +public: + int32_t parse(InputText *det); + +}; + +#if !UCONFIG_ONLY_HTML_CONVERSION +class NGramParser_IBM420 : public NGramParser +{ +public: + NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap); + ~NGramParser_IBM420(); + +private: + int32_t alef; + int32_t isLamAlef(int32_t b); + int32_t nextByte(InputText *det) override; + void parseCharacters(InputText *det) override; +}; +#endif + + +class CharsetRecog_sbcs : public CharsetRecognizer +{ +public: + CharsetRecog_sbcs(); + virtual ~CharsetRecog_sbcs(); + virtual const char *getName() const override = 0; + virtual UBool match(InputText *det, CharsetMatch *results) const override = 0; + virtual int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const; +}; + +class CharsetRecog_8859_1 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_1(); + const char *getName() const override; + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_2 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_2(); + const char *getName() const override; + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_5 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_5(); + const char *getName() const override; +}; + +class CharsetRecog_8859_6 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_6(); + + const char *getName() const override; +}; + +class CharsetRecog_8859_7 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_7(); + + const char *getName() const override; +}; + +class CharsetRecog_8859_8 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_8(); + + virtual const char *getName() const override; +}; + +class CharsetRecog_8859_9 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_8859_9(); + + const char *getName() const override; +}; + + + +class CharsetRecog_8859_5_ru : public CharsetRecog_8859_5 +{ +public: + virtual ~CharsetRecog_8859_5_ru(); + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_6_ar : public CharsetRecog_8859_6 +{ +public: + virtual ~CharsetRecog_8859_6_ar(); + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_7_el : public CharsetRecog_8859_7 +{ +public: + virtual ~CharsetRecog_8859_7_el(); + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_8_I_he : public CharsetRecog_8859_8 +{ +public: + virtual ~CharsetRecog_8859_8_I_he(); + + const char *getName() const override; + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_8_he : public CharsetRecog_8859_8 +{ +public: + virtual ~CharsetRecog_8859_8_he (); + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_8859_9_tr : public CharsetRecog_8859_9 +{ +public: + virtual ~CharsetRecog_8859_9_tr (); + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_windows_1256 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_windows_1256(); + + const char *getName() const override; + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_windows_1251 : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_windows_1251(); + + const char *getName() const override; + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + + +class CharsetRecog_KOI8_R : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_KOI8_R(); + + const char *getName() const override; + + const char *getLanguage() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +#if !UCONFIG_ONLY_HTML_CONVERSION +class CharsetRecog_IBM424_he : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_IBM424_he(); + + const char *getLanguage() const override; +}; + +class CharsetRecog_IBM424_he_rtl : public CharsetRecog_IBM424_he { +public: + virtual ~CharsetRecog_IBM424_he_rtl(); + + const char *getName() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_IBM424_he_ltr : public CharsetRecog_IBM424_he { + virtual ~CharsetRecog_IBM424_he_ltr(); + + const char *getName() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_IBM420_ar : public CharsetRecog_sbcs +{ +public: + virtual ~CharsetRecog_IBM420_ar(); + + const char *getLanguage() const override; + int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const override; + +}; + +class CharsetRecog_IBM420_ar_rtl : public CharsetRecog_IBM420_ar { +public: + virtual ~CharsetRecog_IBM420_ar_rtl(); + + const char *getName() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; + +class CharsetRecog_IBM420_ar_ltr : public CharsetRecog_IBM420_ar { + virtual ~CharsetRecog_IBM420_ar_ltr(); + + const char *getName() const override; + + virtual UBool match(InputText *det, CharsetMatch *results) const override; +}; +#endif + +U_NAMESPACE_END + +#endif /* !UCONFIG_NO_CONVERSION */ +#endif /* __CSRSBCS_H */ diff --git a/intl/icu/source/i18n/csrucode.cpp b/intl/icu/source/i18n/csrucode.cpp new file mode 100644 index 0000000000..a6e6f83f0f --- /dev/null +++ b/intl/icu/source/i18n/csrucode.cpp @@ -0,0 +1,200 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2013, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrucode.h" +#include "csmatch.h" + +U_NAMESPACE_BEGIN + +CharsetRecog_Unicode::~CharsetRecog_Unicode() +{ + // nothing to do +} + +CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_16_BE::getName() const +{ + return "UTF-16BE"; +} + +// UTF-16 confidence calculation. Very simple minded, but better than nothing. +// Any 8 bit non-control characters bump the confidence up. These have a zero high byte, +// and are very likely to be UTF-16, although they could also be part of a UTF-32 code. +// NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32. +// NULs should be rare in actual text. + +static int32_t adjustConfidence(char16_t codeUnit, int32_t confidence) { + if (codeUnit == 0) { + confidence -= 10; + } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) { + confidence += 10; + } + if (confidence < 0) { + confidence = 0; + } else if (confidence > 100) { + confidence = 100; + } + return confidence; +} + + +UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t confidence = 10; + int32_t length = textIn->fRawLength; + + int32_t bytesToCheck = (length > 30) ? 30 : length; + for (int32_t charIndex=0; charIndexset(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_16_LE::getName() const +{ + return "UTF-16LE"; +} + +UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t confidence = 10; + int32_t length = textIn->fRawLength; + + int32_t bytesToCheck = (length > 30) ? 30 : length; + for (int32_t charIndex=0; charIndex= 4 && input[2] == 0 && input[3] == 0) { + confidence = 0; // UTF-32 BOM + } + break; + } + confidence = adjustConfidence(codeUnit, confidence); + if (confidence == 0 || confidence == 100) { + break; + } + } + if (bytesToCheck < 4 && confidence < 100) { + confidence = 0; + } + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_32::~CharsetRecog_UTF_32() +{ + // nothing to do +} + +UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t limit = (textIn->fRawLength / 4) * 4; + int32_t numValid = 0; + int32_t numInvalid = 0; + bool hasBOM = false; + int32_t confidence = 0; + + if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) { + hasBOM = true; + } + + for(int32_t i = 0; i < limit; i += 4) { + int32_t ch = getChar(input, i); + + if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) { + numInvalid += 1; + } else { + numValid += 1; + } + } + + + // Cook up some sort of confidence score, based on presence of a BOM + // and the existence of valid and/or invalid multi-byte sequences. + if (hasBOM && numInvalid==0) { + confidence = 100; + } else if (hasBOM && numValid > numInvalid*10) { + confidence = 80; + } else if (numValid > 3 && numInvalid == 0) { + confidence = 100; + } else if (numValid > 0 && numInvalid == 0) { + confidence = 80; + } else if (numValid > numInvalid*10) { + // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance. + confidence = 25; + } + + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_32_BE::getName() const +{ + return "UTF-32BE"; +} + +int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const +{ + return input[index + 0] << 24 | input[index + 1] << 16 | + input[index + 2] << 8 | input[index + 3]; +} + +CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_32_LE::getName() const +{ + return "UTF-32LE"; +} + +int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const +{ + return input[index + 3] << 24 | input[index + 2] << 16 | + input[index + 1] << 8 | input[index + 0]; +} + +U_NAMESPACE_END +#endif + diff --git a/intl/icu/source/i18n/csrucode.h b/intl/icu/source/i18n/csrucode.h new file mode 100644 index 0000000000..78e08d22f1 --- /dev/null +++ b/intl/icu/source/i18n/csrucode.h @@ -0,0 +1,108 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSRUCODE_H +#define __CSRUCODE_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +/** + * This class matches UTF-16 and UTF-32, both big- and little-endian. The + * BOM will be used if it is present. + * + * @internal + */ +class CharsetRecog_Unicode : public CharsetRecognizer +{ + +public: + + virtual ~CharsetRecog_Unicode(); + /* (non-Javadoc) + * @see com.ibm.icu.text.CharsetRecognizer#getName() + */ + const char* getName() const override = 0; + + /* (non-Javadoc) + * @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector) + */ + UBool match(InputText* textIn, CharsetMatch *results) const override = 0; +}; + + +class CharsetRecog_UTF_16_BE : public CharsetRecog_Unicode +{ +public: + + virtual ~CharsetRecog_UTF_16_BE(); + + const char *getName() const override; + + UBool match(InputText* textIn, CharsetMatch *results) const override; +}; + +class CharsetRecog_UTF_16_LE : public CharsetRecog_Unicode +{ +public: + + virtual ~CharsetRecog_UTF_16_LE(); + + const char *getName() const override; + + UBool match(InputText* textIn, CharsetMatch *results) const override; +}; + +class CharsetRecog_UTF_32 : public CharsetRecog_Unicode +{ +protected: + virtual int32_t getChar(const uint8_t *input, int32_t index) const = 0; +public: + + virtual ~CharsetRecog_UTF_32(); + + const char* getName() const override = 0; + + UBool match(InputText* textIn, CharsetMatch *results) const override; +}; + + +class CharsetRecog_UTF_32_BE : public CharsetRecog_UTF_32 +{ +protected: + int32_t getChar(const uint8_t *input, int32_t index) const override; + +public: + + virtual ~CharsetRecog_UTF_32_BE(); + + const char *getName() const override; +}; + + +class CharsetRecog_UTF_32_LE : public CharsetRecog_UTF_32 +{ +protected: + int32_t getChar(const uint8_t *input, int32_t index) const override; + +public: + virtual ~CharsetRecog_UTF_32_LE(); + + const char* getName() const override; +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSRUCODE_H */ diff --git a/intl/icu/source/i18n/csrutf8.cpp b/intl/icu/source/i18n/csrutf8.cpp new file mode 100644 index 0000000000..f114f09722 --- /dev/null +++ b/intl/icu/source/i18n/csrutf8.cpp @@ -0,0 +1,111 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2014, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrutf8.h" +#include "csmatch.h" + +U_NAMESPACE_BEGIN + +CharsetRecog_UTF8::~CharsetRecog_UTF8() +{ + // nothing to do +} + +const char *CharsetRecog_UTF8::getName() const +{ + return "UTF-8"; +} + +UBool CharsetRecog_UTF8::match(InputText* input, CharsetMatch *results) const { + bool hasBOM = false; + int32_t numValid = 0; + int32_t numInvalid = 0; + const uint8_t *inputBytes = input->fRawInput; + int32_t i; + int32_t trailBytes = 0; + int32_t confidence; + + if (input->fRawLength >= 3 && + inputBytes[0] == 0xEF && inputBytes[1] == 0xBB && inputBytes[2] == 0xBF) { + hasBOM = true; + } + + // Scan for multi-byte sequences + for (i=0; i < input->fRawLength; i += 1) { + int32_t b = inputBytes[i]; + + if ((b & 0x80) == 0) { + continue; // ASCII + } + + // Hi bit on char found. Figure out how long the sequence should be + if ((b & 0x0E0) == 0x0C0) { + trailBytes = 1; + } else if ((b & 0x0F0) == 0x0E0) { + trailBytes = 2; + } else if ((b & 0x0F8) == 0xF0) { + trailBytes = 3; + } else { + numInvalid += 1; + continue; + } + + // Verify that we've got the right number of trail bytes in the sequence + for (;;) { + i += 1; + + if (i >= input->fRawLength) { + break; + } + + b = inputBytes[i]; + + if ((b & 0xC0) != 0x080) { + numInvalid += 1; + break; + } + + if (--trailBytes == 0) { + numValid += 1; + break; + } + } + + } + + // Cook up some sort of confidence score, based on presence of a BOM + // and the existence of valid and/or invalid multi-byte sequences. + confidence = 0; + if (hasBOM && numInvalid == 0) { + confidence = 100; + } else if (hasBOM && numValid > numInvalid*10) { + confidence = 80; + } else if (numValid > 3 && numInvalid == 0) { + confidence = 100; + } else if (numValid > 0 && numInvalid == 0) { + confidence = 80; + } else if (numValid == 0 && numInvalid == 0) { + // Plain ASCII. Confidence must be > 10, it's more likely than UTF-16, which + // accepts ASCII with confidence = 10. + confidence = 15; + } else if (numValid > numInvalid*10) { + // Probably corrupt utf-8 data. Valid sequences aren't likely by chance. + confidence = 25; + } + + results->set(input, this, confidence); + return (confidence > 0); +} + +U_NAMESPACE_END +#endif diff --git a/intl/icu/source/i18n/csrutf8.h b/intl/icu/source/i18n/csrutf8.h new file mode 100644 index 0000000000..bcfb38ac95 --- /dev/null +++ b/intl/icu/source/i18n/csrutf8.h @@ -0,0 +1,44 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#ifndef __CSRUTF8_H +#define __CSRUTF8_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrecog.h" + +U_NAMESPACE_BEGIN + +/** + * Charset recognizer for UTF-8 + * + * @internal + */ +class CharsetRecog_UTF8: public CharsetRecognizer { + + public: + + virtual ~CharsetRecog_UTF8(); + + const char *getName() const override; + + /* (non-Javadoc) + * @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector) + */ + UBool match(InputText *input, CharsetMatch *results) const override; + +}; + +U_NAMESPACE_END + +#endif +#endif /* __CSRUTF8_H */ diff --git a/intl/icu/source/i18n/curramt.cpp b/intl/icu/source/i18n/curramt.cpp new file mode 100644 index 0000000000..43c3e9c151 --- /dev/null +++ b/intl/icu/source/i18n/curramt.cpp @@ -0,0 +1,56 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 26, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/curramt.h" +#include "unicode/currunit.h" + +U_NAMESPACE_BEGIN + +CurrencyAmount::CurrencyAmount(const Formattable& amount, ConstChar16Ptr isoCode, + UErrorCode& ec) : + Measure(amount, new CurrencyUnit(isoCode, ec), ec) { +} + +CurrencyAmount::CurrencyAmount(double amount, ConstChar16Ptr isoCode, + UErrorCode& ec) : + Measure(Formattable(amount), new CurrencyUnit(isoCode, ec), ec) { +} + +CurrencyAmount::CurrencyAmount(const CurrencyAmount& other) : + Measure(other) { +} + +CurrencyAmount& CurrencyAmount::operator=(const CurrencyAmount& other) { + Measure::operator=(other); + return *this; +} + +CurrencyAmount* CurrencyAmount::clone() const { + return new CurrencyAmount(*this); +} + +CurrencyAmount::~CurrencyAmount() { +} + +const CurrencyUnit& CurrencyAmount::getCurrency() const { + return static_cast(getUnit()); +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyAmount) + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/currfmt.cpp b/intl/icu/source/i18n/currfmt.cpp new file mode 100644 index 0000000000..b7dfc09c54 --- /dev/null +++ b/intl/icu/source/i18n/currfmt.cpp @@ -0,0 +1,62 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2014 International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 20, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "currfmt.h" +#include "unicode/numfmt.h" +#include "unicode/curramt.h" + +U_NAMESPACE_BEGIN + +CurrencyFormat::CurrencyFormat(const Locale& locale, UErrorCode& ec) : + MeasureFormat(locale, UMEASFMT_WIDTH_WIDE, ec) +{ +} + +CurrencyFormat::CurrencyFormat(const CurrencyFormat& other) : + MeasureFormat(other) +{ +} + +CurrencyFormat::~CurrencyFormat() { +} + +CurrencyFormat* CurrencyFormat::clone() const { + return new CurrencyFormat(*this); +} + +UnicodeString& CurrencyFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& ec) const +{ + return getCurrencyFormatInternal().format(obj, appendTo, pos, ec); +} + +void CurrencyFormat::parseObject(const UnicodeString& source, + Formattable& result, + ParsePosition& pos) const +{ + CurrencyAmount* currAmt = getCurrencyFormatInternal().parseCurrency(source, pos); + if (currAmt != nullptr) { + result.adoptObject(currAmt); + } +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyFormat) + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/currfmt.h b/intl/icu/source/i18n/currfmt.h new file mode 100644 index 0000000000..2a75cae2bb --- /dev/null +++ b/intl/icu/source/i18n/currfmt.h @@ -0,0 +1,94 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 20, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#ifndef CURRENCYFORMAT_H +#define CURRENCYFORMAT_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/measfmt.h" + +U_NAMESPACE_BEGIN + +class NumberFormat; + +/** + * Temporary internal concrete subclass of MeasureFormat implementing + * parsing and formatting of currency amount objects. This class is + * likely to be redesigned and rewritten in the near future. + * + *

This class currently delegates to DecimalFormat for parsing and + * formatting. + * + * @see MeasureFormat + * @author Alan Liu + * @internal + */ +class CurrencyFormat : public MeasureFormat { + + public: + + /** + * Construct a CurrencyFormat for the given locale. + */ + CurrencyFormat(const Locale& locale, UErrorCode& ec); + + /** + * Copy constructor. + */ + CurrencyFormat(const CurrencyFormat& other); + + /** + * Destructor. + */ + virtual ~CurrencyFormat(); + + /** + * Override Format API. + */ + virtual CurrencyFormat* clone() const override; + + + using MeasureFormat::format; + + /** + * Override Format API. + */ + virtual UnicodeString& format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& ec) const override; + + /** + * Override Format API. + */ + virtual void parseObject(const UnicodeString& source, + Formattable& result, + ParsePosition& pos) const override; + + /** + * Override Format API. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Returns the class ID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); +}; + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_FORMATTING +#endif // #ifndef CURRENCYFORMAT_H diff --git a/intl/icu/source/i18n/currpinf.cpp b/intl/icu/source/i18n/currpinf.cpp new file mode 100644 index 0000000000..7c5adaaf7c --- /dev/null +++ b/intl/icu/source/i18n/currpinf.cpp @@ -0,0 +1,441 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ******************************************************************************* + * Copyright (C) 2009-2014, International Business Machines Corporation and + * others. All Rights Reserved. + ******************************************************************************* + */ + +#include "unicode/currpinf.h" + +#if !UCONFIG_NO_FORMATTING + +//#define CURRENCY_PLURAL_INFO_DEBUG 1 + +#ifdef CURRENCY_PLURAL_INFO_DEBUG +#include +#endif + +#include "unicode/locid.h" +#include "unicode/plurrule.h" +#include "unicode/strenum.h" +#include "unicode/ures.h" +#include "unicode/numsys.h" +#include "cstring.h" +#include "hash.h" +#include "uresimp.h" +#include "ureslocs.h" + +U_NAMESPACE_BEGIN + +static const char16_t gNumberPatternSeparator = 0x3B; // ; + +U_CDECL_BEGIN + +/** + * @internal ICU 4.2 + */ +static UBool U_CALLCONV ValueComparator(UHashTok val1, UHashTok val2); + +UBool +U_CALLCONV ValueComparator(UHashTok val1, UHashTok val2) { + const UnicodeString* affix_1 = (UnicodeString*)val1.pointer; + const UnicodeString* affix_2 = (UnicodeString*)val2.pointer; + return *affix_1 == *affix_2; +} + +U_CDECL_END + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyPluralInfo) + +static const char16_t gDefaultCurrencyPluralPattern[] = {'0', '.', '#', '#', ' ', 0xA4, 0xA4, 0xA4, 0}; +static const char16_t gTripleCurrencySign[] = {0xA4, 0xA4, 0xA4, 0}; +static const char16_t gPluralCountOther[] = {0x6F, 0x74, 0x68, 0x65, 0x72, 0}; +static const char16_t gPart0[] = {0x7B, 0x30, 0x7D, 0}; +static const char16_t gPart1[] = {0x7B, 0x31, 0x7D, 0}; + +static const char gNumberElementsTag[]="NumberElements"; +static const char gLatnTag[]="latn"; +static const char gPatternsTag[]="patterns"; +static const char gDecimalFormatTag[]="decimalFormat"; +static const char gCurrUnitPtnTag[]="CurrencyUnitPatterns"; + +CurrencyPluralInfo::CurrencyPluralInfo(UErrorCode& status) +: fPluralCountToCurrencyUnitPattern(nullptr), + fPluralRules(nullptr), + fLocale(nullptr), + fInternalStatus(U_ZERO_ERROR) { + initialize(Locale::getDefault(), status); +} + +CurrencyPluralInfo::CurrencyPluralInfo(const Locale& locale, UErrorCode& status) +: fPluralCountToCurrencyUnitPattern(nullptr), + fPluralRules(nullptr), + fLocale(nullptr), + fInternalStatus(U_ZERO_ERROR) { + initialize(locale, status); +} + +CurrencyPluralInfo::CurrencyPluralInfo(const CurrencyPluralInfo& info) +: UObject(info), + fPluralCountToCurrencyUnitPattern(nullptr), + fPluralRules(nullptr), + fLocale(nullptr), + fInternalStatus(U_ZERO_ERROR) { + *this = info; +} + +CurrencyPluralInfo& +CurrencyPluralInfo::operator=(const CurrencyPluralInfo& info) { + if (this == &info) { + return *this; + } + + fInternalStatus = info.fInternalStatus; + if (U_FAILURE(fInternalStatus)) { + // bail out early if the object we were copying from was already 'invalid'. + return *this; + } + + deleteHash(fPluralCountToCurrencyUnitPattern); + fPluralCountToCurrencyUnitPattern = initHash(fInternalStatus); + copyHash(info.fPluralCountToCurrencyUnitPattern, + fPluralCountToCurrencyUnitPattern, fInternalStatus); + if ( U_FAILURE(fInternalStatus) ) { + return *this; + } + + delete fPluralRules; + fPluralRules = nullptr; + delete fLocale; + fLocale = nullptr; + + if (info.fPluralRules != nullptr) { + fPluralRules = info.fPluralRules->clone(); + if (fPluralRules == nullptr) { + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + } + if (info.fLocale != nullptr) { + fLocale = info.fLocale->clone(); + if (fLocale == nullptr) { + // Note: If clone had an error parameter, then we could check/set that instead. + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + // If the other locale wasn't bogus, but our clone'd locale is bogus, then OOM happened + // during the call to clone(). + if (!info.fLocale->isBogus() && fLocale->isBogus()) { + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + } + return *this; +} + +CurrencyPluralInfo::~CurrencyPluralInfo() { + deleteHash(fPluralCountToCurrencyUnitPattern); + fPluralCountToCurrencyUnitPattern = nullptr; + delete fPluralRules; + delete fLocale; + fPluralRules = nullptr; + fLocale = nullptr; +} + +bool +CurrencyPluralInfo::operator==(const CurrencyPluralInfo& info) const { +#ifdef CURRENCY_PLURAL_INFO_DEBUG + if (*fPluralRules == *info.fPluralRules) { + std::cout << "same plural rules\n"; + } + if (*fLocale == *info.fLocale) { + std::cout << "same locale\n"; + } + if (fPluralCountToCurrencyUnitPattern->equals(*info.fPluralCountToCurrencyUnitPattern)) { + std::cout << "same pattern\n"; + } +#endif + return *fPluralRules == *info.fPluralRules && + *fLocale == *info.fLocale && + fPluralCountToCurrencyUnitPattern->equals(*info.fPluralCountToCurrencyUnitPattern); +} + + +CurrencyPluralInfo* +CurrencyPluralInfo::clone() const { + CurrencyPluralInfo* newObj = new CurrencyPluralInfo(*this); + // Since clone doesn't have a 'status' parameter, the best we can do is return nullptr + // if the new object was not full constructed properly (an error occurred). + if (newObj != nullptr && U_FAILURE(newObj->fInternalStatus)) { + delete newObj; + newObj = nullptr; + } + return newObj; +} + +const PluralRules* +CurrencyPluralInfo::getPluralRules() const { + return fPluralRules; +} + +UnicodeString& +CurrencyPluralInfo::getCurrencyPluralPattern(const UnicodeString& pluralCount, + UnicodeString& result) const { + const UnicodeString* currencyPluralPattern = + (UnicodeString*)fPluralCountToCurrencyUnitPattern->get(pluralCount); + if (currencyPluralPattern == nullptr) { + // fall back to "other" + if (pluralCount.compare(gPluralCountOther, 5)) { + currencyPluralPattern = + (UnicodeString*)fPluralCountToCurrencyUnitPattern->get(UnicodeString(true, gPluralCountOther, 5)); + } + if (currencyPluralPattern == nullptr) { + // no currencyUnitPatterns defined, + // fallback to predefined default. + // This should never happen when ICU resource files are + // available, since currencyUnitPattern of "other" is always + // defined in root. + result = UnicodeString(gDefaultCurrencyPluralPattern); + return result; + } + } + result = *currencyPluralPattern; + return result; +} + +const Locale& +CurrencyPluralInfo::getLocale() const { + return *fLocale; +} + +void +CurrencyPluralInfo::setPluralRules(const UnicodeString& ruleDescription, + UErrorCode& status) { + if (U_SUCCESS(status)) { + delete fPluralRules; + fPluralRules = PluralRules::createRules(ruleDescription, status); + } +} + +void +CurrencyPluralInfo::setCurrencyPluralPattern(const UnicodeString& pluralCount, + const UnicodeString& pattern, + UErrorCode& status) { + if (U_SUCCESS(status)) { + UnicodeString* oldValue = static_cast( + fPluralCountToCurrencyUnitPattern->get(pluralCount)); + delete oldValue; + LocalPointer p(new UnicodeString(pattern), status); + if (U_SUCCESS(status)) { + // the p object allocated above will be owned by fPluralCountToCurrencyUnitPattern + // after the call to put(), even if the method returns failure. + fPluralCountToCurrencyUnitPattern->put(pluralCount, p.orphan(), status); + } + } +} + +void +CurrencyPluralInfo::setLocale(const Locale& loc, UErrorCode& status) { + initialize(loc, status); +} + +void +CurrencyPluralInfo::initialize(const Locale& loc, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + delete fLocale; + fLocale = nullptr; + delete fPluralRules; + fPluralRules = nullptr; + + fLocale = loc.clone(); + if (fLocale == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + // If the locale passed in wasn't bogus, but our clone'd locale is bogus, then OOM happened + // during the call to loc.clone(). + if (!loc.isBogus() && fLocale->isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + fPluralRules = PluralRules::forLocale(loc, status); + setupCurrencyPluralPattern(loc, status); +} + +void +CurrencyPluralInfo::setupCurrencyPluralPattern(const Locale& loc, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + + deleteHash(fPluralCountToCurrencyUnitPattern); + fPluralCountToCurrencyUnitPattern = initHash(status); + if (U_FAILURE(status)) { + return; + } + + LocalPointer ns(NumberingSystem::createInstance(loc, status), status); + if (U_FAILURE(status)) { + return; + } + UErrorCode ec = U_ZERO_ERROR; + LocalUResourceBundlePointer rb(ures_open(nullptr, loc.getName(), &ec)); + LocalUResourceBundlePointer numElements(ures_getByKeyWithFallback(rb.getAlias(), gNumberElementsTag, nullptr, &ec)); + ures_getByKeyWithFallback(numElements.getAlias(), ns->getName(), rb.getAlias(), &ec); + ures_getByKeyWithFallback(rb.getAlias(), gPatternsTag, rb.getAlias(), &ec); + int32_t ptnLen; + const char16_t* numberStylePattern = ures_getStringByKeyWithFallback(rb.getAlias(), gDecimalFormatTag, &ptnLen, &ec); + // Fall back to "latn" if num sys specific pattern isn't there. + if ( ec == U_MISSING_RESOURCE_ERROR && (uprv_strcmp(ns->getName(), gLatnTag) != 0)) { + ec = U_ZERO_ERROR; + ures_getByKeyWithFallback(numElements.getAlias(), gLatnTag, rb.getAlias(), &ec); + ures_getByKeyWithFallback(rb.getAlias(), gPatternsTag, rb.getAlias(), &ec); + numberStylePattern = ures_getStringByKeyWithFallback(rb.getAlias(), gDecimalFormatTag, &ptnLen, &ec); + } + int32_t numberStylePatternLen = ptnLen; + const char16_t* negNumberStylePattern = nullptr; + int32_t negNumberStylePatternLen = 0; + // TODO: Java + // parse to check whether there is ";" separator in the numberStylePattern + UBool hasSeparator = false; + if (U_SUCCESS(ec)) { + for (int32_t styleCharIndex = 0; styleCharIndex < ptnLen; ++styleCharIndex) { + if (numberStylePattern[styleCharIndex] == gNumberPatternSeparator) { + hasSeparator = true; + // split the number style pattern into positive and negative + negNumberStylePattern = numberStylePattern + styleCharIndex + 1; + negNumberStylePatternLen = ptnLen - styleCharIndex - 1; + numberStylePatternLen = styleCharIndex; + } + } + } + + if (U_FAILURE(ec)) { + // If OOM occurred during the above code, then we want to report that back to the caller. + if (ec == U_MEMORY_ALLOCATION_ERROR) { + status = ec; + } + return; + } + + LocalUResourceBundlePointer currRb(ures_open(U_ICUDATA_CURR, loc.getName(), &ec)); + LocalUResourceBundlePointer currencyRes(ures_getByKeyWithFallback(currRb.getAlias(), gCurrUnitPtnTag, nullptr, &ec)); + +#ifdef CURRENCY_PLURAL_INFO_DEBUG + std::cout << "in set up\n"; +#endif + LocalPointer keywords(fPluralRules->getKeywords(ec), ec); + if (U_SUCCESS(ec)) { + const char* pluralCount; + while (((pluralCount = keywords->next(nullptr, ec)) != nullptr) && U_SUCCESS(ec)) { + int32_t ptnLength; + UErrorCode err = U_ZERO_ERROR; + const char16_t* patternChars = ures_getStringByKeyWithFallback(currencyRes.getAlias(), pluralCount, &ptnLength, &err); + if (err == U_MEMORY_ALLOCATION_ERROR || patternChars == nullptr) { + ec = err; + break; + } + if (U_SUCCESS(err) && ptnLength > 0) { + UnicodeString* pattern = new UnicodeString(patternChars, ptnLength); + if (pattern == nullptr) { + ec = U_MEMORY_ALLOCATION_ERROR; + break; + } +#ifdef CURRENCY_PLURAL_INFO_DEBUG + char result_1[1000]; + pattern->extract(0, pattern->length(), result_1, "UTF-8"); + std::cout << "pluralCount: " << pluralCount << "; pattern: " << result_1 << "\n"; +#endif + pattern->findAndReplace(UnicodeString(true, gPart0, 3), + UnicodeString(numberStylePattern, numberStylePatternLen)); + pattern->findAndReplace(UnicodeString(true, gPart1, 3), UnicodeString(true, gTripleCurrencySign, 3)); + + if (hasSeparator) { + UnicodeString negPattern(patternChars, ptnLength); + negPattern.findAndReplace(UnicodeString(true, gPart0, 3), + UnicodeString(negNumberStylePattern, negNumberStylePatternLen)); + negPattern.findAndReplace(UnicodeString(true, gPart1, 3), UnicodeString(true, gTripleCurrencySign, 3)); + pattern->append(gNumberPatternSeparator); + pattern->append(negPattern); + } +#ifdef CURRENCY_PLURAL_INFO_DEBUG + pattern->extract(0, pattern->length(), result_1, "UTF-8"); + std::cout << "pluralCount: " << pluralCount << "; pattern: " << result_1 << "\n"; +#endif + // the 'pattern' object allocated above will be owned by the fPluralCountToCurrencyUnitPattern after the call to + // put(), even if the method returns failure. + fPluralCountToCurrencyUnitPattern->put(UnicodeString(pluralCount, -1, US_INV), pattern, status); + } + } + } + // If OOM occurred during the above code, then we want to report that back to the caller. + if (ec == U_MEMORY_ALLOCATION_ERROR) { + status = ec; + } +} + +void +CurrencyPluralInfo::deleteHash(Hashtable* hTable) { + if ( hTable == nullptr ) { + return; + } + int32_t pos = UHASH_FIRST; + const UHashElement* element = nullptr; + while ( (element = hTable->nextElement(pos)) != nullptr ) { + const UHashTok valueTok = element->value; + const UnicodeString* value = (UnicodeString*)valueTok.pointer; + delete value; + } + delete hTable; + hTable = nullptr; +} + +Hashtable* +CurrencyPluralInfo::initHash(UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer hTable(new Hashtable(true, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + hTable->setValueComparator(ValueComparator); + return hTable.orphan(); +} + +void +CurrencyPluralInfo::copyHash(const Hashtable* source, + Hashtable* target, + UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + int32_t pos = UHASH_FIRST; + const UHashElement* element = nullptr; + if (source) { + while ( (element = source->nextElement(pos)) != nullptr ) { + const UHashTok keyTok = element->key; + const UnicodeString* key = (UnicodeString*)keyTok.pointer; + const UHashTok valueTok = element->value; + const UnicodeString* value = (UnicodeString*)valueTok.pointer; + LocalPointer copy(new UnicodeString(*value), status); + if (U_FAILURE(status)) { + return; + } + // The HashTable owns the 'copy' object after the call to put(). + target->put(UnicodeString(*key), copy.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } + } +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/currunit.cpp b/intl/icu/source/i18n/currunit.cpp new file mode 100644 index 0000000000..98f28365cf --- /dev/null +++ b/intl/icu/source/i18n/currunit.cpp @@ -0,0 +1,126 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 26, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/currunit.h" +#include "unicode/ustring.h" +#include "unicode/uchar.h" +#include "cstring.h" +#include "uinvchar.h" +#include "charstr.h" +#include "ustr_imp.h" +#include "measunit_impl.h" + +U_NAMESPACE_BEGIN + +CurrencyUnit::CurrencyUnit(ConstChar16Ptr _isoCode, UErrorCode& ec) { + // The constructor always leaves the CurrencyUnit in a valid state (with a 3-character currency code). + // Note: in ICU4J Currency.getInstance(), we check string length for 3, but in ICU4C we allow a + // non-NUL-terminated string to be passed as an argument, so it is not possible to check length. + // However, we allow a NUL-terminated empty string, which should have the same behavior as nullptr. + // Consider NUL-terminated strings of length 1 or 2 as invalid. + bool useDefault = false; + if (U_FAILURE(ec) || _isoCode == nullptr || _isoCode[0] == 0) { + useDefault = true; + } else if (_isoCode[1] == 0 || _isoCode[2] == 0) { + useDefault = true; + ec = U_ILLEGAL_ARGUMENT_ERROR; + } else if (!uprv_isInvariantUString(_isoCode, 3)) { + // TODO: Perform a more strict ASCII check like in ICU4J isAlpha3Code? + useDefault = true; + ec = U_INVARIANT_CONVERSION_ERROR; + } else { + for (int32_t i=0; i<3; i++) { + isoCode[i] = u_asciiToUpper(_isoCode[i]); + } + isoCode[3] = 0; + } + if (useDefault) { + uprv_memcpy(isoCode, kDefaultCurrency, sizeof(char16_t) * 4); + } + char simpleIsoCode[4]; + u_UCharsToChars(isoCode, simpleIsoCode, 4); + initCurrency(simpleIsoCode); +} + +CurrencyUnit::CurrencyUnit(StringPiece _isoCode, UErrorCode& ec) { + // Note: unlike the old constructor, reject empty arguments with an error. + char isoCodeBuffer[4]; + const char* isoCodeToUse; + // uprv_memchr checks that the string contains no internal NULs + if (_isoCode.length() != 3 || uprv_memchr(_isoCode.data(), 0, 3) != nullptr) { + isoCodeToUse = kDefaultCurrency8; + ec = U_ILLEGAL_ARGUMENT_ERROR; + } else if (!uprv_isInvariantString(_isoCode.data(), 3)) { + // TODO: Perform a more strict ASCII check like in ICU4J isAlpha3Code? + isoCodeToUse = kDefaultCurrency8; + ec = U_INVARIANT_CONVERSION_ERROR; + } else { + // Have to use isoCodeBuffer to ensure the string is NUL-terminated + for (int32_t i=0; i<3; i++) { + isoCodeBuffer[i] = uprv_toupper(_isoCode.data()[i]); + } + isoCodeBuffer[3] = 0; + isoCodeToUse = isoCodeBuffer; + } + u_charsToUChars(isoCodeToUse, isoCode, 4); + initCurrency(isoCodeToUse); +} + +CurrencyUnit::CurrencyUnit(const CurrencyUnit& other) : MeasureUnit(other) { + u_strcpy(isoCode, other.isoCode); +} + +CurrencyUnit::CurrencyUnit(const MeasureUnit& other, UErrorCode& ec) : MeasureUnit(other) { + // Make sure this is a currency. + // OK to hard-code the string because we are comparing against another hard-coded string. + if (uprv_strcmp("currency", getType()) != 0) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + isoCode[0] = 0; + } else { + // Get the ISO Code from the subtype field. + u_charsToUChars(getSubtype(), isoCode, 4); + isoCode[3] = 0; // make 100% sure it is NUL-terminated + } +} + +CurrencyUnit::CurrencyUnit() : MeasureUnit() { + u_strcpy(isoCode, kDefaultCurrency); + char simpleIsoCode[4]; + u_UCharsToChars(isoCode, simpleIsoCode, 4); + initCurrency(simpleIsoCode); +} + +CurrencyUnit& CurrencyUnit::operator=(const CurrencyUnit& other) { + if (this == &other) { + return *this; + } + MeasureUnit::operator=(other); + u_strcpy(isoCode, other.isoCode); + return *this; +} + +CurrencyUnit* CurrencyUnit::clone() const { + return new CurrencyUnit(*this); +} + +CurrencyUnit::~CurrencyUnit() { +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyUnit) + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/dangical.cpp b/intl/icu/source/i18n/dangical.cpp new file mode 100644 index 0000000000..2b340ee4b4 --- /dev/null +++ b/intl/icu/source/i18n/dangical.cpp @@ -0,0 +1,167 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ****************************************************************************** + * Copyright (C) 2013, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File DANGICAL.CPP + ***************************************************************************** + */ + +#include "chnsecal.h" +#include "dangical.h" + +#if !UCONFIG_NO_FORMATTING + +#include "gregoimp.h" // Math +#include "uassert.h" +#include "ucln_in.h" +#include "umutex.h" +#include "unicode/rbtz.h" +#include "unicode/tzrule.h" + +// --- The cache -- +static icu::TimeZone *gDangiCalendarZoneAstroCalc = nullptr; +static icu::UInitOnce gDangiCalendarInitOnce {}; + +/** + * The start year of the Korean traditional calendar (Dan-gi) is the inaugural + * year of Dan-gun (BC 2333). + */ +static const int32_t DANGI_EPOCH_YEAR = -2332; // Gregorian year + +U_CDECL_BEGIN +static UBool calendar_dangi_cleanup() { + if (gDangiCalendarZoneAstroCalc) { + delete gDangiCalendarZoneAstroCalc; + gDangiCalendarZoneAstroCalc = nullptr; + } + gDangiCalendarInitOnce.reset(); + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +// Implementation of the DangiCalendar class + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +DangiCalendar::DangiCalendar(const Locale& aLocale, UErrorCode& success) +: ChineseCalendar(aLocale, DANGI_EPOCH_YEAR, getDangiCalZoneAstroCalc(success), success) +{ +} + +DangiCalendar::DangiCalendar (const DangiCalendar& other) +: ChineseCalendar(other) +{ +} + +DangiCalendar::~DangiCalendar() +{ +} + +DangiCalendar* +DangiCalendar::clone() const +{ + return new DangiCalendar(*this); +} + +const char *DangiCalendar::getType() const { + return "dangi"; +} + +/** + * The time zone used for performing astronomical computations for + * Dangi calendar. In Korea various timezones have been used historically + * (cf. http://www.math.snu.ac.kr/~kye/others/lunar.html): + * + * - 1908/04/01: GMT+8 + * 1908/04/01 - 1911/12/31: GMT+8.5 + * 1912/01/01 - 1954/03/20: GMT+9 + * 1954/03/21 - 1961/08/09: GMT+8.5 + * 1961/08/10 - : GMT+9 + * + * Note that, in 1908-1911, the government did not apply the timezone change + * but used GMT+8. In addition, 1954-1961's timezone change does not affect + * the lunar date calculation. Therefore, the following simpler rule works: + * + * -1911: GMT+8 + * 1912-: GMT+9 + * + * Unfortunately, our astronomer's approximation doesn't agree with the + * references (http://www.math.snu.ac.kr/~kye/others/lunar.html and + * http://astro.kasi.re.kr/Life/ConvertSolarLunarForm.aspx?MenuID=115) + * in 1897/7/30. So the following ad hoc fix is used here: + * + * -1896: GMT+8 + * 1897: GMT+7 + * 1898-1911: GMT+8 + * 1912- : GMT+9 + */ +static void U_CALLCONV initDangiCalZoneAstroCalc(UErrorCode &status) { + U_ASSERT(gDangiCalendarZoneAstroCalc == nullptr); + const UDate millis1897[] = { (UDate)((1897 - 1970) * 365 * kOneDay) }; // some days of error is not a problem here + const UDate millis1898[] = { (UDate)((1898 - 1970) * 365 * kOneDay) }; // some days of error is not a problem here + const UDate millis1912[] = { (UDate)((1912 - 1970) * 365 * kOneDay) }; // this doesn't create an issue for 1911/12/20 + LocalPointer initialTimeZone(new InitialTimeZoneRule( + UnicodeString(u"GMT+8"), 8*kOneHour, 0), status); + + LocalPointer rule1897(new TimeArrayTimeZoneRule( + UnicodeString(u"Korean 1897"), 7*kOneHour, 0, millis1897, 1, DateTimeRule::STANDARD_TIME), status); + + LocalPointer rule1898to1911(new TimeArrayTimeZoneRule( + UnicodeString(u"Korean 1898-1911"), 8*kOneHour, 0, millis1898, 1, DateTimeRule::STANDARD_TIME), status); + + LocalPointer ruleFrom1912(new TimeArrayTimeZoneRule( + UnicodeString(u"Korean 1912-"), 9*kOneHour, 0, millis1912, 1, DateTimeRule::STANDARD_TIME), status); + + LocalPointer dangiCalZoneAstroCalc(new RuleBasedTimeZone( + UnicodeString(u"KOREA_ZONE"), initialTimeZone.orphan()), status); // adopts initialTimeZone + + if (U_FAILURE(status)) { + return; + } + dangiCalZoneAstroCalc->addTransitionRule(rule1897.orphan(), status); // adopts rule1897 + dangiCalZoneAstroCalc->addTransitionRule(rule1898to1911.orphan(), status); + dangiCalZoneAstroCalc->addTransitionRule(ruleFrom1912.orphan(), status); + dangiCalZoneAstroCalc->complete(status); + if (U_SUCCESS(status)) { + gDangiCalendarZoneAstroCalc = dangiCalZoneAstroCalc.orphan(); + } + ucln_i18n_registerCleanup(UCLN_I18N_DANGI_CALENDAR, calendar_dangi_cleanup); +} + +const TimeZone* DangiCalendar::getDangiCalZoneAstroCalc(UErrorCode &status) const { + umtx_initOnce(gDangiCalendarInitOnce, &initDangiCalZoneAstroCalc, status); + return gDangiCalendarZoneAstroCalc; +} + +constexpr uint32_t kDangiRelatedYearDiff = -2333; + +int32_t DangiCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kDangiRelatedYearDiff; +} + +void DangiCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kDangiRelatedYearDiff); +} + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DangiCalendar) + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/dangical.h b/intl/icu/source/i18n/dangical.h new file mode 100644 index 0000000000..3e5b0bb859 --- /dev/null +++ b/intl/icu/source/i18n/dangical.h @@ -0,0 +1,135 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ***************************************************************************** + * Copyright (C) 2013, International Business Machines Corporation + * and others. All Rights Reserved. + ***************************************************************************** + * + * File DANGICAL.H + ***************************************************************************** + */ + +#ifndef DANGICAL_H +#define DANGICAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/timezone.h" +#include "chnsecal.h" + +U_NAMESPACE_BEGIN + +/** + *

DangiCalendar is a concrete subclass of {@link Calendar} + * that implements a traditional Korean lunisolar calendar.

+ * + *

DangiCalendar usually should be instantiated using + * {@link com.ibm.icu.util.Calendar#getInstance(ULocale)} passing in a ULocale + * with the tag "@calendar=dangi".

+ * + * @internal + */ +class DangiCalendar : public ChineseCalendar { + public: + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs a DangiCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of DangiCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + DangiCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + DangiCalendar(const DangiCalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~DangiCalendar(); + + /** + * Clone. + * @internal + */ + virtual DangiCalendar* clone() const override; + + //---------------------------------------------------------------------- + // Internal methods & astronomical calculations + //---------------------------------------------------------------------- + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + private: + + const TimeZone* getDangiCalZoneAstroCalc(UErrorCode &status) const; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "dangi". + * + * @return calendar type + * @internal + */ + const char * getType() const override; + + + private: + + DangiCalendar(); // default constructor not implemented +}; + +U_NAMESPACE_END + +#endif +#endif + + + diff --git a/intl/icu/source/i18n/datefmt.cpp b/intl/icu/source/i18n/datefmt.cpp new file mode 100644 index 0000000000..029634e3dc --- /dev/null +++ b/intl/icu/source/i18n/datefmt.cpp @@ -0,0 +1,751 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ******************************************************************************* + * Copyright (C) 1997-2015, International Business Machines Corporation and * + * others. All Rights Reserved. * + ******************************************************************************* + * + * File DATEFMT.CPP + * + * Modification History: + * + * Date Name Description + * 02/19/97 aliu Converted from java. + * 03/31/97 aliu Modified extensively to work with 50 locales. + * 04/01/97 aliu Added support for centuries. + * 08/12/97 aliu Fixed operator== to use Calendar::equivalentTo. + * 07/20/98 stephen Changed ParsePosition initialization + ******************************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/ures.h" +#include "unicode/datefmt.h" +#include "unicode/smpdtfmt.h" +#include "unicode/dtptngen.h" +#include "unicode/udisplaycontext.h" +#include "reldtfmt.h" +#include "sharedobject.h" +#include "unifiedcache.h" +#include "uarrsort.h" + +#include "cstring.h" +#include "windtfmt.h" + +#if defined( U_DEBUG_CALSVC ) || defined (U_DEBUG_CAL) +#include +#endif + +// ***************************************************************************** +// class DateFormat +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +class DateFmtBestPattern : public SharedObject { +public: + UnicodeString fPattern; + + DateFmtBestPattern(const UnicodeString &pattern) + : fPattern(pattern) { } + ~DateFmtBestPattern(); +}; + +DateFmtBestPattern::~DateFmtBestPattern() { +} + +template<> +const DateFmtBestPattern *LocaleCacheKey::createObject( + const void * /*creationContext*/, UErrorCode &status) const { + status = U_UNSUPPORTED_ERROR; + return nullptr; +} + +class DateFmtBestPatternKey : public LocaleCacheKey { +private: + UnicodeString fSkeleton; +protected: + virtual bool equals(const CacheKeyBase &other) const override { + if (!LocaleCacheKey::equals(other)) { + return false; + } + // We know that this and other are of same class if we get this far. + return operator==(static_cast(other)); + } +public: + DateFmtBestPatternKey( + const Locale &loc, + const UnicodeString &skeleton, + UErrorCode &status) + : LocaleCacheKey(loc), + fSkeleton(DateTimePatternGenerator::staticGetSkeleton(skeleton, status)) { } + DateFmtBestPatternKey(const DateFmtBestPatternKey &other) : + LocaleCacheKey(other), + fSkeleton(other.fSkeleton) { } + virtual ~DateFmtBestPatternKey(); + virtual int32_t hashCode() const override { + return (int32_t)(37u * (uint32_t)LocaleCacheKey::hashCode() + (uint32_t)fSkeleton.hashCode()); + } + inline bool operator==(const DateFmtBestPatternKey &other) const { + return fSkeleton == other.fSkeleton; + } + virtual CacheKeyBase *clone() const override { + return new DateFmtBestPatternKey(*this); + } + virtual const DateFmtBestPattern *createObject( + const void * /*unused*/, UErrorCode &status) const override { + LocalPointer dtpg( + DateTimePatternGenerator::createInstance(fLoc, status)); + if (U_FAILURE(status)) { + return nullptr; + } + + LocalPointer pattern( + new DateFmtBestPattern( + dtpg->getBestPattern(fSkeleton, status)), + status); + if (U_FAILURE(status)) { + return nullptr; + } + DateFmtBestPattern *result = pattern.orphan(); + result->addRef(); + return result; + } +}; + +DateFmtBestPatternKey::~DateFmtBestPatternKey() { } + + +DateFormat::DateFormat() +: fCalendar(0), + fNumberFormat(0), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) +{ +} + +//---------------------------------------------------------------------- + +DateFormat::DateFormat(const DateFormat& other) +: Format(other), + fCalendar(0), + fNumberFormat(0), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) +{ + *this = other; +} + +//---------------------------------------------------------------------- + +DateFormat& DateFormat::operator=(const DateFormat& other) +{ + if (this != &other) + { + delete fCalendar; + delete fNumberFormat; + if(other.fCalendar) { + fCalendar = other.fCalendar->clone(); + } else { + fCalendar = nullptr; + } + if(other.fNumberFormat) { + fNumberFormat = other.fNumberFormat->clone(); + } else { + fNumberFormat = nullptr; + } + fBoolFlags = other.fBoolFlags; + fCapitalizationContext = other.fCapitalizationContext; + } + return *this; +} + +//---------------------------------------------------------------------- + +DateFormat::~DateFormat() +{ + delete fCalendar; + delete fNumberFormat; +} + +//---------------------------------------------------------------------- + +bool +DateFormat::operator==(const Format& other) const +{ + if (this == &other) { + return true; + } + if (!(Format::operator==(other))) { + return false; + } + // Format::operator== guarantees that this cast is safe + DateFormat* fmt = (DateFormat*)&other; + return fCalendar&&(fCalendar->isEquivalentTo(*fmt->fCalendar)) && + (fNumberFormat && *fNumberFormat == *fmt->fNumberFormat) && + (fCapitalizationContext == fmt->fCapitalizationContext); +} + +//---------------------------------------------------------------------- + +UnicodeString& +DateFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& fieldPosition, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return appendTo; + + // if the type of the Formattable is double or long, treat it as if it were a Date + UDate date = 0; + switch (obj.getType()) + { + case Formattable::kDate: + date = obj.getDate(); + break; + case Formattable::kDouble: + date = (UDate)obj.getDouble(); + break; + case Formattable::kLong: + date = (UDate)obj.getLong(); + break; + default: + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; + } + + // Is this right? + //if (fieldPosition.getBeginIndex() == fieldPosition.getEndIndex()) + // status = U_ILLEGAL_ARGUMENT_ERROR; + + return format(date, appendTo, fieldPosition); +} + +//---------------------------------------------------------------------- + +UnicodeString& +DateFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPositionIterator* posIter, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return appendTo; + + // if the type of the Formattable is double or long, treat it as if it were a Date + UDate date = 0; + switch (obj.getType()) + { + case Formattable::kDate: + date = obj.getDate(); + break; + case Formattable::kDouble: + date = (UDate)obj.getDouble(); + break; + case Formattable::kLong: + date = (UDate)obj.getLong(); + break; + default: + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; + } + + // Is this right? + //if (fieldPosition.getBeginIndex() == fieldPosition.getEndIndex()) + // status = U_ILLEGAL_ARGUMENT_ERROR; + + return format(date, appendTo, posIter, status); +} + +//---------------------------------------------------------------------- + +// Default implementation for backwards compatibility, subclasses should implement. +UnicodeString& +DateFormat::format(Calendar& /* unused cal */, + UnicodeString& appendTo, + FieldPositionIterator* /* unused posIter */, + UErrorCode& status) const { + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } + return appendTo; +} + +//---------------------------------------------------------------------- + +UnicodeString& +DateFormat::format(UDate date, UnicodeString& appendTo, FieldPosition& fieldPosition) const { + if (fCalendar != nullptr) { + // Use a clone of our calendar instance + Calendar* calClone = fCalendar->clone(); + if (calClone != nullptr) { + UErrorCode ec = U_ZERO_ERROR; + calClone->setTime(date, ec); + if (U_SUCCESS(ec)) { + format(*calClone, appendTo, fieldPosition); + } + delete calClone; + } + } + return appendTo; +} + +//---------------------------------------------------------------------- + +UnicodeString& +DateFormat::format(UDate date, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const { + if (fCalendar != nullptr) { + Calendar* calClone = fCalendar->clone(); + if (calClone != nullptr) { + calClone->setTime(date, status); + if (U_SUCCESS(status)) { + format(*calClone, appendTo, posIter, status); + } + delete calClone; + } + } + return appendTo; +} + +//---------------------------------------------------------------------- + +UnicodeString& +DateFormat::format(UDate date, UnicodeString& appendTo) const +{ + // Note that any error information is just lost. That's okay + // for this convenience method. + FieldPosition fpos(FieldPosition::DONT_CARE); + return format(date, appendTo, fpos); +} + +//---------------------------------------------------------------------- + +UDate +DateFormat::parse(const UnicodeString& text, + ParsePosition& pos) const +{ + UDate d = 0; // Error return UDate is 0 (the epoch) + if (fCalendar != nullptr) { + Calendar* calClone = fCalendar->clone(); + if (calClone != nullptr) { + int32_t start = pos.getIndex(); + calClone->clear(); + parse(text, *calClone, pos); + if (pos.getIndex() != start) { + UErrorCode ec = U_ZERO_ERROR; + d = calClone->getTime(ec); + if (U_FAILURE(ec)) { + // We arrive here if fCalendar => calClone is non-lenient and + // there is an out-of-range field. We don't know which field + // was illegal so we set the error index to the start. + pos.setIndex(start); + pos.setErrorIndex(start); + d = 0; + } + } + delete calClone; + } + } + return d; +} + +//---------------------------------------------------------------------- + +UDate +DateFormat::parse(const UnicodeString& text, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return 0; + + ParsePosition pos(0); + UDate result = parse(text, pos); + if (pos.getIndex() == 0) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d - - failed to parse - err index %d\n" + , __FILE__, __LINE__, pos.getErrorIndex() ); +#endif + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return result; +} + +//---------------------------------------------------------------------- + +void +DateFormat::parseObject(const UnicodeString& source, + Formattable& result, + ParsePosition& pos) const +{ + result.setDate(parse(source, pos)); +} + +//---------------------------------------------------------------------- + +DateFormat* U_EXPORT2 +DateFormat::createTimeInstance(DateFormat::EStyle style, + const Locale& aLocale) +{ + return createDateTimeInstance(kNone, style, aLocale); +} + +//---------------------------------------------------------------------- + +DateFormat* U_EXPORT2 +DateFormat::createDateInstance(DateFormat::EStyle style, + const Locale& aLocale) +{ + return createDateTimeInstance(style, kNone, aLocale); +} + +//---------------------------------------------------------------------- + +DateFormat* U_EXPORT2 +DateFormat::createDateTimeInstance(EStyle dateStyle, + EStyle timeStyle, + const Locale& aLocale) +{ + if(dateStyle != kNone) + { + dateStyle = (EStyle) (dateStyle + kDateOffset); + } + return create(timeStyle, dateStyle, aLocale); +} + +//---------------------------------------------------------------------- + +DateFormat* U_EXPORT2 +DateFormat::createInstance() +{ + return createDateTimeInstance(kShort, kShort, Locale::getDefault()); +} + +//---------------------------------------------------------------------- + +UnicodeString U_EXPORT2 +DateFormat::getBestPattern( + const Locale &locale, + const UnicodeString &skeleton, + UErrorCode &status) { + UnifiedCache *cache = UnifiedCache::getInstance(status); + if (U_FAILURE(status)) { + return UnicodeString(); + } + DateFmtBestPatternKey key(locale, skeleton, status); + const DateFmtBestPattern *patternPtr = nullptr; + cache->get(key, patternPtr, status); + if (U_FAILURE(status)) { + return UnicodeString(); + } + UnicodeString result(patternPtr->fPattern); + patternPtr->removeRef(); + return result; +} + +DateFormat* U_EXPORT2 +DateFormat::createInstanceForSkeleton( + Calendar *calendarToAdopt, + const UnicodeString& skeleton, + const Locale &locale, + UErrorCode &status) { + LocalPointer calendar(calendarToAdopt); + if (U_FAILURE(status)) { + return nullptr; + } + if (calendar.isNull()) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + Locale localeWithCalendar = locale; + localeWithCalendar.setKeywordValue("calendar", calendar->getType(), status); + if (U_FAILURE(status)) { + return nullptr; + } + DateFormat *result = createInstanceForSkeleton(skeleton, localeWithCalendar, status); + if (U_FAILURE(status)) { + return nullptr; + } + result->adoptCalendar(calendar.orphan()); + return result; +} + +DateFormat* U_EXPORT2 +DateFormat::createInstanceForSkeleton( + const UnicodeString& skeleton, + const Locale &locale, + UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer df( + new SimpleDateFormat( + getBestPattern(locale, skeleton, status), + locale, status), + status); + return U_SUCCESS(status) ? df.orphan() : nullptr; +} + +DateFormat* U_EXPORT2 +DateFormat::createInstanceForSkeleton( + const UnicodeString& skeleton, + UErrorCode &status) { + return createInstanceForSkeleton( + skeleton, Locale::getDefault(), status); +} + +//---------------------------------------------------------------------- + +DateFormat* U_EXPORT2 +DateFormat::create(EStyle timeStyle, EStyle dateStyle, const Locale& locale) +{ + UErrorCode status = U_ZERO_ERROR; +#if U_PLATFORM_USES_ONLY_WIN32_API + char buffer[8]; + int32_t count = locale.getKeywordValue("compat", buffer, sizeof(buffer), status); + + // if the locale has "@compat=host", create a host-specific DateFormat... + if (count > 0 && uprv_strcmp(buffer, "host") == 0) { + Win32DateFormat *f = new Win32DateFormat(timeStyle, dateStyle, locale, status); + + if (U_SUCCESS(status)) { + return f; + } + + delete f; + } +#endif + + // is it relative? + if(/*((timeStyle!=UDAT_NONE)&&(timeStyle & UDAT_RELATIVE)) || */((dateStyle!=kNone)&&((dateStyle-kDateOffset) & UDAT_RELATIVE))) { + RelativeDateFormat *r = new RelativeDateFormat((UDateFormatStyle)timeStyle, (UDateFormatStyle)(dateStyle-kDateOffset), locale, status); + if(U_SUCCESS(status)) return r; + delete r; + status = U_ZERO_ERROR; + } + + // Try to create a SimpleDateFormat of the desired style. + SimpleDateFormat *f = new SimpleDateFormat(timeStyle, dateStyle, locale, status); + if (U_SUCCESS(status)) return f; + delete f; + + // If that fails, try to create a format using the default pattern and + // the DateFormatSymbols for this locale. + status = U_ZERO_ERROR; + f = new SimpleDateFormat(locale, status); + if (U_SUCCESS(status)) return f; + delete f; + + // This should never really happen, because the preceding constructor + // should always succeed. If the resource data is unavailable, a last + // resort object should be returned. + return 0; +} + +//---------------------------------------------------------------------- + +const Locale* U_EXPORT2 +DateFormat::getAvailableLocales(int32_t& count) +{ + // Get the list of installed locales. + // Even if root has the correct date format for this locale, + // it's still a valid locale (we don't worry about data fallbacks). + return Locale::getAvailableLocales(count); +} + +//---------------------------------------------------------------------- + +void +DateFormat::adoptCalendar(Calendar* newCalendar) +{ + delete fCalendar; + fCalendar = newCalendar; +} + +//---------------------------------------------------------------------- +void +DateFormat::setCalendar(const Calendar& newCalendar) +{ + Calendar* newCalClone = newCalendar.clone(); + if (newCalClone != nullptr) { + adoptCalendar(newCalClone); + } +} + +//---------------------------------------------------------------------- + +const Calendar* +DateFormat::getCalendar() const +{ + return fCalendar; +} + +//---------------------------------------------------------------------- + +void +DateFormat::adoptNumberFormat(NumberFormat* newNumberFormat) +{ + delete fNumberFormat; + fNumberFormat = newNumberFormat; + newNumberFormat->setParseIntegerOnly(true); + newNumberFormat->setGroupingUsed(false); +} +//---------------------------------------------------------------------- + +void +DateFormat::setNumberFormat(const NumberFormat& newNumberFormat) +{ + NumberFormat* newNumFmtClone = newNumberFormat.clone(); + if (newNumFmtClone != nullptr) { + adoptNumberFormat(newNumFmtClone); + } +} + +//---------------------------------------------------------------------- + +const NumberFormat* +DateFormat::getNumberFormat() const +{ + return fNumberFormat; +} + +//---------------------------------------------------------------------- + +void +DateFormat::adoptTimeZone(TimeZone* zone) +{ + if (fCalendar != nullptr) { + fCalendar->adoptTimeZone(zone); + } +} +//---------------------------------------------------------------------- + +void +DateFormat::setTimeZone(const TimeZone& zone) +{ + if (fCalendar != nullptr) { + fCalendar->setTimeZone(zone); + } +} + +//---------------------------------------------------------------------- + +const TimeZone& +DateFormat::getTimeZone() const +{ + if (fCalendar != nullptr) { + return fCalendar->getTimeZone(); + } + // If calendar doesn't exists, create default timezone. + // fCalendar is rarely null + return *(TimeZone::createDefault()); +} + +//---------------------------------------------------------------------- + +void +DateFormat::setLenient(UBool lenient) +{ + if (fCalendar != nullptr) { + fCalendar->setLenient(lenient); + } + UErrorCode status = U_ZERO_ERROR; + setBooleanAttribute(UDAT_PARSE_ALLOW_WHITESPACE, lenient, status); + setBooleanAttribute(UDAT_PARSE_ALLOW_NUMERIC, lenient, status); +} + +//---------------------------------------------------------------------- + +UBool +DateFormat::isLenient() const +{ + UBool lenient = true; + if (fCalendar != nullptr) { + lenient = fCalendar->isLenient(); + } + UErrorCode status = U_ZERO_ERROR; + return lenient + && getBooleanAttribute(UDAT_PARSE_ALLOW_WHITESPACE, status) + && getBooleanAttribute(UDAT_PARSE_ALLOW_NUMERIC, status); +} + +void +DateFormat::setCalendarLenient(UBool lenient) +{ + if (fCalendar != nullptr) { + fCalendar->setLenient(lenient); + } +} + +//---------------------------------------------------------------------- + +UBool +DateFormat::isCalendarLenient() const +{ + if (fCalendar != nullptr) { + return fCalendar->isLenient(); + } + // fCalendar is rarely null + return false; +} + + +//---------------------------------------------------------------------- + + +void DateFormat::setContext(UDisplayContext value, UErrorCode& status) +{ + if (U_FAILURE(status)) + return; + if ( (UDisplayContextType)((uint32_t)value >> 8) == UDISPCTX_TYPE_CAPITALIZATION ) { + fCapitalizationContext = value; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + } +} + + +//---------------------------------------------------------------------- + + +UDisplayContext DateFormat::getContext(UDisplayContextType type, UErrorCode& status) const +{ + if (U_FAILURE(status)) + return (UDisplayContext)0; + if (type != UDISPCTX_TYPE_CAPITALIZATION) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return (UDisplayContext)0; + } + return fCapitalizationContext; +} + + +//---------------------------------------------------------------------- + + +DateFormat& +DateFormat::setBooleanAttribute(UDateFormatBooleanAttribute attr, + UBool newValue, + UErrorCode &status) { + if(!fBoolFlags.isValidValue(newValue)) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + fBoolFlags.set(attr, newValue); + } + + return *this; +} + +//---------------------------------------------------------------------- + +UBool +DateFormat::getBooleanAttribute(UDateFormatBooleanAttribute attr, UErrorCode &/*status*/) const { + + return static_cast(fBoolFlags.get(attr)); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/dayperiodrules.cpp b/intl/icu/source/i18n/dayperiodrules.cpp new file mode 100644 index 0000000000..3d9ab5bfac --- /dev/null +++ b/intl/icu/source/i18n/dayperiodrules.cpp @@ -0,0 +1,515 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2016, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* dayperiodrules.cpp +* +* created on: 2016-01-20 +* created by: kazede +*/ + +#include "dayperiodrules.h" + +#include "unicode/ures.h" +#include "charstr.h" +#include "cstring.h" +#include "ucln_in.h" +#include "uhash.h" +#include "umutex.h" +#include "uresimp.h" + + +U_NAMESPACE_BEGIN + +namespace { + +struct DayPeriodRulesData : public UMemory { + DayPeriodRulesData() : localeToRuleSetNumMap(nullptr), rules(nullptr), maxRuleSetNum(0) {} + + UHashtable *localeToRuleSetNumMap; + DayPeriodRules *rules; + int32_t maxRuleSetNum; +} *data = nullptr; + +enum CutoffType { + CUTOFF_TYPE_UNKNOWN = -1, + CUTOFF_TYPE_BEFORE, + CUTOFF_TYPE_AFTER, // TODO: AFTER is deprecated in CLDR 29. Remove. + CUTOFF_TYPE_FROM, + CUTOFF_TYPE_AT +}; + +} // namespace + +struct DayPeriodRulesDataSink : public ResourceSink { + DayPeriodRulesDataSink() { + for (int32_t i = 0; i < UPRV_LENGTHOF(cutoffs); ++i) { cutoffs[i] = 0; } + } + virtual ~DayPeriodRulesDataSink(); + + virtual void put(const char *key, ResourceValue &value, UBool, UErrorCode &errorCode) override { + ResourceTable dayPeriodData = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; dayPeriodData.getKeyAndValue(i, key, value); ++i) { + if (uprv_strcmp(key, "locales") == 0) { + ResourceTable locales = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t j = 0; locales.getKeyAndValue(j, key, value); ++j) { + UnicodeString setNum_str = value.getUnicodeString(errorCode); + int32_t setNum = parseSetNum(setNum_str, errorCode); + uhash_puti(data->localeToRuleSetNumMap, const_cast(key), setNum, &errorCode); + } + } else if (uprv_strcmp(key, "rules") == 0) { + // Allocate one more than needed to skip [0]. See comment in parseSetNum(). + data->rules = new DayPeriodRules[data->maxRuleSetNum + 1]; + if (data->rules == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + ResourceTable rules = value.getTable(errorCode); + processRules(rules, key, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } + } + + void processRules(const ResourceTable &rules, const char *key, + ResourceValue &value, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; rules.getKeyAndValue(i, key, value); ++i) { + ruleSetNum = parseSetNum(key, errorCode); + ResourceTable ruleSet = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t j = 0; ruleSet.getKeyAndValue(j, key, value); ++j) { + period = DayPeriodRules::getDayPeriodFromString(key); + if (period == DayPeriodRules::DAYPERIOD_UNKNOWN) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + ResourceTable periodDefinition = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t k = 0; periodDefinition.getKeyAndValue(k, key, value); ++k) { + if (value.getType() == URES_STRING) { + // Key-value pairs (e.g. before{6:00}). + CutoffType type = getCutoffTypeFromString(key); + addCutoff(type, value.getUnicodeString(errorCode), errorCode); + if (U_FAILURE(errorCode)) { return; } + } else { + // Arrays (e.g. before{6:00, 24:00}). + cutoffType = getCutoffTypeFromString(key); + ResourceArray cutoffArray = value.getArray(errorCode); + if (U_FAILURE(errorCode)) { return; } + + int32_t length = cutoffArray.getSize(); + for (int32_t l = 0; l < length; ++l) { + cutoffArray.getValue(l, value); + addCutoff(cutoffType, value.getUnicodeString(errorCode), errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } + } + setDayPeriodForHoursFromCutoffs(errorCode); + for (int32_t k = 0; k < UPRV_LENGTHOF(cutoffs); ++k) { + cutoffs[k] = 0; + } + } + + if (!data->rules[ruleSetNum].allHoursAreSet()) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + } + } + + // Members. + int32_t cutoffs[25]; // [0] thru [24]: 24 is allowed in "before 24". + + // "Path" to data. + int32_t ruleSetNum; + DayPeriodRules::DayPeriod period; + CutoffType cutoffType; + + // Helpers. + static int32_t parseSetNum(const UnicodeString &setNumStr, UErrorCode &errorCode) { + CharString cs; + cs.appendInvariantChars(setNumStr, errorCode); + return parseSetNum(cs.data(), errorCode); + } + + static int32_t parseSetNum(const char *setNumStr, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return -1; } + + if (uprv_strncmp(setNumStr, "set", 3) != 0) { + errorCode = U_INVALID_FORMAT_ERROR; + return -1; + } + + int32_t i = 3; + int32_t setNum = 0; + while (setNumStr[i] != 0) { + int32_t digit = setNumStr[i] - '0'; + if (digit < 0 || 9 < digit) { + errorCode = U_INVALID_FORMAT_ERROR; + return -1; + } + setNum = 10 * setNum + digit; + ++i; + } + + // Rule set number must not be zero. (0 is used to indicate "not found" by hashmap.) + // Currently ICU data conveniently starts numbering rule sets from 1. + if (setNum == 0) { + errorCode = U_INVALID_FORMAT_ERROR; + return -1; + } else { + return setNum; + } + } + + void addCutoff(CutoffType type, const UnicodeString &hour_str, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return; } + + if (type == CUTOFF_TYPE_UNKNOWN) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + + int32_t hour = parseHour(hour_str, errorCode); + if (U_FAILURE(errorCode)) { return; } + + cutoffs[hour] |= 1 << type; + } + + // Translate the cutoffs[] array to day period rules. + void setDayPeriodForHoursFromCutoffs(UErrorCode &errorCode) { + DayPeriodRules &rule = data->rules[ruleSetNum]; + + for (int32_t startHour = 0; startHour <= 24; ++startHour) { + // AT cutoffs must be either midnight or noon. + if (cutoffs[startHour] & (1 << CUTOFF_TYPE_AT)) { + if (startHour == 0 && period == DayPeriodRules::DAYPERIOD_MIDNIGHT) { + rule.fHasMidnight = true; + } else if (startHour == 12 && period == DayPeriodRules::DAYPERIOD_NOON) { + rule.fHasNoon = true; + } else { + errorCode = U_INVALID_FORMAT_ERROR; // Bad data. + return; + } + } + + // FROM/AFTER and BEFORE must come in a pair. + if (cutoffs[startHour] & (1 << CUTOFF_TYPE_FROM) || + cutoffs[startHour] & (1 << CUTOFF_TYPE_AFTER)) { + for (int32_t hour = startHour + 1;; ++hour) { + if (hour == startHour) { + // We've gone around the array once and can't find a BEFORE. + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + if (hour == 25) { hour = 0; } + if (cutoffs[hour] & (1 << CUTOFF_TYPE_BEFORE)) { + rule.add(startHour, hour, period); + break; + } + } + } + } + } + + // Translate "before" to CUTOFF_TYPE_BEFORE, for example. + static CutoffType getCutoffTypeFromString(const char *type_str) { + if (uprv_strcmp(type_str, "from") == 0) { + return CUTOFF_TYPE_FROM; + } else if (uprv_strcmp(type_str, "before") == 0) { + return CUTOFF_TYPE_BEFORE; + } else if (uprv_strcmp(type_str, "after") == 0) { + return CUTOFF_TYPE_AFTER; + } else if (uprv_strcmp(type_str, "at") == 0) { + return CUTOFF_TYPE_AT; + } else { + return CUTOFF_TYPE_UNKNOWN; + } + } + + // Gets the numerical value of the hour from the Unicode string. + static int32_t parseHour(const UnicodeString &time, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return 0; + } + + int32_t hourLimit = time.length() - 3; + // `time` must look like "x:00" or "xx:00". + // If length is wrong or `time` doesn't end with ":00", error out. + if ((hourLimit != 1 && hourLimit != 2) || + time[hourLimit] != 0x3A || time[hourLimit + 1] != 0x30 || + time[hourLimit + 2] != 0x30) { + errorCode = U_INVALID_FORMAT_ERROR; + return 0; + } + + // If `time` doesn't begin with a number in [0, 24], error out. + // Note: "24:00" is possible in "before 24:00". + int32_t hour = time[0] - 0x30; + if (hour < 0 || 9 < hour) { + errorCode = U_INVALID_FORMAT_ERROR; + return 0; + } + if (hourLimit == 2) { + int32_t hourDigit2 = time[1] - 0x30; + if (hourDigit2 < 0 || 9 < hourDigit2) { + errorCode = U_INVALID_FORMAT_ERROR; + return 0; + } + hour = hour * 10 + hourDigit2; + if (hour > 24) { + errorCode = U_INVALID_FORMAT_ERROR; + return 0; + } + } + + return hour; + } +}; // struct DayPeriodRulesDataSink + +struct DayPeriodRulesCountSink : public ResourceSink { + virtual ~DayPeriodRulesCountSink(); + + virtual void put(const char *key, ResourceValue &value, UBool, UErrorCode &errorCode) override { + ResourceTable rules = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; rules.getKeyAndValue(i, key, value); ++i) { + int32_t setNum = DayPeriodRulesDataSink::parseSetNum(key, errorCode); + if (setNum > data->maxRuleSetNum) { + data->maxRuleSetNum = setNum; + } + } + } +}; + +// Out-of-line virtual destructors. +DayPeriodRulesDataSink::~DayPeriodRulesDataSink() {} +DayPeriodRulesCountSink::~DayPeriodRulesCountSink() {} + +namespace { + +UInitOnce initOnce {}; + +U_CFUNC UBool U_CALLCONV dayPeriodRulesCleanup() { + delete[] data->rules; + uhash_close(data->localeToRuleSetNumMap); + delete data; + data = nullptr; + return true; +} + +} // namespace + +void U_CALLCONV DayPeriodRules::load(UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return; + } + + data = new DayPeriodRulesData(); + data->localeToRuleSetNumMap = uhash_open(uhash_hashChars, uhash_compareChars, nullptr, &errorCode); + LocalUResourceBundlePointer rb_dayPeriods(ures_openDirect(nullptr, "dayPeriods", &errorCode)); + + // Get the largest rule set number (so we allocate enough objects). + DayPeriodRulesCountSink countSink; + ures_getAllItemsWithFallback(rb_dayPeriods.getAlias(), "rules", countSink, errorCode); + + // Populate rules. + DayPeriodRulesDataSink sink; + ures_getAllItemsWithFallback(rb_dayPeriods.getAlias(), "", sink, errorCode); + + ucln_i18n_registerCleanup(UCLN_I18N_DAYPERIODRULES, dayPeriodRulesCleanup); +} + +const DayPeriodRules *DayPeriodRules::getInstance(const Locale &locale, UErrorCode &errorCode) { + umtx_initOnce(initOnce, DayPeriodRules::load, errorCode); + + // If the entire day period rules data doesn't conform to spec (even if the part we want + // does), return nullptr. + if(U_FAILURE(errorCode)) { return nullptr; } + + const char *localeCode = locale.getBaseName(); + char name[ULOC_FULLNAME_CAPACITY]; + char parentName[ULOC_FULLNAME_CAPACITY]; + + if (uprv_strlen(localeCode) < ULOC_FULLNAME_CAPACITY) { + uprv_strcpy(name, localeCode); + + // Treat empty string as root. + if (*name == '\0') { + uprv_strcpy(name, "root"); + } + } else { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return nullptr; + } + + int32_t ruleSetNum = 0; // NB there is no rule set 0 and 0 is returned upon lookup failure. + while (*name != '\0') { + ruleSetNum = uhash_geti(data->localeToRuleSetNumMap, name); + if (ruleSetNum == 0) { + // name and parentName can't be the same pointer, so fill in parent then copy to child. + uloc_getParent(name, parentName, ULOC_FULLNAME_CAPACITY, &errorCode); + if (*parentName == '\0') { + // Saves a lookup in the hash table. + break; + } + uprv_strcpy(name, parentName); + } else { + break; + } + } + + if (ruleSetNum <= 0 || data->rules[ruleSetNum].getDayPeriodForHour(0) == DAYPERIOD_UNKNOWN) { + // If day period for hour 0 is UNKNOWN then day period for all hours are UNKNOWN. + // Data doesn't exist even with fallback. + return nullptr; + } else { + return &data->rules[ruleSetNum]; + } +} + +DayPeriodRules::DayPeriodRules() : fHasMidnight(false), fHasNoon(false) { + for (int32_t i = 0; i < 24; ++i) { + fDayPeriodForHour[i] = DayPeriodRules::DAYPERIOD_UNKNOWN; + } +} + +double DayPeriodRules::getMidPointForDayPeriod( + DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const { + if (U_FAILURE(errorCode)) { return -1; } + + int32_t startHour = getStartHourForDayPeriod(dayPeriod, errorCode); + int32_t endHour = getEndHourForDayPeriod(dayPeriod, errorCode); + // Can't obtain startHour or endHour; bail out. + if (U_FAILURE(errorCode)) { return -1; } + + double midPoint = (startHour + endHour) / 2.0; + + if (startHour > endHour) { + // dayPeriod wraps around midnight. Shift midPoint by 12 hours, in the direction that + // lands it in [0, 24). + midPoint += 12; + if (midPoint >= 24) { + midPoint -= 24; + } + } + + return midPoint; +} + +int32_t DayPeriodRules::getStartHourForDayPeriod( + DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const { + if (U_FAILURE(errorCode)) { return -1; } + + if (dayPeriod == DAYPERIOD_MIDNIGHT) { return 0; } + if (dayPeriod == DAYPERIOD_NOON) { return 12; } + + if (fDayPeriodForHour[0] == dayPeriod && fDayPeriodForHour[23] == dayPeriod) { + // dayPeriod wraps around midnight. Start hour is later than end hour. + for (int32_t i = 22; i >= 1; --i) { + if (fDayPeriodForHour[i] != dayPeriod) { + return (i + 1); + } + } + } else { + for (int32_t i = 0; i <= 23; ++i) { + if (fDayPeriodForHour[i] == dayPeriod) { + return i; + } + } + } + + // dayPeriod doesn't exist in rule set; set error and exit. + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return -1; +} + +int32_t DayPeriodRules::getEndHourForDayPeriod( + DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const { + if (U_FAILURE(errorCode)) { return -1; } + + if (dayPeriod == DAYPERIOD_MIDNIGHT) { return 0; } + if (dayPeriod == DAYPERIOD_NOON) { return 12; } + + if (fDayPeriodForHour[0] == dayPeriod && fDayPeriodForHour[23] == dayPeriod) { + // dayPeriod wraps around midnight. End hour is before start hour. + for (int32_t i = 1; i <= 22; ++i) { + if (fDayPeriodForHour[i] != dayPeriod) { + // i o'clock is when a new period starts, therefore when the old period ends. + return i; + } + } + } else { + for (int32_t i = 23; i >= 0; --i) { + if (fDayPeriodForHour[i] == dayPeriod) { + return (i + 1); + } + } + } + + // dayPeriod doesn't exist in rule set; set error and exit. + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return -1; +} + +DayPeriodRules::DayPeriod DayPeriodRules::getDayPeriodFromString(const char *type_str) { + if (uprv_strcmp(type_str, "midnight") == 0) { + return DAYPERIOD_MIDNIGHT; + } else if (uprv_strcmp(type_str, "noon") == 0) { + return DAYPERIOD_NOON; + } else if (uprv_strcmp(type_str, "morning1") == 0) { + return DAYPERIOD_MORNING1; + } else if (uprv_strcmp(type_str, "afternoon1") == 0) { + return DAYPERIOD_AFTERNOON1; + } else if (uprv_strcmp(type_str, "evening1") == 0) { + return DAYPERIOD_EVENING1; + } else if (uprv_strcmp(type_str, "night1") == 0) { + return DAYPERIOD_NIGHT1; + } else if (uprv_strcmp(type_str, "morning2") == 0) { + return DAYPERIOD_MORNING2; + } else if (uprv_strcmp(type_str, "afternoon2") == 0) { + return DAYPERIOD_AFTERNOON2; + } else if (uprv_strcmp(type_str, "evening2") == 0) { + return DAYPERIOD_EVENING2; + } else if (uprv_strcmp(type_str, "night2") == 0) { + return DAYPERIOD_NIGHT2; + } else if (uprv_strcmp(type_str, "am") == 0) { + return DAYPERIOD_AM; + } else if (uprv_strcmp(type_str, "pm") == 0) { + return DAYPERIOD_PM; + } else { + return DAYPERIOD_UNKNOWN; + } +} + +void DayPeriodRules::add(int32_t startHour, int32_t limitHour, DayPeriod period) { + for (int32_t i = startHour; i != limitHour; ++i) { + if (i == 24) { i = 0; } + fDayPeriodForHour[i] = period; + } +} + +UBool DayPeriodRules::allHoursAreSet() { + for (int32_t i = 0; i < 24; ++i) { + if (fDayPeriodForHour[i] == DAYPERIOD_UNKNOWN) { return false; } + } + + return true; +} + + + +U_NAMESPACE_END diff --git a/intl/icu/source/i18n/dayperiodrules.h b/intl/icu/source/i18n/dayperiodrules.h new file mode 100644 index 0000000000..4bfca762b8 --- /dev/null +++ b/intl/icu/source/i18n/dayperiodrules.h @@ -0,0 +1,89 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2016, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* dayperiodrules.h +* +* created on: 2016-01-20 +* created by: kazede +*/ + +#ifndef DAYPERIODRULES_H +#define DAYPERIODRULES_H + +#include "unicode/locid.h" +#include "unicode/unistr.h" +#include "unicode/uobject.h" +#include "unicode/utypes.h" +#include "resource.h" +#include "uhash.h" + + + +U_NAMESPACE_BEGIN + +struct DayPeriodRulesDataSink; + +class DayPeriodRules : public UMemory { + friend struct DayPeriodRulesDataSink; +public: + enum DayPeriod { + DAYPERIOD_UNKNOWN = -1, + DAYPERIOD_MIDNIGHT, + DAYPERIOD_NOON, + DAYPERIOD_MORNING1, + DAYPERIOD_AFTERNOON1, + DAYPERIOD_EVENING1, + DAYPERIOD_NIGHT1, + DAYPERIOD_MORNING2, + DAYPERIOD_AFTERNOON2, + DAYPERIOD_EVENING2, + DAYPERIOD_NIGHT2, + DAYPERIOD_AM, + DAYPERIOD_PM + }; + + static const DayPeriodRules *getInstance(const Locale &locale, UErrorCode &errorCode); + + UBool hasMidnight() const { return fHasMidnight; } + UBool hasNoon() const { return fHasNoon; } + DayPeriod getDayPeriodForHour(int32_t hour) const { return fDayPeriodForHour[hour]; } + + // Returns the center of dayPeriod. Half hours are indicated with a .5 . + double getMidPointForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const; + +private: + DayPeriodRules(); + + // Translates "morning1" to DAYPERIOD_MORNING1, for example. + static DayPeriod getDayPeriodFromString(const char *type_str); + + static void U_CALLCONV load(UErrorCode &errorCode); + + // Sets period type for all hours in [startHour, limitHour). + void add(int32_t startHour, int32_t limitHour, DayPeriod period); + + // Returns true if for all i, DayPeriodForHour[i] has a type other than UNKNOWN. + // Values of HasNoon and HasMidnight do not affect the return value. + UBool allHoursAreSet(); + + // Returns the hour that starts dayPeriod. Returns 0 for MIDNIGHT and 12 for NOON. + int32_t getStartHourForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const; + + // Returns the hour that ends dayPeriod, i.e. that starts the next period. + // E.g. if fDayPeriodForHour[13] thru [16] are AFTERNOON1, then this function returns 17 if + // queried with AFTERNOON1. + // Returns 0 for MIDNIGHT and 12 for NOON. + int32_t getEndHourForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const; + + UBool fHasMidnight; + UBool fHasNoon; + DayPeriod fDayPeriodForHour[24]; +}; + +U_NAMESPACE_END + +#endif /* DAYPERIODRULES_H */ diff --git a/intl/icu/source/i18n/dcfmtsym.cpp b/intl/icu/source/i18n/dcfmtsym.cpp new file mode 100644 index 0000000000..ac1f777399 --- /dev/null +++ b/intl/icu/source/i18n/dcfmtsym.cpp @@ -0,0 +1,603 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DCFMTSYM.CPP +* +* Modification History: +* +* Date Name Description +* 02/19/97 aliu Converted from java. +* 03/18/97 clhuang Implemented with C++ APIs. +* 03/27/97 helena Updated to pass the simple test after code review. +* 08/26/97 aliu Added currency/intl currency symbol support. +* 07/20/98 stephen Slightly modified initialization of monetarySeparator +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/dcfmtsym.h" +#include "unicode/ures.h" +#include "unicode/decimfmt.h" +#include "unicode/ucurr.h" +#include "unicode/choicfmt.h" +#include "unicode/unistr.h" +#include "unicode/numsys.h" +#include "unicode/unum.h" +#include "unicode/utf16.h" +#include "ucurrimp.h" +#include "cstring.h" +#include "locbased.h" +#include "uresimp.h" +#include "ureslocs.h" +#include "charstr.h" +#include "uassert.h" + +// ***************************************************************************** +// class DecimalFormatSymbols +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DecimalFormatSymbols) + +static const char gNumberElements[] = "NumberElements"; +static const char gCurrencySpacingTag[] = "currencySpacing"; +static const char gBeforeCurrencyTag[] = "beforeCurrency"; +static const char gAfterCurrencyTag[] = "afterCurrency"; +static const char gCurrencyMatchTag[] = "currencyMatch"; +static const char gCurrencySudMatchTag[] = "surroundingMatch"; +static const char gCurrencyInsertBtnTag[] = "insertBetween"; +static const char gLatn[] = "latn"; +static const char gSymbols[] = "symbols"; +static const char gNumberElementsLatnSymbols[] = "NumberElements/latn/symbols"; + +static const char16_t INTL_CURRENCY_SYMBOL_STR[] = {0xa4, 0xa4, 0}; + +// List of field names to be loaded from the data files. +// These are parallel with the enum ENumberFormatSymbol in unicode/dcfmtsym.h. +static const char *gNumberElementKeys[DecimalFormatSymbols::kFormatSymbolCount] = { + "decimal", + "group", + nullptr, /* #11897: the symbol is NOT the pattern separator symbol */ + "percentSign", + nullptr, /* Native zero digit is deprecated from CLDR - get it from the numbering system */ + nullptr, /* Pattern digit character is deprecated from CLDR - use # by default always */ + "minusSign", + "plusSign", + nullptr, /* currency symbol - Wait until we know the currency before loading from CLDR */ + nullptr, /* intl currency symbol - Wait until we know the currency before loading from CLDR */ + "currencyDecimal", + "exponential", + "perMille", + nullptr, /* Escape padding character - not in CLDR */ + "infinity", + "nan", + nullptr, /* Significant digit symbol - not in CLDR */ + "currencyGroup", + nullptr, /* one digit - get it from the numbering system */ + nullptr, /* two digit - get it from the numbering system */ + nullptr, /* three digit - get it from the numbering system */ + nullptr, /* four digit - get it from the numbering system */ + nullptr, /* five digit - get it from the numbering system */ + nullptr, /* six digit - get it from the numbering system */ + nullptr, /* seven digit - get it from the numbering system */ + nullptr, /* eight digit - get it from the numbering system */ + nullptr, /* nine digit - get it from the numbering system */ + "superscriptingExponent", /* Multiplication (x) symbol for exponents */ + "approximatelySign" /* Approximately sign symbol */ +}; + +// ------------------------------------- +// Initializes this with the decimal format symbols in the default locale. + +DecimalFormatSymbols::DecimalFormatSymbols(UErrorCode& status) + : UObject(), locale() { + initialize(locale, status, true); +} + +// ------------------------------------- +// Initializes this with the decimal format symbols in the desired locale. + +DecimalFormatSymbols::DecimalFormatSymbols(const Locale& loc, UErrorCode& status) + : UObject(), locale(loc) { + initialize(locale, status); +} + +DecimalFormatSymbols::DecimalFormatSymbols(const Locale& loc, const NumberingSystem& ns, UErrorCode& status) + : UObject(), locale(loc) { + initialize(locale, status, false, &ns); +} + +DecimalFormatSymbols::DecimalFormatSymbols() + : UObject(), locale(Locale::getRoot()) { + *validLocale = *actualLocale = 0; + initialize(); +} + +DecimalFormatSymbols* +DecimalFormatSymbols::createWithLastResortData(UErrorCode& status) { + if (U_FAILURE(status)) { return nullptr; } + DecimalFormatSymbols* sym = new DecimalFormatSymbols(); + if (sym == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return sym; +} + +// ------------------------------------- + +DecimalFormatSymbols::~DecimalFormatSymbols() +{ +} + +// ------------------------------------- +// copy constructor + +DecimalFormatSymbols::DecimalFormatSymbols(const DecimalFormatSymbols &source) + : UObject(source) +{ + *this = source; +} + +// ------------------------------------- +// assignment operator + +DecimalFormatSymbols& +DecimalFormatSymbols::operator=(const DecimalFormatSymbols& rhs) +{ + if (this != &rhs) { + for(int32_t i = 0; i < (int32_t)kFormatSymbolCount; ++i) { + // fastCopyFrom is safe, see docs on fSymbols + fSymbols[(ENumberFormatSymbol)i].fastCopyFrom(rhs.fSymbols[(ENumberFormatSymbol)i]); + } + for(int32_t i = 0; i < (int32_t)UNUM_CURRENCY_SPACING_COUNT; ++i) { + currencySpcBeforeSym[i].fastCopyFrom(rhs.currencySpcBeforeSym[i]); + currencySpcAfterSym[i].fastCopyFrom(rhs.currencySpcAfterSym[i]); + } + locale = rhs.locale; + uprv_strcpy(validLocale, rhs.validLocale); + uprv_strcpy(actualLocale, rhs.actualLocale); + fIsCustomCurrencySymbol = rhs.fIsCustomCurrencySymbol; + fIsCustomIntlCurrencySymbol = rhs.fIsCustomIntlCurrencySymbol; + fCodePointZero = rhs.fCodePointZero; + currPattern = rhs.currPattern; + uprv_strcpy(nsName, rhs.nsName); + } + return *this; +} + +// ------------------------------------- + +bool +DecimalFormatSymbols::operator==(const DecimalFormatSymbols& that) const +{ + if (this == &that) { + return true; + } + if (fIsCustomCurrencySymbol != that.fIsCustomCurrencySymbol) { + return false; + } + if (fIsCustomIntlCurrencySymbol != that.fIsCustomIntlCurrencySymbol) { + return false; + } + for(int32_t i = 0; i < (int32_t)kFormatSymbolCount; ++i) { + if(fSymbols[(ENumberFormatSymbol)i] != that.fSymbols[(ENumberFormatSymbol)i]) { + return false; + } + } + for(int32_t i = 0; i < (int32_t)UNUM_CURRENCY_SPACING_COUNT; ++i) { + if(currencySpcBeforeSym[i] != that.currencySpcBeforeSym[i]) { + return false; + } + if(currencySpcAfterSym[i] != that.currencySpcAfterSym[i]) { + return false; + } + } + // No need to check fCodePointZero since it is based on fSymbols + return locale == that.locale && + uprv_strcmp(validLocale, that.validLocale) == 0 && + uprv_strcmp(actualLocale, that.actualLocale) == 0; +} + +// ------------------------------------- + +namespace { + +/** + * Sink for enumerating all of the decimal format symbols (more specifically, anything + * under the "NumberElements.symbols" tree). + * + * More specific bundles (en_GB) are enumerated before their parents (en_001, en, root): + * Only store a value if it is still missing, that is, it has not been overridden. + */ +struct DecFmtSymDataSink : public ResourceSink { + + // Destination for data, modified via setters. + DecimalFormatSymbols& dfs; + // Boolean array of whether or not we have seen a particular symbol yet. + // Can't simply check fSymbols because it is pre-populated with defaults. + UBool seenSymbol[DecimalFormatSymbols::kFormatSymbolCount]; + + // Constructor/Destructor + DecFmtSymDataSink(DecimalFormatSymbols& _dfs) : dfs(_dfs) { + uprv_memset(seenSymbol, false, sizeof(seenSymbol)); + } + virtual ~DecFmtSymDataSink(); + + virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/, + UErrorCode &errorCode) override { + ResourceTable symbolsTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + for (int32_t j = 0; symbolsTable.getKeyAndValue(j, key, value); ++j) { + for (int32_t i=0; i nsLocal; + if (ns == nullptr) { + // Use the numbering system according to the locale. + // Save it into a LocalPointer so it gets cleaned up. + nsLocal.adoptInstead(NumberingSystem::createInstance(loc, status)); + ns = nsLocal.getAlias(); + } + const char *nsName; + if (U_SUCCESS(status) && ns->getRadix() == 10 && !ns->isAlgorithmic()) { + nsName = ns->getName(); + UnicodeString digitString(ns->getDescription()); + int32_t digitIndex = 0; + UChar32 digit = digitString.char32At(0); + fSymbols[kZeroDigitSymbol].setTo(digit); + for (int32_t i = kOneDigitSymbol; i <= kNineDigitSymbol; ++i) { + digitIndex += U16_LENGTH(digit); + digit = digitString.char32At(digitIndex); + fSymbols[i].setTo(digit); + } + } else { + nsName = gLatn; + } + uprv_strcpy(this->nsName, nsName); + + // Open resource bundles + const char* locStr = loc.getName(); + LocalUResourceBundlePointer resource(ures_open(nullptr, locStr, &status)); + LocalUResourceBundlePointer numberElementsRes( + ures_getByKeyWithFallback(resource.getAlias(), gNumberElements, nullptr, &status)); + + if (U_FAILURE(status)) { + if ( useLastResortData ) { + status = U_USING_DEFAULT_WARNING; + initialize(); + } + return; + } + + // Set locale IDs + // TODO: Is there a way to do this without depending on the resource bundle instance? + U_LOCALE_BASED(locBased, *this); + locBased.setLocaleIDs( + ures_getLocaleByType( + numberElementsRes.getAlias(), + ULOC_VALID_LOCALE, &status), + ures_getLocaleByType( + numberElementsRes.getAlias(), + ULOC_ACTUAL_LOCALE, &status)); + + // Now load the rest of the data from the data sink. + // Start with loading this nsName if it is not Latin. + DecFmtSymDataSink sink(*this); + if (uprv_strcmp(nsName, gLatn) != 0) { + CharString path; + path.append(gNumberElements, status) + .append('/', status) + .append(nsName, status) + .append('/', status) + .append(gSymbols, status); + ures_getAllItemsWithFallback(resource.getAlias(), path.data(), sink, status); + + // If no symbols exist for the given nsName and resource bundle, silently ignore + // and fall back to Latin. + if (status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + } else if (U_FAILURE(status)) { + return; + } + } + + // Continue with Latin if necessary. + if (!sink.seenAll()) { + ures_getAllItemsWithFallback(resource.getAlias(), gNumberElementsLatnSymbols, sink, status); + if (U_FAILURE(status)) { return; } + } + + // Let the monetary number separators equal the default number separators if necessary. + sink.resolveMissingMonetarySeparators(fSymbols); + + // Resolve codePointZero + UChar32 tempCodePointZero = -1; + for (int32_t i=0; i<=9; i++) { + const UnicodeString& stringDigit = getConstDigitSymbol(i); + if (stringDigit.countChar32() != 1) { + tempCodePointZero = -1; + break; + } + UChar32 cp = stringDigit.char32At(0); + if (i == 0) { + tempCodePointZero = cp; + } else if (cp != tempCodePointZero + i) { + tempCodePointZero = -1; + break; + } + } + fCodePointZero = tempCodePointZero; + + // Get the default currency from the currency API. + UErrorCode internalStatus = U_ZERO_ERROR; // don't propagate failures out + char16_t curriso[4]; + UnicodeString tempStr; + int32_t currisoLength = ucurr_forLocale(locStr, curriso, UPRV_LENGTHOF(curriso), &internalStatus); + if (U_SUCCESS(internalStatus) && currisoLength == 3) { + setCurrency(curriso, status); + } else { + setCurrency(nullptr, status); + } + + // Currency Spacing. + LocalUResourceBundlePointer currencyResource(ures_open(U_ICUDATA_CURR, locStr, &status)); + CurrencySpacingSink currencySink(*this); + ures_getAllItemsWithFallback(currencyResource.getAlias(), gCurrencySpacingTag, currencySink, status); + currencySink.resolveMissing(); + if (U_FAILURE(status)) { return; } +} + +void +DecimalFormatSymbols::initialize() { + /* + * These strings used to be in static arrays, but the HP/UX aCC compiler + * cannot initialize a static array with class constructors. + * markus 2000may25 + */ + fSymbols[kDecimalSeparatorSymbol] = (char16_t)0x2e; // '.' decimal separator + fSymbols[kGroupingSeparatorSymbol].remove(); // group (thousands) separator + fSymbols[kPatternSeparatorSymbol] = (char16_t)0x3b; // ';' pattern separator + fSymbols[kPercentSymbol] = (char16_t)0x25; // '%' percent sign + fSymbols[kZeroDigitSymbol] = (char16_t)0x30; // '0' native 0 digit + fSymbols[kOneDigitSymbol] = (char16_t)0x31; // '1' native 1 digit + fSymbols[kTwoDigitSymbol] = (char16_t)0x32; // '2' native 2 digit + fSymbols[kThreeDigitSymbol] = (char16_t)0x33; // '3' native 3 digit + fSymbols[kFourDigitSymbol] = (char16_t)0x34; // '4' native 4 digit + fSymbols[kFiveDigitSymbol] = (char16_t)0x35; // '5' native 5 digit + fSymbols[kSixDigitSymbol] = (char16_t)0x36; // '6' native 6 digit + fSymbols[kSevenDigitSymbol] = (char16_t)0x37; // '7' native 7 digit + fSymbols[kEightDigitSymbol] = (char16_t)0x38; // '8' native 8 digit + fSymbols[kNineDigitSymbol] = (char16_t)0x39; // '9' native 9 digit + fSymbols[kDigitSymbol] = (char16_t)0x23; // '#' pattern digit + fSymbols[kPlusSignSymbol] = (char16_t)0x002b; // '+' plus sign + fSymbols[kMinusSignSymbol] = (char16_t)0x2d; // '-' minus sign + fSymbols[kCurrencySymbol] = (char16_t)0xa4; // 'OX' currency symbol + fSymbols[kIntlCurrencySymbol].setTo(true, INTL_CURRENCY_SYMBOL_STR, 2); + fSymbols[kMonetarySeparatorSymbol] = (char16_t)0x2e; // '.' monetary decimal separator + fSymbols[kExponentialSymbol] = (char16_t)0x45; // 'E' exponential + fSymbols[kPerMillSymbol] = (char16_t)0x2030; // '%o' per mill + fSymbols[kPadEscapeSymbol] = (char16_t)0x2a; // '*' pad escape symbol + fSymbols[kInfinitySymbol] = (char16_t)0x221e; // 'oo' infinite + fSymbols[kNaNSymbol] = (char16_t)0xfffd; // SUB NaN + fSymbols[kSignificantDigitSymbol] = (char16_t)0x0040; // '@' significant digit + fSymbols[kMonetaryGroupingSeparatorSymbol].remove(); // + fSymbols[kExponentMultiplicationSymbol] = (char16_t)0xd7; // 'x' multiplication symbol for exponents + fSymbols[kApproximatelySignSymbol] = u'~'; // '~' approximately sign + fIsCustomCurrencySymbol = false; + fIsCustomIntlCurrencySymbol = false; + fCodePointZero = 0x30; + U_ASSERT(fCodePointZero == fSymbols[kZeroDigitSymbol].char32At(0)); + currPattern = nullptr; + nsName[0] = 0; +} + +void DecimalFormatSymbols::setCurrency(const char16_t* currency, UErrorCode& status) { + // TODO: If this method is made public: + // - Adopt ICU4J behavior of not allowing currency to be null. + // - Also verify that the length of currency is 3. + if (!currency) { + return; + } + + UnicodeString tempStr; + uprv_getStaticCurrencyName(currency, locale.getName(), tempStr, status); + if (U_SUCCESS(status)) { + fSymbols[kIntlCurrencySymbol].setTo(currency, 3); + fSymbols[kCurrencySymbol] = tempStr; + } + + char cc[4]={0}; + u_UCharsToChars(currency, cc, 3); + + /* An explicit currency was requested */ + // TODO(ICU-13297): Move this data loading logic into a centralized place + UErrorCode localStatus = U_ZERO_ERROR; + LocalUResourceBundlePointer rbTop(ures_open(U_ICUDATA_CURR, locale.getName(), &localStatus)); + LocalUResourceBundlePointer rb( + ures_getByKeyWithFallback(rbTop.getAlias(), "Currencies", nullptr, &localStatus)); + ures_getByKeyWithFallback(rb.getAlias(), cc, rb.getAlias(), &localStatus); + if(U_SUCCESS(localStatus) && ures_getSize(rb.getAlias())>2) { // the length is 3 if more data is present + ures_getByIndex(rb.getAlias(), 2, rb.getAlias(), &localStatus); + int32_t currPatternLen = 0; + currPattern = + ures_getStringByIndex(rb.getAlias(), (int32_t)0, &currPatternLen, &localStatus); + UnicodeString decimalSep = + ures_getUnicodeStringByIndex(rb.getAlias(), (int32_t)1, &localStatus); + UnicodeString groupingSep = + ures_getUnicodeStringByIndex(rb.getAlias(), (int32_t)2, &localStatus); + if(U_SUCCESS(localStatus)){ + fSymbols[kMonetaryGroupingSeparatorSymbol] = groupingSep; + fSymbols[kMonetarySeparatorSymbol] = decimalSep; + //pattern.setTo(true, currPattern, currPatternLen); + } + } + /* else An explicit currency was requested and is unknown or locale data is malformed. */ + /* ucurr_* API will get the correct value later on. */ +} + +Locale +DecimalFormatSymbols::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocale(type, status); +} + +const UnicodeString& +DecimalFormatSymbols::getPatternForCurrencySpacing(UCurrencySpacing type, + UBool beforeCurrency, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return fNoSymbol; // always empty. + } + if (beforeCurrency) { + return currencySpcBeforeSym[(int32_t)type]; + } else { + return currencySpcAfterSym[(int32_t)type]; + } +} + +void +DecimalFormatSymbols::setPatternForCurrencySpacing(UCurrencySpacing type, + UBool beforeCurrency, + const UnicodeString& pattern) { + if (beforeCurrency) { + currencySpcBeforeSym[(int32_t)type] = pattern; + } else { + currencySpcAfterSym[(int32_t)type] = pattern; + } +} +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/decContext.cpp b/intl/icu/source/i18n/decContext.cpp new file mode 100644 index 0000000000..bdc5f22fe5 --- /dev/null +++ b/intl/icu/source/i18n/decContext.cpp @@ -0,0 +1,432 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* ------------------------------------------------------------------ */ +/* Decimal Context module */ +/* ------------------------------------------------------------------ */ +/* Copyright (c) IBM Corporation, 2000-2012. All rights reserved. */ +/* */ +/* This software is made available under the terms of the */ +/* ICU License -- ICU 1.8.1 and later. */ +/* */ +/* The description and User's Guide ("The decNumber C Library") for */ +/* this software is called decNumber.pdf. This document is */ +/* available, together with arithmetic and format specifications, */ +/* testcases, and Web links, on the General Decimal Arithmetic page. */ +/* */ +/* Please send comments, suggestions, and corrections to the author: */ +/* mfc@uk.ibm.com */ +/* Mike Cowlishaw, IBM Fellow */ +/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */ +/* ------------------------------------------------------------------ */ +/* This module comprises the routines for handling arithmetic */ +/* context structures. */ +/* ------------------------------------------------------------------ */ + +#include /* for strcmp */ +#include /* for printf if DECCHECK */ +#include "decContext.h" /* context and base types */ +#include "decNumberLocal.h" /* decNumber local types, etc. */ + +#if 0 /* ICU: No need to test endianness at runtime. */ +/* compile-time endian tester [assumes sizeof(Int)>1] */ +static const Int mfcone=1; /* constant 1 */ +static const Flag *mfctop=(Flag *)&mfcone; /* -> top byte */ +#define LITEND *mfctop /* named flag; 1=little-endian */ +#endif + +/* ------------------------------------------------------------------ */ +/* decContextClearStatus -- clear bits in current status */ +/* */ +/* context is the context structure to be queried */ +/* mask indicates the bits to be cleared (the status bit that */ +/* corresponds to each 1 bit in the mask is cleared) */ +/* returns context */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextClearStatus(decContext *context, uInt mask) { + context->status&=~mask; + return context; + } /* decContextClearStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextDefault -- initialize a context structure */ +/* */ +/* context is the structure to be initialized */ +/* kind selects the required set of default values, one of: */ +/* DEC_INIT_BASE -- select ANSI X3-274 defaults */ +/* DEC_INIT_DECIMAL32 -- select IEEE 754 defaults, 32-bit */ +/* DEC_INIT_DECIMAL64 -- select IEEE 754 defaults, 64-bit */ +/* DEC_INIT_DECIMAL128 -- select IEEE 754 defaults, 128-bit */ +/* For any other value a valid context is returned, but with */ +/* Invalid_operation set in the status field. */ +/* returns a context structure with the appropriate initial values. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextDefault(decContext *context, Int kind) { + /* set defaults... */ + context->digits=9; /* 9 digits */ + context->emax=DEC_MAX_EMAX; /* 9-digit exponents */ + context->emin=DEC_MIN_EMIN; /* .. balanced */ + context->round=DEC_ROUND_HALF_UP; /* 0.5 rises */ + context->traps=DEC_Errors; /* all but informational */ + context->status=0; /* cleared */ + context->clamp=0; /* no clamping */ + #if DECSUBSET + context->extended=0; /* cleared */ + #endif + switch (kind) { + case DEC_INIT_BASE: + /* [use defaults] */ + break; + case DEC_INIT_DECIMAL32: + context->digits=7; /* digits */ + context->emax=96; /* Emax */ + context->emin=-95; /* Emin */ + context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */ + context->traps=0; /* no traps set */ + context->clamp=1; /* clamp exponents */ + #if DECSUBSET + context->extended=1; /* set */ + #endif + break; + case DEC_INIT_DECIMAL64: + context->digits=16; /* digits */ + context->emax=384; /* Emax */ + context->emin=-383; /* Emin */ + context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */ + context->traps=0; /* no traps set */ + context->clamp=1; /* clamp exponents */ + #if DECSUBSET + context->extended=1; /* set */ + #endif + break; + case DEC_INIT_DECIMAL128: + context->digits=34; /* digits */ + context->emax=6144; /* Emax */ + context->emin=-6143; /* Emin */ + context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */ + context->traps=0; /* no traps set */ + context->clamp=1; /* clamp exponents */ + #if DECSUBSET + context->extended=1; /* set */ + #endif + break; + + default: /* invalid Kind */ + /* use defaults, and .. */ + uprv_decContextSetStatus(context, DEC_Invalid_operation); /* trap */ + } + + return context;} /* decContextDefault */ + +/* ------------------------------------------------------------------ */ +/* decContextGetRounding -- return current rounding mode */ +/* */ +/* context is the context structure to be queried */ +/* returns the rounding mode */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI enum rounding U_EXPORT2 uprv_decContextGetRounding(decContext *context) { + return context->round; + } /* decContextGetRounding */ + +/* ------------------------------------------------------------------ */ +/* decContextGetStatus -- return current status */ +/* */ +/* context is the context structure to be queried */ +/* returns status */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI uInt U_EXPORT2 uprv_decContextGetStatus(decContext *context) { + return context->status; + } /* decContextGetStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextRestoreStatus -- restore bits in current status */ +/* */ +/* context is the context structure to be updated */ +/* newstatus is the source for the bits to be restored */ +/* mask indicates the bits to be restored (the status bit that */ +/* corresponds to each 1 bit in the mask is set to the value of */ +/* the corresponding bit in newstatus) */ +/* returns context */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextRestoreStatus(decContext *context, + uInt newstatus, uInt mask) { + context->status&=~mask; /* clear the selected bits */ + context->status|=(mask&newstatus); /* or in the new bits */ + return context; + } /* decContextRestoreStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextSaveStatus -- save bits in current status */ +/* */ +/* context is the context structure to be queried */ +/* mask indicates the bits to be saved (the status bits that */ +/* correspond to each 1 bit in the mask are saved) */ +/* returns the AND of the mask and the current status */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI uInt U_EXPORT2 uprv_decContextSaveStatus(decContext *context, uInt mask) { + return context->status&mask; + } /* decContextSaveStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextSetRounding -- set current rounding mode */ +/* */ +/* context is the context structure to be updated */ +/* newround is the value which will replace the current mode */ +/* returns context */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextSetRounding(decContext *context, + enum rounding newround) { + context->round=newround; + return context; + } /* decContextSetRounding */ + +/* ------------------------------------------------------------------ */ +/* decContextSetStatus -- set status and raise trap if appropriate */ +/* */ +/* context is the context structure to be updated */ +/* status is the DEC_ exception code */ +/* returns the context structure */ +/* */ +/* Control may never return from this routine, if there is a signal */ +/* handler and it takes a long jump. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatus(decContext *context, uInt status) { + context->status|=status; +#if 0 /* ICU: Do not raise signals. */ + if (status & context->traps) raise(SIGFPE); +#endif + return context;} /* decContextSetStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextSetStatusFromString -- set status from a string + trap */ +/* */ +/* context is the context structure to be updated */ +/* string is a string exactly equal to one that might be returned */ +/* by decContextStatusToString */ +/* */ +/* The status bit corresponding to the string is set, and a trap */ +/* is raised if appropriate. */ +/* */ +/* returns the context structure, unless the string is equal to */ +/* DEC_Condition_MU or is not recognized. In these cases nullptr is */ +/* returned. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromString(decContext *context, + const char *string) { + if (strcmp(string, DEC_Condition_CS)==0) + return uprv_decContextSetStatus(context, DEC_Conversion_syntax); + if (strcmp(string, DEC_Condition_DZ)==0) + return uprv_decContextSetStatus(context, DEC_Division_by_zero); + if (strcmp(string, DEC_Condition_DI)==0) + return uprv_decContextSetStatus(context, DEC_Division_impossible); + if (strcmp(string, DEC_Condition_DU)==0) + return uprv_decContextSetStatus(context, DEC_Division_undefined); + if (strcmp(string, DEC_Condition_IE)==0) + return uprv_decContextSetStatus(context, DEC_Inexact); + if (strcmp(string, DEC_Condition_IS)==0) + return uprv_decContextSetStatus(context, DEC_Insufficient_storage); + if (strcmp(string, DEC_Condition_IC)==0) + return uprv_decContextSetStatus(context, DEC_Invalid_context); + if (strcmp(string, DEC_Condition_IO)==0) + return uprv_decContextSetStatus(context, DEC_Invalid_operation); + #if DECSUBSET + if (strcmp(string, DEC_Condition_LD)==0) + return uprv_decContextSetStatus(context, DEC_Lost_digits); + #endif + if (strcmp(string, DEC_Condition_OV)==0) + return uprv_decContextSetStatus(context, DEC_Overflow); + if (strcmp(string, DEC_Condition_PA)==0) + return uprv_decContextSetStatus(context, DEC_Clamped); + if (strcmp(string, DEC_Condition_RO)==0) + return uprv_decContextSetStatus(context, DEC_Rounded); + if (strcmp(string, DEC_Condition_SU)==0) + return uprv_decContextSetStatus(context, DEC_Subnormal); + if (strcmp(string, DEC_Condition_UN)==0) + return uprv_decContextSetStatus(context, DEC_Underflow); + if (strcmp(string, DEC_Condition_ZE)==0) + return context; + return nullptr; /* Multiple status, or unknown */ + } /* decContextSetStatusFromString */ + +/* ------------------------------------------------------------------ */ +/* decContextSetStatusFromStringQuiet -- set status from a string */ +/* */ +/* context is the context structure to be updated */ +/* string is a string exactly equal to one that might be returned */ +/* by decContextStatusToString */ +/* */ +/* The status bit corresponding to the string is set; no trap is */ +/* raised. */ +/* */ +/* returns the context structure, unless the string is equal to */ +/* DEC_Condition_MU or is not recognized. In these cases nullptr is */ +/* returned. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromStringQuiet(decContext *context, + const char *string) { + if (strcmp(string, DEC_Condition_CS)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Conversion_syntax); + if (strcmp(string, DEC_Condition_DZ)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Division_by_zero); + if (strcmp(string, DEC_Condition_DI)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Division_impossible); + if (strcmp(string, DEC_Condition_DU)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Division_undefined); + if (strcmp(string, DEC_Condition_IE)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Inexact); + if (strcmp(string, DEC_Condition_IS)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Insufficient_storage); + if (strcmp(string, DEC_Condition_IC)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Invalid_context); + if (strcmp(string, DEC_Condition_IO)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Invalid_operation); + #if DECSUBSET + if (strcmp(string, DEC_Condition_LD)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Lost_digits); + #endif + if (strcmp(string, DEC_Condition_OV)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Overflow); + if (strcmp(string, DEC_Condition_PA)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Clamped); + if (strcmp(string, DEC_Condition_RO)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Rounded); + if (strcmp(string, DEC_Condition_SU)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Subnormal); + if (strcmp(string, DEC_Condition_UN)==0) + return uprv_decContextSetStatusQuiet(context, DEC_Underflow); + if (strcmp(string, DEC_Condition_ZE)==0) + return context; + return nullptr; /* Multiple status, or unknown */ + } /* decContextSetStatusFromStringQuiet */ + +/* ------------------------------------------------------------------ */ +/* decContextSetStatusQuiet -- set status without trap */ +/* */ +/* context is the context structure to be updated */ +/* status is the DEC_ exception code */ +/* returns the context structure */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusQuiet(decContext *context, uInt status) { + context->status|=status; + return context;} /* decContextSetStatusQuiet */ + +/* ------------------------------------------------------------------ */ +/* decContextStatusToString -- convert status flags to a string */ +/* */ +/* context is a context with valid status field */ +/* */ +/* returns a constant string describing the condition. If multiple */ +/* (or no) flags are set, a generic constant message is returned. */ +/* ------------------------------------------------------------------ */ +U_CAPI const char * U_EXPORT2 uprv_decContextStatusToString(const decContext *context) { + Int status=context->status; + + /* test the five IEEE first, as some of the others are ambiguous when */ + /* DECEXTFLAG=0 */ + if (status==DEC_Invalid_operation ) return DEC_Condition_IO; + if (status==DEC_Division_by_zero ) return DEC_Condition_DZ; + if (status==DEC_Overflow ) return DEC_Condition_OV; + if (status==DEC_Underflow ) return DEC_Condition_UN; + if (status==DEC_Inexact ) return DEC_Condition_IE; + + if (status==DEC_Division_impossible ) return DEC_Condition_DI; + if (status==DEC_Division_undefined ) return DEC_Condition_DU; + if (status==DEC_Rounded ) return DEC_Condition_RO; + if (status==DEC_Clamped ) return DEC_Condition_PA; + if (status==DEC_Subnormal ) return DEC_Condition_SU; + if (status==DEC_Conversion_syntax ) return DEC_Condition_CS; + if (status==DEC_Insufficient_storage ) return DEC_Condition_IS; + if (status==DEC_Invalid_context ) return DEC_Condition_IC; + #if DECSUBSET + if (status==DEC_Lost_digits ) return DEC_Condition_LD; + #endif + if (status==0 ) return DEC_Condition_ZE; + return DEC_Condition_MU; /* Multiple errors */ + } /* decContextStatusToString */ + +/* ------------------------------------------------------------------ */ +/* decContextTestEndian -- test whether DECLITEND is set correctly */ +/* */ +/* quiet is 1 to suppress message; 0 otherwise */ +/* returns 0 if DECLITEND is correct */ +/* 1 if DECLITEND is incorrect and should be 1 */ +/* -1 if DECLITEND is incorrect and should be 0 */ +/* */ +/* A message is displayed if the return value is not 0 and quiet==0. */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +#if 0 /* ICU: Unused function. Anyway, do not call printf(). */ +U_CAPI Int U_EXPORT2 uprv_decContextTestEndian(Flag quiet) { + Int res=0; /* optimist */ + uInt dle=(uInt)DECLITEND; /* unsign */ + if (dle>1) dle=1; /* ensure 0 or 1 */ + + if (LITEND!=DECLITEND) { + const char *adj; + if (!quiet) { + if (LITEND) adj="little"; + else adj="big"; + printf("Warning: DECLITEND is set to %d, but this computer appears to be %s-endian\n", + DECLITEND, adj); + } + res=(Int)LITEND-dle; + } + return res; + } /* decContextTestEndian */ +#endif + +/* ------------------------------------------------------------------ */ +/* decContextTestSavedStatus -- test bits in saved status */ +/* */ +/* oldstatus is the status word to be tested */ +/* mask indicates the bits to be tested (the oldstatus bits that */ +/* correspond to each 1 bit in the mask are tested) */ +/* returns 1 if any of the tested bits are 1, or 0 otherwise */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI uInt U_EXPORT2 uprv_decContextTestSavedStatus(uInt oldstatus, uInt mask) { + return (oldstatus&mask)!=0; + } /* decContextTestSavedStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextTestStatus -- test bits in current status */ +/* */ +/* context is the context structure to be updated */ +/* mask indicates the bits to be tested (the status bits that */ +/* correspond to each 1 bit in the mask are tested) */ +/* returns 1 if any of the tested bits are 1, or 0 otherwise */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI uInt U_EXPORT2 uprv_decContextTestStatus(decContext *context, uInt mask) { + return (context->status&mask)!=0; + } /* decContextTestStatus */ + +/* ------------------------------------------------------------------ */ +/* decContextZeroStatus -- clear all status bits */ +/* */ +/* context is the context structure to be updated */ +/* returns context */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decContext * U_EXPORT2 uprv_decContextZeroStatus(decContext *context) { + context->status=0; + return context; + } /* decContextZeroStatus */ + diff --git a/intl/icu/source/i18n/decContext.h b/intl/icu/source/i18n/decContext.h new file mode 100644 index 0000000000..48324575e3 --- /dev/null +++ b/intl/icu/source/i18n/decContext.h @@ -0,0 +1,271 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* ------------------------------------------------------------------ */ +/* Decimal Context module header */ +/* ------------------------------------------------------------------ */ +/* Copyright (c) IBM Corporation, 2000-2011. All rights reserved. */ +/* */ +/* This software is made available under the terms of the */ +/* ICU License -- ICU 1.8.1 and later. */ +/* */ +/* The description and User's Guide ("The decNumber C Library") for */ +/* this software is called decNumber.pdf. This document is */ +/* available, together with arithmetic and format specifications, */ +/* testcases, and Web links, on the General Decimal Arithmetic page. */ +/* */ +/* Please send comments, suggestions, and corrections to the author: */ +/* mfc@uk.ibm.com */ +/* Mike Cowlishaw, IBM Fellow */ +/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */ +/* ------------------------------------------------------------------ */ + +/* Modified version, for use from within ICU. + * Renamed public functions, to avoid an unwanted export of the + * standard names from the ICU library. + * + * Use ICU's uprv_malloc() and uprv_free() + * + * Revert comment syntax to plain C + * + * Remove a few compiler warnings. + */ +#include "unicode/utypes.h" +#include "putilimp.h" + +/* */ +/* Context variables must always have valid values: */ +/* */ +/* status -- [any bits may be cleared, but not set, by user] */ +/* round -- must be one of the enumerated rounding modes */ +/* */ +/* The following variables are implied for fixed size formats (i.e., */ +/* they are ignored) but should still be set correctly in case used */ +/* with decNumber functions: */ +/* */ +/* clamp -- must be either 0 or 1 */ +/* digits -- must be in the range 1 through 999999999 */ +/* emax -- must be in the range 0 through 999999999 */ +/* emin -- must be in the range 0 through -999999999 */ +/* extended -- must be either 0 or 1 [present only if DECSUBSET] */ +/* traps -- only defined bits may be set */ +/* */ +/* ------------------------------------------------------------------ */ + +#if !defined(DECCONTEXT) + #define DECCONTEXT + #define DECCNAME "decContext" /* Short name */ + #define DECCFULLNAME "Decimal Context Descriptor" /* Verbose name */ + #define DECCAUTHOR "Mike Cowlishaw" /* Who to blame */ + + #if !defined(int32_t) +/* #include */ /* C99 standard integers */ + #endif + #include /* for printf, etc. */ +#ifndef __wasi__ + #include /* for traps */ +#endif + + /* Extended flags setting -- set this to 0 to use only IEEE flags */ + #if !defined(DECEXTFLAG) + #define DECEXTFLAG 1 /* 1=enable extended flags */ + #endif + + /* Conditional code flag -- set this to 0 for best performance */ + #if !defined(DECSUBSET) + #define DECSUBSET 0 /* 1=enable subset arithmetic */ + #endif + + /* Context for operations, with associated constants */ + enum rounding { + DEC_ROUND_CEILING, /* round towards +infinity */ + DEC_ROUND_UP, /* round away from 0 */ + DEC_ROUND_HALF_UP, /* 0.5 rounds up */ + DEC_ROUND_HALF_EVEN, /* 0.5 rounds to nearest even */ + DEC_ROUND_HALF_DOWN, /* 0.5 rounds down */ + DEC_ROUND_DOWN, /* round towards 0 (truncate) */ + DEC_ROUND_FLOOR, /* round towards -infinity */ + DEC_ROUND_05UP, /* round for reround */ + DEC_ROUND_MAX /* enum must be less than this */ + }; + #define DEC_ROUND_DEFAULT DEC_ROUND_HALF_EVEN; + + typedef struct { + int32_t digits; /* working precision */ + int32_t emax; /* maximum positive exponent */ + int32_t emin; /* minimum negative exponent */ + enum rounding round; /* rounding mode */ + uint32_t traps; /* trap-enabler flags */ + uint32_t status; /* status flags */ + uint8_t clamp; /* flag: apply IEEE exponent clamp */ + #if DECSUBSET + uint8_t extended; /* flag: special-values allowed */ + #endif + } decContext; + + /* Maxima and Minima for context settings */ + #define DEC_MAX_DIGITS 999999999 + #define DEC_MIN_DIGITS 1 + #define DEC_MAX_EMAX 999999999 + #define DEC_MIN_EMAX 0 + #define DEC_MAX_EMIN 0 + #define DEC_MIN_EMIN -999999999 + #define DEC_MAX_MATH 999999 /* max emax, etc., for math funcs. */ + + /* Classifications for decimal numbers, aligned with 754 (note that */ + /* 'normal' and 'subnormal' are meaningful only with a decContext */ + /* or a fixed size format). */ + enum decClass { + DEC_CLASS_SNAN, + DEC_CLASS_QNAN, + DEC_CLASS_NEG_INF, + DEC_CLASS_NEG_NORMAL, + DEC_CLASS_NEG_SUBNORMAL, + DEC_CLASS_NEG_ZERO, + DEC_CLASS_POS_ZERO, + DEC_CLASS_POS_SUBNORMAL, + DEC_CLASS_POS_NORMAL, + DEC_CLASS_POS_INF + }; + /* Strings for the decClasses */ + #define DEC_ClassString_SN "sNaN" + #define DEC_ClassString_QN "NaN" + #define DEC_ClassString_NI "-Infinity" + #define DEC_ClassString_NN "-Normal" + #define DEC_ClassString_NS "-Subnormal" + #define DEC_ClassString_NZ "-Zero" + #define DEC_ClassString_PZ "+Zero" + #define DEC_ClassString_PS "+Subnormal" + #define DEC_ClassString_PN "+Normal" + #define DEC_ClassString_PI "+Infinity" + #define DEC_ClassString_UN "Invalid" + + /* Trap-enabler and Status flags (exceptional conditions), and */ + /* their names. The top byte is reserved for internal use */ + #if DECEXTFLAG + /* Extended flags */ + #define DEC_Conversion_syntax 0x00000001 + #define DEC_Division_by_zero 0x00000002 + #define DEC_Division_impossible 0x00000004 + #define DEC_Division_undefined 0x00000008 + #define DEC_Insufficient_storage 0x00000010 /* [when malloc fails] */ + #define DEC_Inexact 0x00000020 + #define DEC_Invalid_context 0x00000040 + #define DEC_Invalid_operation 0x00000080 + #if DECSUBSET + #define DEC_Lost_digits 0x00000100 + #endif + #define DEC_Overflow 0x00000200 + #define DEC_Clamped 0x00000400 + #define DEC_Rounded 0x00000800 + #define DEC_Subnormal 0x00001000 + #define DEC_Underflow 0x00002000 + #else + /* IEEE flags only */ + #define DEC_Conversion_syntax 0x00000010 + #define DEC_Division_by_zero 0x00000002 + #define DEC_Division_impossible 0x00000010 + #define DEC_Division_undefined 0x00000010 + #define DEC_Insufficient_storage 0x00000010 /* [when malloc fails] */ + #define DEC_Inexact 0x00000001 + #define DEC_Invalid_context 0x00000010 + #define DEC_Invalid_operation 0x00000010 + #if DECSUBSET + #define DEC_Lost_digits 0x00000000 + #endif + #define DEC_Overflow 0x00000008 + #define DEC_Clamped 0x00000000 + #define DEC_Rounded 0x00000000 + #define DEC_Subnormal 0x00000000 + #define DEC_Underflow 0x00000004 + #endif + + /* IEEE 754 groupings for the flags */ + /* [DEC_Clamped, DEC_Lost_digits, DEC_Rounded, and DEC_Subnormal */ + /* are not in IEEE 754] */ + #define DEC_IEEE_754_Division_by_zero (DEC_Division_by_zero) + #if DECSUBSET + #define DEC_IEEE_754_Inexact (DEC_Inexact | DEC_Lost_digits) + #else + #define DEC_IEEE_754_Inexact (DEC_Inexact) + #endif + #define DEC_IEEE_754_Invalid_operation (DEC_Conversion_syntax | \ + DEC_Division_impossible | \ + DEC_Division_undefined | \ + DEC_Insufficient_storage | \ + DEC_Invalid_context | \ + DEC_Invalid_operation) + #define DEC_IEEE_754_Overflow (DEC_Overflow) + #define DEC_IEEE_754_Underflow (DEC_Underflow) + + /* flags which are normally errors (result is qNaN, infinite, or 0) */ + #define DEC_Errors (DEC_IEEE_754_Division_by_zero | \ + DEC_IEEE_754_Invalid_operation | \ + DEC_IEEE_754_Overflow | DEC_IEEE_754_Underflow) + /* flags which cause a result to become qNaN */ + #define DEC_NaNs DEC_IEEE_754_Invalid_operation + + /* flags which are normally for information only (finite results) */ + #if DECSUBSET + #define DEC_Information (DEC_Clamped | DEC_Rounded | DEC_Inexact \ + | DEC_Lost_digits) + #else + #define DEC_Information (DEC_Clamped | DEC_Rounded | DEC_Inexact) + #endif + + /* IEEE 854 names (for compatibility with older decNumber versions) */ + #define DEC_IEEE_854_Division_by_zero DEC_IEEE_754_Division_by_zero + #define DEC_IEEE_854_Inexact DEC_IEEE_754_Inexact + #define DEC_IEEE_854_Invalid_operation DEC_IEEE_754_Invalid_operation + #define DEC_IEEE_854_Overflow DEC_IEEE_754_Overflow + #define DEC_IEEE_854_Underflow DEC_IEEE_754_Underflow + + /* Name strings for the exceptional conditions */ + #define DEC_Condition_CS "Conversion syntax" + #define DEC_Condition_DZ "Division by zero" + #define DEC_Condition_DI "Division impossible" + #define DEC_Condition_DU "Division undefined" + #define DEC_Condition_IE "Inexact" + #define DEC_Condition_IS "Insufficient storage" + #define DEC_Condition_IC "Invalid context" + #define DEC_Condition_IO "Invalid operation" + #if DECSUBSET + #define DEC_Condition_LD "Lost digits" + #endif + #define DEC_Condition_OV "Overflow" + #define DEC_Condition_PA "Clamped" + #define DEC_Condition_RO "Rounded" + #define DEC_Condition_SU "Subnormal" + #define DEC_Condition_UN "Underflow" + #define DEC_Condition_ZE "No status" + #define DEC_Condition_MU "Multiple status" + #define DEC_Condition_Length 21 /* length of the longest string, */ + /* including terminator */ + + /* Initialization descriptors, used by decContextDefault */ + #define DEC_INIT_BASE 0 + #define DEC_INIT_DECIMAL32 32 + #define DEC_INIT_DECIMAL64 64 + #define DEC_INIT_DECIMAL128 128 + /* Synonyms */ + #define DEC_INIT_DECSINGLE DEC_INIT_DECIMAL32 + #define DEC_INIT_DECDOUBLE DEC_INIT_DECIMAL64 + #define DEC_INIT_DECQUAD DEC_INIT_DECIMAL128 + + /* decContext routines */ + U_CAPI decContext * U_EXPORT2 uprv_decContextClearStatus(decContext *, uint32_t); + U_CAPI decContext * U_EXPORT2 uprv_decContextDefault(decContext *, int32_t); + U_CAPI enum rounding U_EXPORT2 uprv_decContextGetRounding(decContext *); + U_CAPI uint32_t U_EXPORT2 uprv_decContextGetStatus(decContext *); + U_CAPI decContext * U_EXPORT2 uprv_decContextRestoreStatus(decContext *, uint32_t, uint32_t); + U_CAPI uint32_t U_EXPORT2 uprv_decContextSaveStatus(decContext *, uint32_t); + U_CAPI decContext * U_EXPORT2 uprv_decContextSetRounding(decContext *, enum rounding); + U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatus(decContext *, uint32_t); + U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromString(decContext *, const char *); + U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromStringQuiet(decContext *, const char *); + U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusQuiet(decContext *, uint32_t); + U_CAPI const char * U_EXPORT2 uprv_decContextStatusToString(const decContext *); + U_CAPI uint32_t U_EXPORT2 uprv_decContextTestSavedStatus(uint32_t, uint32_t); + U_CAPI uint32_t U_EXPORT2 uprv_decContextTestStatus(decContext *, uint32_t); + U_CAPI decContext * U_EXPORT2 uprv_decContextZeroStatus(decContext *); + +#endif diff --git a/intl/icu/source/i18n/decNumber.cpp b/intl/icu/source/i18n/decNumber.cpp new file mode 100644 index 0000000000..42da36dc4b --- /dev/null +++ b/intl/icu/source/i18n/decNumber.cpp @@ -0,0 +1,8190 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* ------------------------------------------------------------------ */ +/* Decimal Number arithmetic module */ +/* ------------------------------------------------------------------ */ +/* Copyright (c) IBM Corporation, 2000-2014. All rights reserved. */ +/* */ +/* This software is made available under the terms of the */ +/* ICU License -- ICU 1.8.1 and later. */ +/* */ +/* The description and User's Guide ("The decNumber C Library") for */ +/* this software is called decNumber.pdf. This document is */ +/* available, together with arithmetic and format specifications, */ +/* testcases, and Web links, on the General Decimal Arithmetic page. */ +/* */ +/* Please send comments, suggestions, and corrections to the author: */ +/* mfc@uk.ibm.com */ +/* Mike Cowlishaw, IBM Fellow */ +/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */ +/* ------------------------------------------------------------------ */ + +/* Modified version, for use from within ICU. + * Renamed public functions, to avoid an unwanted export of the + * standard names from the ICU library. + * + * Use ICU's uprv_malloc() and uprv_free() + * + * Revert comment syntax to plain C + * + * Remove a few compiler warnings. + */ + +/* This module comprises the routines for arbitrary-precision General */ +/* Decimal Arithmetic as defined in the specification which may be */ +/* found on the General Decimal Arithmetic pages. It implements both */ +/* the full ('extended') arithmetic and the simpler ('subset') */ +/* arithmetic. */ +/* */ +/* Usage notes: */ +/* */ +/* 1. This code is ANSI C89 except: */ +/* */ +/* a) C99 line comments (double forward slash) are used. (Most C */ +/* compilers accept these. If yours does not, a simple script */ +/* can be used to convert them to ANSI C comments.) */ +/* */ +/* b) Types from C99 stdint.h are used. If you do not have this */ +/* header file, see the User's Guide section of the decNumber */ +/* documentation; this lists the necessary definitions. */ +/* */ +/* c) If DECDPUN>4 or DECUSE64=1, the C99 64-bit int64_t and */ +/* uint64_t types may be used. To avoid these, set DECUSE64=0 */ +/* and DECDPUN<=4 (see documentation). */ +/* */ +/* The code also conforms to C99 restrictions; in particular, */ +/* strict aliasing rules are observed. */ +/* */ +/* 2. The decNumber format which this library uses is optimized for */ +/* efficient processing of relatively short numbers; in particular */ +/* it allows the use of fixed sized structures and minimizes copy */ +/* and move operations. It does, however, support arbitrary */ +/* precision (up to 999,999,999 digits) and arbitrary exponent */ +/* range (Emax in the range 0 through 999,999,999 and Emin in the */ +/* range -999,999,999 through 0). Mathematical functions (for */ +/* example decNumberExp) as identified below are restricted more */ +/* tightly: digits, emax, and -emin in the context must be <= */ +/* DEC_MAX_MATH (999999), and their operand(s) must be within */ +/* these bounds. */ +/* */ +/* 3. Logical functions are further restricted; their operands must */ +/* be finite, positive, have an exponent of zero, and all digits */ +/* must be either 0 or 1. The result will only contain digits */ +/* which are 0 or 1 (and will have exponent=0 and a sign of 0). */ +/* */ +/* 4. Operands to operator functions are never modified unless they */ +/* are also specified to be the result number (which is always */ +/* permitted). Other than that case, operands must not overlap. */ +/* */ +/* 5. Error handling: the type of the error is ORed into the status */ +/* flags in the current context (decContext structure). The */ +/* SIGFPE signal is then raised if the corresponding trap-enabler */ +/* flag in the decContext is set (is 1). */ +/* */ +/* It is the responsibility of the caller to clear the status */ +/* flags as required. */ +/* */ +/* The result of any routine which returns a number will always */ +/* be a valid number (which may be a special value, such as an */ +/* Infinity or NaN). */ +/* */ +/* 6. The decNumber format is not an exchangeable concrete */ +/* representation as it comprises fields which may be machine- */ +/* dependent (packed or unpacked, or special length, for example). */ +/* Canonical conversions to and from strings are provided; other */ +/* conversions are available in separate modules. */ +/* */ +/* 7. Normally, input operands are assumed to be valid. Set DECCHECK */ +/* to 1 for extended operand checking (including nullptr operands). */ +/* Results are undefined if a badly-formed structure (or a nullptr */ +/* pointer to a structure) is provided, though with DECCHECK */ +/* enabled the operator routines are protected against exceptions. */ +/* (Except if the result pointer is nullptr, which is unrecoverable.) */ +/* */ +/* However, the routines will never cause exceptions if they are */ +/* given well-formed operands, even if the value of the operands */ +/* is inappropriate for the operation and DECCHECK is not set. */ +/* (Except for SIGFPE, as and where documented.) */ +/* */ +/* 8. Subset arithmetic is available only if DECSUBSET is set to 1. */ +/* ------------------------------------------------------------------ */ +/* Implementation notes for maintenance of this module: */ +/* */ +/* 1. Storage leak protection: Routines which use malloc are not */ +/* permitted to use return for fastpath or error exits (i.e., */ +/* they follow strict structured programming conventions). */ +/* Instead they have a do{}while(0); construct surrounding the */ +/* code which is protected -- break may be used to exit this. */ +/* Other routines can safely use the return statement inline. */ +/* */ +/* Storage leak accounting can be enabled using DECALLOC. */ +/* */ +/* 2. All loops use the for(;;) construct. Any do construct does */ +/* not loop; it is for allocation protection as just described. */ +/* */ +/* 3. Setting status in the context must always be the very last */ +/* action in a routine, as non-0 status may raise a trap and hence */ +/* the call to set status may not return (if the handler uses long */ +/* jump). Therefore all cleanup must be done first. In general, */ +/* to achieve this status is accumulated and is only applied just */ +/* before return by calling decContextSetStatus (via decStatus). */ +/* */ +/* Routines which allocate storage cannot, in general, use the */ +/* 'top level' routines which could cause a non-returning */ +/* transfer of control. The decXxxxOp routines are safe (do not */ +/* call decStatus even if traps are set in the context) and should */ +/* be used instead (they are also a little faster). */ +/* */ +/* 4. Exponent checking is minimized by allowing the exponent to */ +/* grow outside its limits during calculations, provided that */ +/* the decFinalize function is called later. Multiplication and */ +/* division, and intermediate calculations in exponentiation, */ +/* require more careful checks because of the risk of 31-bit */ +/* overflow (the most negative valid exponent is -1999999997, for */ +/* a 999999999-digit number with adjusted exponent of -999999999). */ +/* */ +/* 5. Rounding is deferred until finalization of results, with any */ +/* 'off to the right' data being represented as a single digit */ +/* residue (in the range -1 through 9). This avoids any double- */ +/* rounding when more than one shortening takes place (for */ +/* example, when a result is subnormal). */ +/* */ +/* 6. The digits count is allowed to rise to a multiple of DECDPUN */ +/* during many operations, so whole Units are handled and exact */ +/* accounting of digits is not needed. The correct digits value */ +/* is found by decGetDigits, which accounts for leading zeros. */ +/* This must be called before any rounding if the number of digits */ +/* is not known exactly. */ +/* */ +/* 7. The multiply-by-reciprocal 'trick' is used for partitioning */ +/* numbers up to four digits, using appropriate constants. This */ +/* is not useful for longer numbers because overflow of 32 bits */ +/* would lead to 4 multiplies, which is almost as expensive as */ +/* a divide (unless a floating-point or 64-bit multiply is */ +/* assumed to be available). */ +/* */ +/* 8. Unusual abbreviations that may be used in the commentary: */ +/* lhs -- left hand side (operand, of an operation) */ +/* lsd -- least significant digit (of coefficient) */ +/* lsu -- least significant Unit (of coefficient) */ +/* msd -- most significant digit (of coefficient) */ +/* msi -- most significant item (in an array) */ +/* msu -- most significant Unit (of coefficient) */ +/* rhs -- right hand side (operand, of an operation) */ +/* +ve -- positive */ +/* -ve -- negative */ +/* ** -- raise to the power */ +/* ------------------------------------------------------------------ */ + +#include /* for malloc, free, etc. */ +/* #include */ /* for printf [if needed] */ +#include /* for strcpy */ +#include /* for lower */ +#include "cmemory.h" /* for uprv_malloc, etc., in ICU */ +#include "decNumber.h" /* base number library */ +#include "decNumberLocal.h" /* decNumber local types, etc. */ +#include "uassert.h" + +/* Constants */ +/* Public lookup table used by the D2U macro */ +static const uByte d2utable[DECMAXD2U+1]=D2UTABLE; + +#define DECVERB 1 /* set to 1 for verbose DECCHECK */ +#define powers DECPOWERS /* old internal name */ + +/* Local constants */ +#define DIVIDE 0x80 /* Divide operators */ +#define REMAINDER 0x40 /* .. */ +#define DIVIDEINT 0x20 /* .. */ +#define REMNEAR 0x10 /* .. */ +#define COMPARE 0x01 /* Compare operators */ +#define COMPMAX 0x02 /* .. */ +#define COMPMIN 0x03 /* .. */ +#define COMPTOTAL 0x04 /* .. */ +#define COMPNAN 0x05 /* .. [NaN processing] */ +#define COMPSIG 0x06 /* .. [signaling COMPARE] */ +#define COMPMAXMAG 0x07 /* .. */ +#define COMPMINMAG 0x08 /* .. */ + +#define DEC_sNaN 0x40000000 /* local status: sNaN signal */ +#define BADINT (Int)0x80000000 /* most-negative Int; error indicator */ +/* Next two indicate an integer >= 10**6, and its parity (bottom bit) */ +#define BIGEVEN (Int)0x80000002 +#define BIGODD (Int)0x80000003 + +static const Unit uarrone[1]={1}; /* Unit array of 1, used for incrementing */ + +/* ------------------------------------------------------------------ */ +/* round-for-reround digits */ +/* ------------------------------------------------------------------ */ +#if 0 +static const uByte DECSTICKYTAB[10]={1,1,2,3,4,6,6,7,8,9}; /* used if sticky */ +#endif + +/* ------------------------------------------------------------------ */ +/* Powers of ten (powers[n]==10**n, 0<=n<=9) */ +/* ------------------------------------------------------------------ */ +static const uInt DECPOWERS[10]={1, 10, 100, 1000, 10000, 100000, 1000000, + 10000000, 100000000, 1000000000}; + + +/* Granularity-dependent code */ +#if DECDPUN<=4 + #define eInt Int /* extended integer */ + #define ueInt uInt /* unsigned extended integer */ + /* Constant multipliers for divide-by-power-of five using reciprocal */ + /* multiply, after removing powers of 2 by shifting, and final shift */ + /* of 17 [we only need up to **4] */ + static const uInt multies[]={131073, 26215, 5243, 1049, 210}; + /* QUOT10 -- macro to return the quotient of unit u divided by 10**n */ + #define QUOT10(u, n) ((((uInt)(u)>>(n))*multies[n])>>17) +#else + /* For DECDPUN>4 non-ANSI-89 64-bit types are needed. */ + #if !DECUSE64 + #error decNumber.c: DECUSE64 must be 1 when DECDPUN>4 + #endif + #define eInt Long /* extended integer */ + #define ueInt uLong /* unsigned extended integer */ +#endif + +/* Local routines */ +static decNumber * decAddOp(decNumber *, const decNumber *, const decNumber *, + decContext *, uByte, uInt *); +static Flag decBiStr(const char *, const char *, const char *); +static uInt decCheckMath(const decNumber *, decContext *, uInt *); +static void decApplyRound(decNumber *, decContext *, Int, uInt *); +static Int decCompare(const decNumber *lhs, const decNumber *rhs, Flag); +static decNumber * decCompareOp(decNumber *, const decNumber *, + const decNumber *, decContext *, + Flag, uInt *); +static void decCopyFit(decNumber *, const decNumber *, decContext *, + Int *, uInt *); +static decNumber * decDecap(decNumber *, Int); +static decNumber * decDivideOp(decNumber *, const decNumber *, + const decNumber *, decContext *, Flag, uInt *); +static decNumber * decExpOp(decNumber *, const decNumber *, + decContext *, uInt *); +static void decFinalize(decNumber *, decContext *, Int *, uInt *); +static Int decGetDigits(Unit *, Int); +static Int decGetInt(const decNumber *); +static decNumber * decLnOp(decNumber *, const decNumber *, + decContext *, uInt *); +static decNumber * decMultiplyOp(decNumber *, const decNumber *, + const decNumber *, decContext *, + uInt *); +static decNumber * decNaNs(decNumber *, const decNumber *, + const decNumber *, decContext *, uInt *); +static decNumber * decQuantizeOp(decNumber *, const decNumber *, + const decNumber *, decContext *, Flag, + uInt *); +static void decReverse(Unit *, Unit *); +static void decSetCoeff(decNumber *, decContext *, const Unit *, + Int, Int *, uInt *); +static void decSetMaxValue(decNumber *, decContext *); +static void decSetOverflow(decNumber *, decContext *, uInt *); +static void decSetSubnormal(decNumber *, decContext *, Int *, uInt *); +static Int decShiftToLeast(Unit *, Int, Int); +static Int decShiftToMost(Unit *, Int, Int); +static void decStatus(decNumber *, uInt, decContext *); +static void decToString(const decNumber *, char[], Flag); +static decNumber * decTrim(decNumber *, decContext *, Flag, Flag, Int *); +static Int decUnitAddSub(const Unit *, Int, const Unit *, Int, Int, + Unit *, Int); +static Int decUnitCompare(const Unit *, Int, const Unit *, Int, Int); + +#if !DECSUBSET +/* decFinish == decFinalize when no subset arithmetic needed */ +#define decFinish(a,b,c,d) decFinalize(a,b,c,d) +#else +static void decFinish(decNumber *, decContext *, Int *, uInt *); +static decNumber * decRoundOperand(const decNumber *, decContext *, uInt *); +#endif + +/* Local macros */ +/* masked special-values bits */ +#define SPECIALARG (rhs->bits & DECSPECIAL) +#define SPECIALARGS ((lhs->bits | rhs->bits) & DECSPECIAL) + +/* For use in ICU */ +#define malloc(a) uprv_malloc(a) +#define free(a) uprv_free(a) + +/* Diagnostic macros, etc. */ +#if DECALLOC +/* Handle malloc/free accounting. If enabled, our accountable routines */ +/* are used; otherwise the code just goes straight to the system malloc */ +/* and free routines. */ +#define malloc(a) decMalloc(a) +#define free(a) decFree(a) +#define DECFENCE 0x5a /* corruption detector */ +/* 'Our' malloc and free: */ +static void *decMalloc(size_t); +static void decFree(void *); +uInt decAllocBytes=0; /* count of bytes allocated */ +/* Note that DECALLOC code only checks for storage buffer overflow. */ +/* To check for memory leaks, the decAllocBytes variable must be */ +/* checked to be 0 at appropriate times (e.g., after the test */ +/* harness completes a set of tests). This checking may be unreliable */ +/* if the testing is done in a multi-thread environment. */ +#endif + +#if DECCHECK +/* Optional checking routines. Enabling these means that decNumber */ +/* and decContext operands to operator routines are checked for */ +/* correctness. This roughly doubles the execution time of the */ +/* fastest routines (and adds 600+ bytes), so should not normally be */ +/* used in 'production'. */ +/* decCheckInexact is used to check that inexact results have a full */ +/* complement of digits (where appropriate -- this is not the case */ +/* for Quantize, for example) */ +#define DECUNRESU ((decNumber *)(void *)0xffffffff) +#define DECUNUSED ((const decNumber *)(void *)0xffffffff) +#define DECUNCONT ((decContext *)(void *)(0xffffffff)) +static Flag decCheckOperands(decNumber *, const decNumber *, + const decNumber *, decContext *); +static Flag decCheckNumber(const decNumber *); +static void decCheckInexact(const decNumber *, decContext *); +#endif + +#if DECTRACE || DECCHECK +/* Optional trace/debugging routines (may or may not be used) */ +void decNumberShow(const decNumber *); /* displays the components of a number */ +static void decDumpAr(char, const Unit *, Int); +#endif + +/* ================================================================== */ +/* Conversions */ +/* ================================================================== */ + +/* ------------------------------------------------------------------ */ +/* from-int32 -- conversion from Int or uInt */ +/* */ +/* dn is the decNumber to receive the integer */ +/* in or uin is the integer to be converted */ +/* returns dn */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromInt32(decNumber *dn, Int in) { + uInt unsig; + if (in>=0) unsig=in; + else { /* negative (possibly BADINT) */ + if (in==BADINT) unsig=(uInt)1073741824*2; /* special case */ + else unsig=-in; /* invert */ + } + /* in is now positive */ + uprv_decNumberFromUInt32(dn, unsig); + if (in<0) dn->bits=DECNEG; /* sign needed */ + return dn; + } /* decNumberFromInt32 */ + +U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromUInt32(decNumber *dn, uInt uin) { + Unit *up; /* work pointer */ + uprv_decNumberZero(dn); /* clean */ + if (uin==0) return dn; /* [or decGetDigits bad call] */ + for (up=dn->lsu; uin>0; up++) { + *up=(Unit)(uin%(DECDPUNMAX+1)); + uin=uin/(DECDPUNMAX+1); + } + dn->digits=decGetDigits(dn->lsu, static_cast(up - dn->lsu)); + return dn; + } /* decNumberFromUInt32 */ + +/* ------------------------------------------------------------------ */ +/* to-int32 -- conversion to Int or uInt */ +/* */ +/* dn is the decNumber to convert */ +/* set is the context for reporting errors */ +/* returns the converted decNumber, or 0 if Invalid is set */ +/* */ +/* Invalid is set if the decNumber does not have exponent==0 or if */ +/* it is a NaN, Infinite, or out-of-range. */ +/* ------------------------------------------------------------------ */ +U_CAPI Int U_EXPORT2 uprv_decNumberToInt32(const decNumber *dn, decContext *set) { + #if DECCHECK + if (decCheckOperands(DECUNRESU, DECUNUSED, dn, set)) return 0; + #endif + + /* special or too many digits, or bad exponent */ + if (dn->bits&DECSPECIAL || dn->digits>10 || dn->exponent!=0) ; /* bad */ + else { /* is a finite integer with 10 or fewer digits */ + Int d; /* work */ + const Unit *up; /* .. */ + uInt hi=0, lo; /* .. */ + up=dn->lsu; /* -> lsu */ + lo=*up; /* get 1 to 9 digits */ + #if DECDPUN>1 /* split to higher */ + hi=lo/10; + lo=lo%10; + #endif + up++; + /* collect remaining Units, if any, into hi */ + for (d=DECDPUN; ddigits; up++, d+=DECDPUN) hi+=*up*powers[d-1]; + /* now low has the lsd, hi the remainder */ + if (hi>214748364 || (hi==214748364 && lo>7)) { /* out of range? */ + /* most-negative is a reprieve */ + if (dn->bits&DECNEG && hi==214748364 && lo==8) return 0x80000000; + /* bad -- drop through */ + } + else { /* in-range always */ + Int i=X10(hi)+lo; + if (dn->bits&DECNEG) return -i; + return i; + } + } /* integer */ + uprv_decContextSetStatus(set, DEC_Invalid_operation); /* [may not return] */ + return 0; + } /* decNumberToInt32 */ + +U_CAPI uInt U_EXPORT2 uprv_decNumberToUInt32(const decNumber *dn, decContext *set) { + #if DECCHECK + if (decCheckOperands(DECUNRESU, DECUNUSED, dn, set)) return 0; + #endif + /* special or too many digits, or bad exponent, or negative (<0) */ + if (dn->bits&DECSPECIAL || dn->digits>10 || dn->exponent!=0 + || (dn->bits&DECNEG && !ISZERO(dn))); /* bad */ + else { /* is a finite integer with 10 or fewer digits */ + Int d; /* work */ + const Unit *up; /* .. */ + uInt hi=0, lo; /* .. */ + up=dn->lsu; /* -> lsu */ + lo=*up; /* get 1 to 9 digits */ + #if DECDPUN>1 /* split to higher */ + hi=lo/10; + lo=lo%10; + #endif + up++; + /* collect remaining Units, if any, into hi */ + for (d=DECDPUN; ddigits; up++, d+=DECDPUN) hi+=*up*powers[d-1]; + + /* now low has the lsd, hi the remainder */ + if (hi>429496729 || (hi==429496729 && lo>5)) ; /* no reprieve possible */ + else return X10(hi)+lo; + } /* integer */ + uprv_decContextSetStatus(set, DEC_Invalid_operation); /* [may not return] */ + return 0; + } /* decNumberToUInt32 */ + +/* ------------------------------------------------------------------ */ +/* to-scientific-string -- conversion to numeric string */ +/* to-engineering-string -- conversion to numeric string */ +/* */ +/* decNumberToString(dn, string); */ +/* decNumberToEngString(dn, string); */ +/* */ +/* dn is the decNumber to convert */ +/* string is the string where the result will be laid out */ +/* */ +/* string must be at least dn->digits+14 characters long */ +/* */ +/* No error is possible, and no status can be set. */ +/* ------------------------------------------------------------------ */ +U_CAPI char * U_EXPORT2 uprv_decNumberToString(const decNumber *dn, char *string){ + decToString(dn, string, 0); + return string; + } /* DecNumberToString */ + +U_CAPI char * U_EXPORT2 uprv_decNumberToEngString(const decNumber *dn, char *string){ + decToString(dn, string, 1); + return string; + } /* DecNumberToEngString */ + +/* ------------------------------------------------------------------ */ +/* to-number -- conversion from numeric string */ +/* */ +/* decNumberFromString -- convert string to decNumber */ +/* dn -- the number structure to fill */ +/* chars[] -- the string to convert ('\0' terminated) */ +/* set -- the context used for processing any error, */ +/* determining the maximum precision available */ +/* (set.digits), determining the maximum and minimum */ +/* exponent (set.emax and set.emin), determining if */ +/* extended values are allowed, and checking the */ +/* rounding mode if overflow occurs or rounding is */ +/* needed. */ +/* */ +/* The length of the coefficient and the size of the exponent are */ +/* checked by this routine, so the correct error (Underflow or */ +/* Overflow) can be reported or rounding applied, as necessary. */ +/* */ +/* If bad syntax is detected, the result will be a quiet NaN. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromString(decNumber *dn, const char chars[], + decContext *set) { + Int exponent=0; /* working exponent [assume 0] */ + uByte bits=0; /* working flags [assume +ve] */ + Unit *res; /* where result will be built */ + Unit resbuff[SD2U(DECBUFFER+9)];/* local buffer in case need temporary */ + /* [+9 allows for ln() constants] */ + Unit *allocres=nullptr; /* -> allocated result, iff allocated */ + Int d=0; /* count of digits found in decimal part */ + const char *dotchar=nullptr; /* where dot was found */ + const char *cfirst=chars; /* -> first character of decimal part */ + const char *last=nullptr; /* -> last digit of decimal part */ + const char *c; /* work */ + Unit *up; /* .. */ + #if DECDPUN>1 + Int cut, out; /* .. */ + #endif + Int residue; /* rounding residue */ + uInt status=0; /* error code */ + + #if DECCHECK + if (decCheckOperands(DECUNRESU, DECUNUSED, DECUNUSED, set)) + return uprv_decNumberZero(dn); + #endif + + do { /* status & malloc protection */ + for (c=chars;; c++) { /* -> input character */ + if (*c>='0' && *c<='9') { /* test for Arabic digit */ + last=c; + d++; /* count of real digits */ + continue; /* still in decimal part */ + } + if (*c=='.' && dotchar==nullptr) { /* first '.' */ + dotchar=c; /* record offset into decimal part */ + if (c==cfirst) cfirst++; /* first digit must follow */ + continue;} + if (c==chars) { /* first in string... */ + if (*c=='-') { /* valid - sign */ + cfirst++; + bits=DECNEG; + continue;} + if (*c=='+') { /* valid + sign */ + cfirst++; + continue;} + } + /* *c is not a digit, or a valid +, -, or '.' */ + break; + } /* c */ + + if (last==nullptr) { /* no digits yet */ + status=DEC_Conversion_syntax;/* assume the worst */ + if (*c=='\0') break; /* and no more to come... */ + #if DECSUBSET + /* if subset then infinities and NaNs are not allowed */ + if (!set->extended) break; /* hopeless */ + #endif + /* Infinities and NaNs are possible, here */ + if (dotchar!=nullptr) break; /* .. unless had a dot */ + uprv_decNumberZero(dn); /* be optimistic */ + if (decBiStr(c, "infinity", "INFINITY") + || decBiStr(c, "inf", "INF")) { + dn->bits=bits | DECINF; + status=0; /* is OK */ + break; /* all done */ + } + /* a NaN expected */ + /* 2003.09.10 NaNs are now permitted to have a sign */ + dn->bits=bits | DECNAN; /* assume simple NaN */ + if (*c=='s' || *c=='S') { /* looks like an sNaN */ + c++; + dn->bits=bits | DECSNAN; + } + if (*c!='n' && *c!='N') break; /* check caseless "NaN" */ + c++; + if (*c!='a' && *c!='A') break; /* .. */ + c++; + if (*c!='n' && *c!='N') break; /* .. */ + c++; + /* now either nothing, or nnnn payload, expected */ + /* -> start of integer and skip leading 0s [including plain 0] */ + for (cfirst=c; *cfirst=='0';) cfirst++; + if (*cfirst=='\0') { /* "NaN" or "sNaN", maybe with all 0s */ + status=0; /* it's good */ + break; /* .. */ + } + /* something other than 0s; setup last and d as usual [no dots] */ + for (c=cfirst;; c++, d++) { + if (*c<'0' || *c>'9') break; /* test for Arabic digit */ + last=c; + } + if (*c!='\0') break; /* not all digits */ + if (d>set->digits-1) { + /* [NB: payload in a decNumber can be full length unless */ + /* clamped, in which case can only be digits-1] */ + if (set->clamp) break; + if (d>set->digits) break; + } /* too many digits? */ + /* good; drop through to convert the integer to coefficient */ + status=0; /* syntax is OK */ + bits=dn->bits; /* for copy-back */ + } /* last==nullptr */ + + else if (*c!='\0') { /* more to process... */ + /* had some digits; exponent is only valid sequence now */ + Flag nege; /* 1=negative exponent */ + const char *firstexp; /* -> first significant exponent digit */ + status=DEC_Conversion_syntax;/* assume the worst */ + if (*c!='e' && *c!='E') break; + /* Found 'e' or 'E' -- now process explicit exponent */ + /* 1998.07.11: sign no longer required */ + nege=0; + c++; /* to (possible) sign */ + if (*c=='-') {nege=1; c++;} + else if (*c=='+') c++; + if (*c=='\0') break; + + for (; *c=='0' && *(c+1)!='\0';) c++; /* strip insignificant zeros */ + firstexp=c; /* save exponent digit place */ + uInt uexponent = 0; /* Avoid undefined behavior on signed int overflow */ + for (; ;c++) { + if (*c<'0' || *c>'9') break; /* not a digit */ + uexponent=X10(uexponent)+(uInt)*c-(uInt)'0'; + } /* c */ + exponent = (Int)uexponent; + /* if not now on a '\0', *c must not be a digit */ + if (*c!='\0') break; + + /* (this next test must be after the syntax checks) */ + /* if it was too long the exponent may have wrapped, so check */ + /* carefully and set it to a certain overflow if wrap possible */ + if (c>=firstexp+9+1) { + if (c>firstexp+9+1 || *firstexp>'1') exponent=DECNUMMAXE*2; + /* [up to 1999999999 is OK, for example 1E-1000000998] */ + } + if (nege) exponent=-exponent; /* was negative */ + status=0; /* is OK */ + } /* stuff after digits */ + + /* Here when whole string has been inspected; syntax is good */ + /* cfirst->first digit (never dot), last->last digit (ditto) */ + + /* strip leading zeros/dot [leave final 0 if all 0's] */ + if (*cfirst=='0') { /* [cfirst has stepped over .] */ + for (c=cfirst; cextended) { + uprv_decNumberZero(dn); /* clean result */ + break; /* [could be return] */ + } + #endif + } /* at least one leading 0 */ + + /* Handle decimal point... */ + if (dotchar!=nullptr && dotchar(last-dotchar); /* adjust exponent */ + /* [we can now ignore the .] */ + + /* OK, the digits string is good. Assemble in the decNumber, or in */ + /* a temporary units array if rounding is needed */ + if (d<=set->digits) res=dn->lsu; /* fits into supplied decNumber */ + else { /* rounding needed */ + Int needbytes=D2U(d)*sizeof(Unit);/* bytes needed */ + res=resbuff; /* assume use local buffer */ + if (needbytes>(Int)sizeof(resbuff)) { /* too big for local */ + allocres=(Unit *)malloc(needbytes); + if (allocres==nullptr) {status|=DEC_Insufficient_storage; break;} + res=allocres; + } + } + /* res now -> number lsu, buffer, or allocated storage for Unit array */ + + /* Place the coefficient into the selected Unit array */ + /* [this is often 70% of the cost of this function when DECDPUN>1] */ + #if DECDPUN>1 + out=0; /* accumulator */ + up=res+D2U(d)-1; /* -> msu */ + cut=d-(up-res)*DECDPUN; /* digits in top unit */ + for (c=cfirst;; c++) { /* along the digits */ + if (*c=='.') continue; /* ignore '.' [don't decrement cut] */ + out=X10(out)+(Int)*c-(Int)'0'; + if (c==last) break; /* done [never get to trailing '.'] */ + cut--; + if (cut>0) continue; /* more for this unit */ + *up=(Unit)out; /* write unit */ + up--; /* prepare for unit below.. */ + cut=DECDPUN; /* .. */ + out=0; /* .. */ + } /* c */ + *up=(Unit)out; /* write lsu */ + + #else + /* DECDPUN==1 */ + up=res; /* -> lsu */ + for (c=last; c>=cfirst; c--) { /* over each character, from least */ + if (*c=='.') continue; /* ignore . [don't step up] */ + *up=(Unit)((Int)*c-(Int)'0'); + up++; + } /* c */ + #endif + + dn->bits=bits; + dn->exponent=exponent; + dn->digits=d; + + /* if not in number (too long) shorten into the number */ + if (d>set->digits) { + residue=0; + decSetCoeff(dn, set, res, d, &residue, &status); + /* always check for overflow or subnormal and round as needed */ + decFinalize(dn, set, &residue, &status); + } + else { /* no rounding, but may still have overflow or subnormal */ + /* [these tests are just for performance; finalize repeats them] */ + if ((dn->exponent-1emin-dn->digits) + || (dn->exponent-1>set->emax-set->digits)) { + residue=0; + decFinalize(dn, set, &residue, &status); + } + } + /* decNumberShow(dn); */ + } while(0); /* [for break] */ + + if (allocres!=nullptr) free(allocres); /* drop any storage used */ + if (status!=0) decStatus(dn, status, set); + return dn; + } /* decNumberFromString */ + +/* ================================================================== */ +/* Operators */ +/* ================================================================== */ + +/* ------------------------------------------------------------------ */ +/* decNumberAbs -- absolute value operator */ +/* */ +/* This computes C = abs(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* See also decNumberCopyAbs for a quiet bitwise version of this. */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* This has the same effect as decNumberPlus unless A is negative, */ +/* in which case it has the same effect as decNumberMinus. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberAbs(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dzero; /* for 0 */ + uInt status=0; /* accumulator */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + uprv_decNumberZero(&dzero); /* set 0 */ + dzero.exponent=rhs->exponent; /* [no coefficient expansion] */ + decAddOp(res, &dzero, rhs, set, (uByte)(rhs->bits & DECNEG), &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberAbs */ + +/* ------------------------------------------------------------------ */ +/* decNumberAdd -- add two Numbers */ +/* */ +/* This computes C = A + B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X+X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* This just calls the routine shared with Subtract */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberAdd(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decAddOp(res, lhs, rhs, set, 0, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberAdd */ + +/* ------------------------------------------------------------------ */ +/* decNumberAnd -- AND two Numbers, digitwise */ +/* */ +/* This computes C = A & B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X&X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context (used for result length and error report) */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Logical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberAnd(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + const Unit *ua, *ub; /* -> operands */ + const Unit *msua, *msub; /* -> operand msus */ + Unit *uc, *msuc; /* -> result and its msu */ + Int msudigs; /* digits in res msu */ + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + if (lhs->exponent!=0 || decNumberIsSpecial(lhs) || decNumberIsNegative(lhs) + || rhs->exponent!=0 || decNumberIsSpecial(rhs) || decNumberIsNegative(rhs)) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + + /* operands are valid */ + ua=lhs->lsu; /* bottom-up */ + ub=rhs->lsu; /* .. */ + uc=res->lsu; /* .. */ + msua=ua+D2U(lhs->digits)-1; /* -> msu of lhs */ + msub=ub+D2U(rhs->digits)-1; /* -> msu of rhs */ + msuc=uc+D2U(set->digits)-1; /* -> msu of result */ + msudigs=MSUDIGITS(set->digits); /* [faster than remainder] */ + for (; uc<=msuc; ua++, ub++, uc++) { /* Unit loop */ + Unit a, b; /* extract units */ + if (ua>msua) a=0; + else a=*ua; + if (ub>msub) b=0; + else b=*ub; + *uc=0; /* can now write back */ + if (a|b) { /* maybe 1 bits to examine */ + Int i, j; + *uc=0; /* can now write back */ + /* This loop could be unrolled and/or use BIN2BCD tables */ + for (i=0; i1) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + if (uc==msuc && i==msudigs-1) break; /* just did final digit */ + } /* each digit */ + } /* both OK */ + } /* each unit */ + /* [here uc-1 is the msu of the result] */ + res->digits=decGetDigits(res->lsu, static_cast(uc - res->lsu)); + res->exponent=0; /* integer */ + res->bits=0; /* sign=0 */ + return res; /* [no status to set] */ + } /* decNumberAnd */ + +/* ------------------------------------------------------------------ */ +/* decNumberCompare -- compare two Numbers */ +/* */ +/* This computes C = A ? B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for one digit (or NaN). */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompare(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPARE, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberCompare */ + +/* ------------------------------------------------------------------ */ +/* decNumberCompareSignal -- compare, signalling on all NaNs */ +/* */ +/* This computes C = A ? B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for one digit (or NaN). */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareSignal(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPSIG, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberCompareSignal */ + +/* ------------------------------------------------------------------ */ +/* decNumberCompareTotal -- compare two Numbers, using total ordering */ +/* */ +/* This computes C = A ? B, under total ordering */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for one digit; the result will always be one of */ +/* -1, 0, or 1. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotal(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPTOTAL, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberCompareTotal */ + +/* ------------------------------------------------------------------ */ +/* decNumberCompareTotalMag -- compare, total ordering of magnitudes */ +/* */ +/* This computes C = |A| ? |B|, under total ordering */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for one digit; the result will always be one of */ +/* -1, 0, or 1. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotalMag(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + uInt needbytes; /* for space calculations */ + decNumber bufa[D2N(DECBUFFER+1)];/* +1 in case DECBUFFER=0 */ + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber bufb[D2N(DECBUFFER+1)]; + decNumber *allocbufb=nullptr; /* -> allocated bufb, iff allocated */ + decNumber *a, *b; /* temporary pointers */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + /* if either is negative, take a copy and absolute */ + if (decNumberIsNegative(lhs)) { /* lhs<0 */ + a=bufa; + needbytes=sizeof(decNumber)+(D2U(lhs->digits)-1)*sizeof(Unit); + if (needbytes>sizeof(bufa)) { /* need malloc space */ + allocbufa=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + a=allocbufa; /* use the allocated space */ + } + uprv_decNumberCopy(a, lhs); /* copy content */ + a->bits&=~DECNEG; /* .. and clear the sign */ + lhs=a; /* use copy from here on */ + } + if (decNumberIsNegative(rhs)) { /* rhs<0 */ + b=bufb; + needbytes=sizeof(decNumber)+(D2U(rhs->digits)-1)*sizeof(Unit); + if (needbytes>sizeof(bufb)) { /* need malloc space */ + allocbufb=(decNumber *)malloc(needbytes); + if (allocbufb==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + b=allocbufb; /* use the allocated space */ + } + uprv_decNumberCopy(b, rhs); /* copy content */ + b->bits&=~DECNEG; /* .. and clear the sign */ + rhs=b; /* use copy from here on */ + } + decCompareOp(res, lhs, rhs, set, COMPTOTAL, &status); + } while(0); /* end protected */ + + if (allocbufa!=nullptr) free(allocbufa); /* drop any storage used */ + if (allocbufb!=nullptr) free(allocbufb); /* .. */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberCompareTotalMag */ + +/* ------------------------------------------------------------------ */ +/* decNumberDivide -- divide one number by another */ +/* */ +/* This computes C = A / B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X/X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivide(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decDivideOp(res, lhs, rhs, set, DIVIDE, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberDivide */ + +/* ------------------------------------------------------------------ */ +/* decNumberDivideInteger -- divide and return integer quotient */ +/* */ +/* This computes C = A # B, where # is the integer divide operator */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X#X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivideInteger(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decDivideOp(res, lhs, rhs, set, DIVIDEINT, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberDivideInteger */ + +/* ------------------------------------------------------------------ */ +/* decNumberExp -- exponentiation */ +/* */ +/* This computes C = exp(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Mathematical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* */ +/* Finite results will always be full precision and Inexact, except */ +/* when A is a zero or -Infinity (giving 1 or 0 respectively). */ +/* */ +/* An Inexact result is rounded using DEC_ROUND_HALF_EVEN; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* ------------------------------------------------------------------ */ +/* This is a wrapper for decExpOp which can handle the slightly wider */ +/* (double) range needed by Ln (which has to be able to calculate */ +/* exp(-a) where a can be the tiniest number (Ntiny). */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberExp(decNumber *res, const decNumber *rhs, + decContext *set) { + uInt status=0; /* accumulator */ + #if DECSUBSET + decNumber *allocrhs=nullptr; /* non-nullptr if rounded rhs allocated */ + #endif + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* Check restrictions; these restrictions ensure that if h=8 (see */ + /* decExpOp) then the result will either overflow or underflow to 0. */ + /* Other math functions restrict the input range, too, for inverses. */ + /* If not violated then carry out the operation. */ + if (!decCheckMath(rhs, set, &status)) do { /* protect allocation */ + #if DECSUBSET + if (!set->extended) { + /* reduce operand and set lostDigits status, as needed */ + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + decExpOp(res, rhs, set, &status); + } while(0); /* end protected */ + + #if DECSUBSET + if (allocrhs !=nullptr) free(allocrhs); /* drop any storage used */ + #endif + /* apply significant status */ + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberExp */ + +/* ------------------------------------------------------------------ */ +/* decNumberFMA -- fused multiply add */ +/* */ +/* This computes D = (A * B) + C with only one rounding */ +/* */ +/* res is D, the result. D may be A or B or C (e.g., X=FMA(X,X,X)) */ +/* lhs is A */ +/* rhs is B */ +/* fhs is C [far hand side] */ +/* set is the context */ +/* */ +/* Mathematical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberFMA(decNumber *res, const decNumber *lhs, + const decNumber *rhs, const decNumber *fhs, + decContext *set) { + uInt status=0; /* accumulator */ + decContext dcmul; /* context for the multiplication */ + uInt needbytes; /* for space calculations */ + decNumber bufa[D2N(DECBUFFER*2+1)]; + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *acc; /* accumulator pointer */ + decNumber dzero; /* work */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + if (decCheckOperands(res, fhs, DECUNUSED, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { /* [undefined if subset] */ + status|=DEC_Invalid_operation; + break;} + #endif + /* Check math restrictions [these ensure no overflow or underflow] */ + if ((!decNumberIsSpecial(lhs) && decCheckMath(lhs, set, &status)) + || (!decNumberIsSpecial(rhs) && decCheckMath(rhs, set, &status)) + || (!decNumberIsSpecial(fhs) && decCheckMath(fhs, set, &status))) break; + /* set up context for multiply */ + dcmul=*set; + dcmul.digits=lhs->digits+rhs->digits; /* just enough */ + /* [The above may be an over-estimate for subset arithmetic, but that's OK] */ + dcmul.emax=DEC_MAX_EMAX; /* effectively unbounded .. */ + dcmul.emin=DEC_MIN_EMIN; /* [thanks to Math restrictions] */ + /* set up decNumber space to receive the result of the multiply */ + acc=bufa; /* may fit */ + needbytes=sizeof(decNumber)+(D2U(dcmul.digits)-1)*sizeof(Unit); + if (needbytes>sizeof(bufa)) { /* need malloc space */ + allocbufa=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + acc=allocbufa; /* use the allocated space */ + } + /* multiply with extended range and necessary precision */ + /*printf("emin=%ld\n", dcmul.emin); */ + decMultiplyOp(acc, lhs, rhs, &dcmul, &status); + /* Only Invalid operation (from sNaN or Inf * 0) is possible in */ + /* status; if either is seen than ignore fhs (in case it is */ + /* another sNaN) and set acc to NaN unless we had an sNaN */ + /* [decMultiplyOp leaves that to caller] */ + /* Note sNaN has to go through addOp to shorten payload if */ + /* necessary */ + if ((status&DEC_Invalid_operation)!=0) { + if (!(status&DEC_sNaN)) { /* but be true invalid */ + uprv_decNumberZero(res); /* acc not yet set */ + res->bits=DECNAN; + break; + } + uprv_decNumberZero(&dzero); /* make 0 (any non-NaN would do) */ + fhs=&dzero; /* use that */ + } + #if DECCHECK + else { /* multiply was OK */ + if (status!=0) printf("Status=%08lx after FMA multiply\n", (LI)status); + } + #endif + /* add the third operand and result -> res, and all is done */ + decAddOp(res, acc, fhs, set, 0, &status); + } while(0); /* end protected */ + + if (allocbufa!=nullptr) free(allocbufa); /* drop any storage used */ + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberFMA */ + +/* ------------------------------------------------------------------ */ +/* decNumberInvert -- invert a Number, digitwise */ +/* */ +/* This computes C = ~A */ +/* */ +/* res is C, the result. C may be A (e.g., X=~X) */ +/* rhs is A */ +/* set is the context (used for result length and error report) */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Logical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberInvert(decNumber *res, const decNumber *rhs, + decContext *set) { + const Unit *ua, *msua; /* -> operand and its msu */ + Unit *uc, *msuc; /* -> result and its msu */ + Int msudigs; /* digits in res msu */ + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + if (rhs->exponent!=0 || decNumberIsSpecial(rhs) || decNumberIsNegative(rhs)) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + /* operand is valid */ + ua=rhs->lsu; /* bottom-up */ + uc=res->lsu; /* .. */ + msua=ua+D2U(rhs->digits)-1; /* -> msu of rhs */ + msuc=uc+D2U(set->digits)-1; /* -> msu of result */ + msudigs=MSUDIGITS(set->digits); /* [faster than remainder] */ + for (; uc<=msuc; ua++, uc++) { /* Unit loop */ + Unit a; /* extract unit */ + Int i, j; /* work */ + if (ua>msua) a=0; + else a=*ua; + *uc=0; /* can now write back */ + /* always need to examine all bits in rhs */ + /* This loop could be unrolled and/or use BIN2BCD tables */ + for (i=0; i1) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + if (uc==msuc && i==msudigs-1) break; /* just did final digit */ + } /* each digit */ + } /* each unit */ + /* [here uc-1 is the msu of the result] */ + res->digits=decGetDigits(res->lsu, static_cast(uc - res->lsu)); + res->exponent=0; /* integer */ + res->bits=0; /* sign=0 */ + return res; /* [no status to set] */ + } /* decNumberInvert */ + +/* ------------------------------------------------------------------ */ +/* decNumberLn -- natural logarithm */ +/* */ +/* This computes C = ln(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Notable cases: */ +/* A<0 -> Invalid */ +/* A=0 -> -Infinity (Exact) */ +/* A=+Infinity -> +Infinity (Exact) */ +/* A=1 exactly -> 0 (Exact) */ +/* */ +/* Mathematical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* */ +/* An Inexact result is rounded using DEC_ROUND_HALF_EVEN; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* ------------------------------------------------------------------ */ +/* This is a wrapper for decLnOp which can handle the slightly wider */ +/* (+11) range needed by Ln, Log10, etc. (which may have to be able */ +/* to calculate at p+e+2). */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberLn(decNumber *res, const decNumber *rhs, + decContext *set) { + uInt status=0; /* accumulator */ + #if DECSUBSET + decNumber *allocrhs=nullptr; /* non-nullptr if rounded rhs allocated */ + #endif + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* Check restrictions; this is a math function; if not violated */ + /* then carry out the operation. */ + if (!decCheckMath(rhs, set, &status)) do { /* protect allocation */ + #if DECSUBSET + if (!set->extended) { + /* reduce operand and set lostDigits status, as needed */ + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + /* special check in subset for rhs=0 */ + if (ISZERO(rhs)) { /* +/- zeros -> error */ + status|=DEC_Invalid_operation; + break;} + } /* extended=0 */ + #endif + decLnOp(res, rhs, set, &status); + } while(0); /* end protected */ + + #if DECSUBSET + if (allocrhs !=nullptr) free(allocrhs); /* drop any storage used */ + #endif + /* apply significant status */ + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberLn */ + +/* ------------------------------------------------------------------ */ +/* decNumberLogB - get adjusted exponent, by 754 rules */ +/* */ +/* This computes C = adjustedexponent(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context, used only for digits and status */ +/* */ +/* C must have space for 10 digits (A might have 10**9 digits and */ +/* an exponent of +999999999, or one digit and an exponent of */ +/* -1999999999). */ +/* */ +/* This returns the adjusted exponent of A after (in theory) padding */ +/* with zeros on the right to set->digits digits while keeping the */ +/* same value. The exponent is not limited by emin/emax. */ +/* */ +/* Notable cases: */ +/* A<0 -> Use |A| */ +/* A=0 -> -Infinity (Division by zero) */ +/* A=Infinite -> +Infinity (Exact) */ +/* A=1 exactly -> 0 (Exact) */ +/* NaNs are propagated as usual */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberLogB(decNumber *res, const decNumber *rhs, + decContext *set) { + uInt status=0; /* accumulator */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* NaNs as usual; Infinities return +Infinity; 0->oops */ + if (decNumberIsNaN(rhs)) decNaNs(res, rhs, nullptr, set, &status); + else if (decNumberIsInfinite(rhs)) uprv_decNumberCopyAbs(res, rhs); + else if (decNumberIsZero(rhs)) { + uprv_decNumberZero(res); /* prepare for Infinity */ + res->bits=DECNEG|DECINF; /* -Infinity */ + status|=DEC_Division_by_zero; /* as per 754 */ + } + else { /* finite non-zero */ + Int ae=rhs->exponent+rhs->digits-1; /* adjusted exponent */ + uprv_decNumberFromInt32(res, ae); /* lay it out */ + } + + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberLogB */ + +/* ------------------------------------------------------------------ */ +/* decNumberLog10 -- logarithm in base 10 */ +/* */ +/* This computes C = log10(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Notable cases: */ +/* A<0 -> Invalid */ +/* A=0 -> -Infinity (Exact) */ +/* A=+Infinity -> +Infinity (Exact) */ +/* A=10**n (if n is an integer) -> n (Exact) */ +/* */ +/* Mathematical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* */ +/* An Inexact result is rounded using DEC_ROUND_HALF_EVEN; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* ------------------------------------------------------------------ */ +/* This calculates ln(A)/ln(10) using appropriate precision. For */ +/* ln(A) this is the max(p, rhs->digits + t) + 3, where p is the */ +/* requested digits and t is the number of digits in the exponent */ +/* (maximum 6). For ln(10) it is p + 3; this is often handled by the */ +/* fastpath in decLnOp. The final division is done to the requested */ +/* precision. */ +/* ------------------------------------------------------------------ */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif +U_CAPI decNumber * U_EXPORT2 uprv_decNumberLog10(decNumber *res, const decNumber *rhs, + decContext *set) { + uInt status=0, ignore=0; /* status accumulators */ + uInt needbytes; /* for space calculations */ + Int p; /* working precision */ + Int t; /* digits in exponent of A */ + + /* buffers for a and b working decimals */ + /* (adjustment calculator, same size) */ + decNumber bufa[D2N(DECBUFFER+2)]; + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *a=bufa; /* temporary a */ + decNumber bufb[D2N(DECBUFFER+2)]; + decNumber *allocbufb=nullptr; /* -> allocated bufb, iff allocated */ + decNumber *b=bufb; /* temporary b */ + decNumber bufw[D2N(10)]; /* working 2-10 digit number */ + decNumber *w=bufw; /* .. */ + #if DECSUBSET + decNumber *allocrhs=nullptr; /* non-nullptr if rounded rhs allocated */ + #endif + + decContext aset; /* working context */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* Check restrictions; this is a math function; if not violated */ + /* then carry out the operation. */ + if (!decCheckMath(rhs, set, &status)) do { /* protect malloc */ + #if DECSUBSET + if (!set->extended) { + /* reduce operand and set lostDigits status, as needed */ + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + /* special check in subset for rhs=0 */ + if (ISZERO(rhs)) { /* +/- zeros -> error */ + status|=DEC_Invalid_operation; + break;} + } /* extended=0 */ + #endif + + uprv_decContextDefault(&aset, DEC_INIT_DECIMAL64); /* clean context */ + + /* handle exact powers of 10; only check if +ve finite */ + if (!(rhs->bits&(DECNEG|DECSPECIAL)) && !ISZERO(rhs)) { + Int residue=0; /* (no residue) */ + uInt copystat=0; /* clean status */ + + /* round to a single digit... */ + aset.digits=1; + decCopyFit(w, rhs, &aset, &residue, ©stat); /* copy & shorten */ + /* if exact and the digit is 1, rhs is a power of 10 */ + if (!(copystat&DEC_Inexact) && w->lsu[0]==1) { + /* the exponent, conveniently, is the power of 10; making */ + /* this the result needs a little care as it might not fit, */ + /* so first convert it into the working number, and then move */ + /* to res */ + uprv_decNumberFromInt32(w, w->exponent); + residue=0; + decCopyFit(res, w, set, &residue, &status); /* copy & round */ + decFinish(res, set, &residue, &status); /* cleanup/set flags */ + break; + } /* not a power of 10 */ + } /* not a candidate for exact */ + + /* simplify the information-content calculation to use 'total */ + /* number of digits in a, including exponent' as compared to the */ + /* requested digits, as increasing this will only rarely cost an */ + /* iteration in ln(a) anyway */ + t=6; /* it can never be >6 */ + + /* allocate space when needed... */ + p=(rhs->digits+t>set->digits?rhs->digits+t:set->digits)+3; + needbytes=sizeof(decNumber)+(D2U(p)-1)*sizeof(Unit); + if (needbytes>sizeof(bufa)) { /* need malloc space */ + allocbufa=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + a=allocbufa; /* use the allocated space */ + } + aset.digits=p; /* as calculated */ + aset.emax=DEC_MAX_MATH; /* usual bounds */ + aset.emin=-DEC_MAX_MATH; /* .. */ + aset.clamp=0; /* and no concrete format */ + decLnOp(a, rhs, &aset, &status); /* a=ln(rhs) */ + + /* skip the division if the result so far is infinite, NaN, or */ + /* zero, or there was an error; note NaN from sNaN needs copy */ + if (status&DEC_NaNs && !(status&DEC_sNaN)) break; + if (a->bits&DECSPECIAL || ISZERO(a)) { + uprv_decNumberCopy(res, a); /* [will fit] */ + break;} + + /* for ln(10) an extra 3 digits of precision are needed */ + p=set->digits+3; + needbytes=sizeof(decNumber)+(D2U(p)-1)*sizeof(Unit); + if (needbytes>sizeof(bufb)) { /* need malloc space */ + allocbufb=(decNumber *)malloc(needbytes); + if (allocbufb==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + b=allocbufb; /* use the allocated space */ + } + uprv_decNumberZero(w); /* set up 10... */ + #if DECDPUN==1 + w->lsu[1]=1; w->lsu[0]=0; /* .. */ + #else + w->lsu[0]=10; /* .. */ + #endif + w->digits=2; /* .. */ + + aset.digits=p; + decLnOp(b, w, &aset, &ignore); /* b=ln(10) */ + + aset.digits=set->digits; /* for final divide */ + decDivideOp(res, a, b, &aset, DIVIDE, &status); /* into result */ + } while(0); /* [for break] */ + + if (allocbufa!=nullptr) free(allocbufa); /* drop any storage used */ + if (allocbufb!=nullptr) free(allocbufb); /* .. */ + #if DECSUBSET + if (allocrhs !=nullptr) free(allocrhs); /* .. */ + #endif + /* apply significant status */ + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberLog10 */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic pop +#endif + +/* ------------------------------------------------------------------ */ +/* decNumberMax -- compare two Numbers and return the maximum */ +/* */ +/* This computes C = A ? B, returning the maximum by 754 rules */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMax(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPMAX, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMax */ + +/* ------------------------------------------------------------------ */ +/* decNumberMaxMag -- compare and return the maximum by magnitude */ +/* */ +/* This computes C = A ? B, returning the maximum by 754 rules */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMaxMag(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPMAXMAG, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMaxMag */ + +/* ------------------------------------------------------------------ */ +/* decNumberMin -- compare two Numbers and return the minimum */ +/* */ +/* This computes C = A ? B, returning the minimum by 754 rules */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMin(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPMIN, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMin */ + +/* ------------------------------------------------------------------ */ +/* decNumberMinMag -- compare and return the minimum by magnitude */ +/* */ +/* This computes C = A ? B, returning the minimum by 754 rules */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinMag(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decCompareOp(res, lhs, rhs, set, COMPMINMAG, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMinMag */ + +/* ------------------------------------------------------------------ */ +/* decNumberMinus -- prefix minus operator */ +/* */ +/* This computes C = 0 - A */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* See also decNumberCopyNegate for a quiet bitwise version of this. */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* Simply use AddOp for the subtract, which will do the necessary. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinus(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dzero; + uInt status=0; /* accumulator */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + uprv_decNumberZero(&dzero); /* make 0 */ + dzero.exponent=rhs->exponent; /* [no coefficient expansion] */ + decAddOp(res, &dzero, rhs, set, DECNEG, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMinus */ + +/* ------------------------------------------------------------------ */ +/* decNumberNextMinus -- next towards -Infinity */ +/* */ +/* This computes C = A - infinitesimal, rounded towards -Infinity */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* This is a generalization of 754 NextDown. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextMinus(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dtiny; /* constant */ + decContext workset=*set; /* work */ + uInt status=0; /* accumulator */ + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* +Infinity is the special case */ + if ((rhs->bits&(DECINF|DECNEG))==DECINF) { + decSetMaxValue(res, set); /* is +ve */ + /* there is no status to set */ + return res; + } + uprv_decNumberZero(&dtiny); /* start with 0 */ + dtiny.lsu[0]=1; /* make number that is .. */ + dtiny.exponent=DEC_MIN_EMIN-1; /* .. smaller than tiniest */ + workset.round=DEC_ROUND_FLOOR; + decAddOp(res, rhs, &dtiny, &workset, DECNEG, &status); + status&=DEC_Invalid_operation|DEC_sNaN; /* only sNaN Invalid please */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberNextMinus */ + +/* ------------------------------------------------------------------ */ +/* decNumberNextPlus -- next towards +Infinity */ +/* */ +/* This computes C = A + infinitesimal, rounded towards +Infinity */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* This is a generalization of 754 NextUp. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextPlus(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dtiny; /* constant */ + decContext workset=*set; /* work */ + uInt status=0; /* accumulator */ + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* -Infinity is the special case */ + if ((rhs->bits&(DECINF|DECNEG))==(DECINF|DECNEG)) { + decSetMaxValue(res, set); + res->bits=DECNEG; /* negative */ + /* there is no status to set */ + return res; + } + uprv_decNumberZero(&dtiny); /* start with 0 */ + dtiny.lsu[0]=1; /* make number that is .. */ + dtiny.exponent=DEC_MIN_EMIN-1; /* .. smaller than tiniest */ + workset.round=DEC_ROUND_CEILING; + decAddOp(res, rhs, &dtiny, &workset, 0, &status); + status&=DEC_Invalid_operation|DEC_sNaN; /* only sNaN Invalid please */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberNextPlus */ + +/* ------------------------------------------------------------------ */ +/* decNumberNextToward -- next towards rhs */ +/* */ +/* This computes C = A +/- infinitesimal, rounded towards */ +/* +/-Infinity in the direction of B, as per 754-1985 nextafter */ +/* modified during revision but dropped from 754-2008. */ +/* */ +/* res is C, the result. C may be A or B. */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* This is a generalization of 754-1985 NextAfter. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextToward(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + decNumber dtiny; /* constant */ + decContext workset=*set; /* work */ + Int result; /* .. */ + uInt status=0; /* accumulator */ + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + if (decNumberIsNaN(lhs) || decNumberIsNaN(rhs)) { + decNaNs(res, lhs, rhs, set, &status); + } + else { /* Is numeric, so no chance of sNaN Invalid, etc. */ + result=decCompare(lhs, rhs, 0); /* sign matters */ + if (result==BADINT) status|=DEC_Insufficient_storage; /* rare */ + else { /* valid compare */ + if (result==0) uprv_decNumberCopySign(res, lhs, rhs); /* easy */ + else { /* differ: need NextPlus or NextMinus */ + uByte sub; /* add or subtract */ + if (result<0) { /* lhsbits&(DECINF|DECNEG))==(DECINF|DECNEG)) { + decSetMaxValue(res, set); + res->bits=DECNEG; /* negative */ + return res; /* there is no status to set */ + } + workset.round=DEC_ROUND_CEILING; + sub=0; /* add, please */ + } /* plus */ + else { /* lhs>rhs, do nextminus */ + /* +Infinity is the special case */ + if ((lhs->bits&(DECINF|DECNEG))==DECINF) { + decSetMaxValue(res, set); + return res; /* there is no status to set */ + } + workset.round=DEC_ROUND_FLOOR; + sub=DECNEG; /* subtract, please */ + } /* minus */ + uprv_decNumberZero(&dtiny); /* start with 0 */ + dtiny.lsu[0]=1; /* make number that is .. */ + dtiny.exponent=DEC_MIN_EMIN-1; /* .. smaller than tiniest */ + decAddOp(res, lhs, &dtiny, &workset, sub, &status); /* + or - */ + /* turn off exceptions if the result is a normal number */ + /* (including Nmin), otherwise let all status through */ + if (uprv_decNumberIsNormal(res, set)) status=0; + } /* unequal */ + } /* compare OK */ + } /* numeric */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberNextToward */ + +/* ------------------------------------------------------------------ */ +/* decNumberOr -- OR two Numbers, digitwise */ +/* */ +/* This computes C = A | B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X|X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context (used for result length and error report) */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Logical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberOr(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + const Unit *ua, *ub; /* -> operands */ + const Unit *msua, *msub; /* -> operand msus */ + Unit *uc, *msuc; /* -> result and its msu */ + Int msudigs; /* digits in res msu */ + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + if (lhs->exponent!=0 || decNumberIsSpecial(lhs) || decNumberIsNegative(lhs) + || rhs->exponent!=0 || decNumberIsSpecial(rhs) || decNumberIsNegative(rhs)) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + /* operands are valid */ + ua=lhs->lsu; /* bottom-up */ + ub=rhs->lsu; /* .. */ + uc=res->lsu; /* .. */ + msua=ua+D2U(lhs->digits)-1; /* -> msu of lhs */ + msub=ub+D2U(rhs->digits)-1; /* -> msu of rhs */ + msuc=uc+D2U(set->digits)-1; /* -> msu of result */ + msudigs=MSUDIGITS(set->digits); /* [faster than remainder] */ + for (; uc<=msuc; ua++, ub++, uc++) { /* Unit loop */ + Unit a, b; /* extract units */ + if (ua>msua) a=0; + else a=*ua; + if (ub>msub) b=0; + else b=*ub; + *uc=0; /* can now write back */ + if (a|b) { /* maybe 1 bits to examine */ + Int i, j; + /* This loop could be unrolled and/or use BIN2BCD tables */ + for (i=0; i1) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + if (uc==msuc && i==msudigs-1) break; /* just did final digit */ + } /* each digit */ + } /* non-zero */ + } /* each unit */ + /* [here uc-1 is the msu of the result] */ + res->digits=decGetDigits(res->lsu, static_cast(uc-res->lsu)); + res->exponent=0; /* integer */ + res->bits=0; /* sign=0 */ + return res; /* [no status to set] */ + } /* decNumberOr */ + +/* ------------------------------------------------------------------ */ +/* decNumberPlus -- prefix plus operator */ +/* */ +/* This computes C = 0 + A */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* See also decNumberCopy for a quiet bitwise version of this. */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* This simply uses AddOp; Add will take fast path after preparing A. */ +/* Performance is a concern here, as this routine is often used to */ +/* check operands and apply rounding and overflow/underflow testing. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberPlus(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dzero; + uInt status=0; /* accumulator */ + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + uprv_decNumberZero(&dzero); /* make 0 */ + dzero.exponent=rhs->exponent; /* [no coefficient expansion] */ + decAddOp(res, &dzero, rhs, set, 0, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberPlus */ + +/* ------------------------------------------------------------------ */ +/* decNumberMultiply -- multiply two Numbers */ +/* */ +/* This computes C = A x B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X+X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberMultiply(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decMultiplyOp(res, lhs, rhs, set, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberMultiply */ + +/* ------------------------------------------------------------------ */ +/* decNumberPower -- raise a number to a power */ +/* */ +/* This computes C = A ** B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X**X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Mathematical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* */ +/* However, if 1999999997<=B<=999999999 and B is an integer then the */ +/* restrictions on A and the context are relaxed to the usual bounds, */ +/* for compatibility with the earlier (integer power only) version */ +/* of this function. */ +/* */ +/* When B is an integer, the result may be exact, even if rounded. */ +/* */ +/* The final result is rounded according to the context; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberPower(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + #if DECSUBSET + decNumber *alloclhs=nullptr; /* non-nullptr if rounded lhs allocated */ + decNumber *allocrhs=nullptr; /* .., rhs */ + #endif + decNumber *allocdac=nullptr; /* -> allocated acc buffer, iff used */ + decNumber *allocinv=nullptr; /* -> allocated 1/x buffer, iff used */ + Int reqdigits=set->digits; /* requested DIGITS */ + Int n; /* rhs in binary */ + Flag rhsint=0; /* 1 if rhs is an integer */ + Flag useint=0; /* 1 if can use integer calculation */ + Flag isoddint=0; /* 1 if rhs is an integer and odd */ + Int i; /* work */ + #if DECSUBSET + Int dropped; /* .. */ + #endif + uInt needbytes; /* buffer size needed */ + Flag seenbit; /* seen a bit while powering */ + Int residue=0; /* rounding residue */ + uInt status=0; /* accumulators */ + uByte bits=0; /* result sign if errors */ + decContext aset; /* working context */ + decNumber dnOne; /* work value 1... */ + /* local accumulator buffer [a decNumber, with digits+elength+1 digits] */ + decNumber dacbuff[D2N(DECBUFFER+9)]; + decNumber *dac=dacbuff; /* -> result accumulator */ + /* same again for possible 1/lhs calculation */ + decNumber invbuff[D2N(DECBUFFER+9)]; + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { /* reduce operands and set status, as needed */ + if (lhs->digits>reqdigits) { + alloclhs=decRoundOperand(lhs, set, &status); + if (alloclhs==nullptr) break; + lhs=alloclhs; + } + if (rhs->digits>reqdigits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* handle NaNs and rhs Infinity (lhs infinity is harder) */ + if (SPECIALARGS) { + if (decNumberIsNaN(lhs) || decNumberIsNaN(rhs)) { /* NaNs */ + decNaNs(res, lhs, rhs, set, &status); + break;} + if (decNumberIsInfinite(rhs)) { /* rhs Infinity */ + Flag rhsneg=rhs->bits&DECNEG; /* save rhs sign */ + if (decNumberIsNegative(lhs) /* lhs<0 */ + && !decNumberIsZero(lhs)) /* .. */ + status|=DEC_Invalid_operation; + else { /* lhs >=0 */ + uprv_decNumberZero(&dnOne); /* set up 1 */ + dnOne.lsu[0]=1; + uprv_decNumberCompare(dac, lhs, &dnOne, set); /* lhs ? 1 */ + uprv_decNumberZero(res); /* prepare for 0/1/Infinity */ + if (decNumberIsNegative(dac)) { /* lhs<1 */ + if (rhsneg) res->bits|=DECINF; /* +Infinity [else is +0] */ + } + else if (dac->lsu[0]==0) { /* lhs=1 */ + /* 1**Infinity is inexact, so return fully-padded 1.0000 */ + Int shift=set->digits-1; + *res->lsu=1; /* was 0, make int 1 */ + res->digits=decShiftToMost(res->lsu, 1, shift); + res->exponent=-shift; /* make 1.0000... */ + status|=DEC_Inexact|DEC_Rounded; /* deemed inexact */ + } + else { /* lhs>1 */ + if (!rhsneg) res->bits|=DECINF; /* +Infinity [else is +0] */ + } + } /* lhs>=0 */ + break;} + /* [lhs infinity drops through] */ + } /* specials */ + + /* Original rhs may be an integer that fits and is in range */ + n=decGetInt(rhs); + if (n!=BADINT) { /* it is an integer */ + rhsint=1; /* record the fact for 1**n */ + isoddint=(Flag)n&1; /* [works even if big] */ + if (n!=BIGEVEN && n!=BIGODD) /* can use integer path? */ + useint=1; /* looks good */ + } + + if (decNumberIsNegative(lhs) /* -x .. */ + && isoddint) bits=DECNEG; /* .. to an odd power */ + + /* handle LHS infinity */ + if (decNumberIsInfinite(lhs)) { /* [NaNs already handled] */ + uByte rbits=rhs->bits; /* save */ + uprv_decNumberZero(res); /* prepare */ + if (n==0) *res->lsu=1; /* [-]Inf**0 => 1 */ + else { + /* -Inf**nonint -> error */ + if (!rhsint && decNumberIsNegative(lhs)) { + status|=DEC_Invalid_operation; /* -Inf**nonint is error */ + break;} + if (!(rbits & DECNEG)) bits|=DECINF; /* was not a **-n */ + /* [otherwise will be 0 or -0] */ + res->bits=bits; + } + break;} + + /* similarly handle LHS zero */ + if (decNumberIsZero(lhs)) { + if (n==0) { /* 0**0 => Error */ + #if DECSUBSET + if (!set->extended) { /* [unless subset] */ + uprv_decNumberZero(res); + *res->lsu=1; /* return 1 */ + break;} + #endif + status|=DEC_Invalid_operation; + } + else { /* 0**x */ + uByte rbits=rhs->bits; /* save */ + if (rbits & DECNEG) { /* was a 0**(-n) */ + #if DECSUBSET + if (!set->extended) { /* [bad if subset] */ + status|=DEC_Invalid_operation; + break;} + #endif + bits|=DECINF; + } + uprv_decNumberZero(res); /* prepare */ + /* [otherwise will be 0 or -0] */ + res->bits=bits; + } + break;} + + /* here both lhs and rhs are finite; rhs==0 is handled in the */ + /* integer path. Next handle the non-integer cases */ + if (!useint) { /* non-integral rhs */ + /* any -ve lhs is bad, as is either operand or context out of */ + /* bounds */ + if (decNumberIsNegative(lhs)) { + status|=DEC_Invalid_operation; + break;} + if (decCheckMath(lhs, set, &status) + || decCheckMath(rhs, set, &status)) break; /* variable status */ + + uprv_decContextDefault(&aset, DEC_INIT_DECIMAL64); /* clean context */ + aset.emax=DEC_MAX_MATH; /* usual bounds */ + aset.emin=-DEC_MAX_MATH; /* .. */ + aset.clamp=0; /* and no concrete format */ + + /* calculate the result using exp(ln(lhs)*rhs), which can */ + /* all be done into the accumulator, dac. The precision needed */ + /* is enough to contain the full information in the lhs (which */ + /* is the total digits, including exponent), or the requested */ + /* precision, if larger, + 4; 6 is used for the exponent */ + /* maximum length, and this is also used when it is shorter */ + /* than the requested digits as it greatly reduces the >0.5 ulp */ + /* cases at little cost (because Ln doubles digits each */ + /* iteration so a few extra digits rarely causes an extra */ + /* iteration) */ + aset.digits=MAXI(lhs->digits, set->digits)+6+4; + } /* non-integer rhs */ + + else { /* rhs is in-range integer */ + if (n==0) { /* x**0 = 1 */ + /* (0**0 was handled above) */ + uprv_decNumberZero(res); /* result=1 */ + *res->lsu=1; /* .. */ + break;} + /* rhs is a non-zero integer */ + if (n<0) n=-n; /* use abs(n) */ + + aset=*set; /* clone the context */ + aset.round=DEC_ROUND_HALF_EVEN; /* internally use balanced */ + /* calculate the working DIGITS */ + aset.digits=reqdigits+(rhs->digits+rhs->exponent)+2; + #if DECSUBSET + if (!set->extended) aset.digits--; /* use classic precision */ + #endif + /* it's an error if this is more than can be handled */ + if (aset.digits>DECNUMMAXP) {status|=DEC_Invalid_operation; break;} + } /* integer path */ + + /* aset.digits is the count of digits for the accumulator needed */ + /* if accumulator is too long for local storage, then allocate */ + needbytes=sizeof(decNumber)+(D2U(aset.digits)-1)*sizeof(Unit); + /* [needbytes also used below if 1/lhs needed] */ + if (needbytes>sizeof(dacbuff)) { + allocdac=(decNumber *)malloc(needbytes); + if (allocdac==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + dac=allocdac; /* use the allocated space */ + } + /* here, aset is set up and accumulator is ready for use */ + + if (!useint) { /* non-integral rhs */ + /* x ** y; special-case x=1 here as it will otherwise always */ + /* reduce to integer 1; decLnOp has a fastpath which detects */ + /* the case of x=1 */ + decLnOp(dac, lhs, &aset, &status); /* dac=ln(lhs) */ + /* [no error possible, as lhs 0 already handled] */ + if (ISZERO(dac)) { /* x==1, 1.0, etc. */ + /* need to return fully-padded 1.0000 etc., but rhsint->1 */ + *dac->lsu=1; /* was 0, make int 1 */ + if (!rhsint) { /* add padding */ + Int shift=set->digits-1; + dac->digits=decShiftToMost(dac->lsu, 1, shift); + dac->exponent=-shift; /* make 1.0000... */ + status|=DEC_Inexact|DEC_Rounded; /* deemed inexact */ + } + } + else { + decMultiplyOp(dac, dac, rhs, &aset, &status); /* dac=dac*rhs */ + decExpOp(dac, dac, &aset, &status); /* dac=exp(dac) */ + } + /* and drop through for final rounding */ + } /* non-integer rhs */ + + else { /* carry on with integer */ + uprv_decNumberZero(dac); /* acc=1 */ + *dac->lsu=1; /* .. */ + + /* if a negative power the constant 1 is needed, and if not subset */ + /* invert the lhs now rather than inverting the result later */ + if (decNumberIsNegative(rhs)) { /* was a **-n [hence digits>0] */ + decNumber *inv=invbuff; /* assume use fixed buffer */ + uprv_decNumberCopy(&dnOne, dac); /* dnOne=1; [needed now or later] */ + #if DECSUBSET + if (set->extended) { /* need to calculate 1/lhs */ + #endif + /* divide lhs into 1, putting result in dac [dac=1/dac] */ + decDivideOp(dac, &dnOne, lhs, &aset, DIVIDE, &status); + /* now locate or allocate space for the inverted lhs */ + if (needbytes>sizeof(invbuff)) { + allocinv=(decNumber *)malloc(needbytes); + if (allocinv==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + inv=allocinv; /* use the allocated space */ + } + /* [inv now points to big-enough buffer or allocated storage] */ + uprv_decNumberCopy(inv, dac); /* copy the 1/lhs */ + uprv_decNumberCopy(dac, &dnOne); /* restore acc=1 */ + lhs=inv; /* .. and go forward with new lhs */ + #if DECSUBSET + } + #endif + } + + /* Raise-to-the-power loop... */ + seenbit=0; /* set once a 1-bit is encountered */ + for (i=1;;i++){ /* for each bit [top bit ignored] */ + /* abandon if had overflow or terminal underflow */ + if (status & (DEC_Overflow|DEC_Underflow)) { /* interesting? */ + if (status&DEC_Overflow || ISZERO(dac)) break; + } + /* [the following two lines revealed an optimizer bug in a C++ */ + /* compiler, with symptom: 5**3 -> 25, when n=n+n was used] */ + n=n<<1; /* move next bit to testable position */ + if (n<0) { /* top bit is set */ + seenbit=1; /* OK, significant bit seen */ + decMultiplyOp(dac, dac, lhs, &aset, &status); /* dac=dac*x */ + } + if (i==31) break; /* that was the last bit */ + if (!seenbit) continue; /* no need to square 1 */ + decMultiplyOp(dac, dac, dac, &aset, &status); /* dac=dac*dac [square] */ + } /*i*/ /* 32 bits */ + + /* complete internal overflow or underflow processing */ + if (status & (DEC_Overflow|DEC_Underflow)) { + #if DECSUBSET + /* If subset, and power was negative, reverse the kind of -erflow */ + /* [1/x not yet done] */ + if (!set->extended && decNumberIsNegative(rhs)) { + if (status & DEC_Overflow) + status^=DEC_Overflow | DEC_Underflow | DEC_Subnormal; + else { /* trickier -- Underflow may or may not be set */ + status&=~(DEC_Underflow | DEC_Subnormal); /* [one or both] */ + status|=DEC_Overflow; + } + } + #endif + dac->bits=(dac->bits & ~DECNEG) | bits; /* force correct sign */ + /* round subnormals [to set.digits rather than aset.digits] */ + /* or set overflow result similarly as required */ + decFinalize(dac, set, &residue, &status); + uprv_decNumberCopy(res, dac); /* copy to result (is now OK length) */ + break; + } + + #if DECSUBSET + if (!set->extended && /* subset math */ + decNumberIsNegative(rhs)) { /* was a **-n [hence digits>0] */ + /* so divide result into 1 [dac=1/dac] */ + decDivideOp(dac, &dnOne, dac, &aset, DIVIDE, &status); + } + #endif + } /* rhs integer path */ + + /* reduce result to the requested length and copy to result */ + decCopyFit(res, dac, set, &residue, &status); + decFinish(res, set, &residue, &status); /* final cleanup */ + #if DECSUBSET + if (!set->extended) decTrim(res, set, 0, 1, &dropped); /* trailing zeros */ + #endif + } while(0); /* end protected */ + + if (allocdac!=nullptr) free(allocdac); /* drop any storage used */ + if (allocinv!=nullptr) free(allocinv); /* .. */ + #if DECSUBSET + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + if (allocrhs!=nullptr) free(allocrhs); /* .. */ + #endif + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberPower */ + +/* ------------------------------------------------------------------ */ +/* decNumberQuantize -- force exponent to requested value */ +/* */ +/* This computes C = op(A, B), where op adjusts the coefficient */ +/* of C (by rounding or shifting) such that the exponent (-scale) */ +/* of C has exponent of B. The numerical value of C will equal A, */ +/* except for the effects of any rounding that occurred. */ +/* */ +/* res is C, the result. C may be A or B */ +/* lhs is A, the number to adjust */ +/* rhs is B, the number with exponent to match */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Unless there is an error or the result is infinite, the exponent */ +/* after the operation is guaranteed to be equal to that of B. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberQuantize(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decQuantizeOp(res, lhs, rhs, set, 1, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberQuantize */ + +/* ------------------------------------------------------------------ */ +/* decNumberReduce -- remove trailing zeros */ +/* */ +/* This computes C = 0 + A, and normalizes the result */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* Previously known as Normalize */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberNormalize(decNumber *res, const decNumber *rhs, + decContext *set) { + return uprv_decNumberReduce(res, rhs, set); + } /* decNumberNormalize */ + +U_CAPI decNumber * U_EXPORT2 uprv_decNumberReduce(decNumber *res, const decNumber *rhs, + decContext *set) { + #if DECSUBSET + decNumber *allocrhs=nullptr; /* non-nullptr if rounded rhs allocated */ + #endif + uInt status=0; /* as usual */ + Int residue=0; /* as usual */ + Int dropped; /* work */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operand and set lostDigits status, as needed */ + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* Infinities copy through; NaNs need usual treatment */ + if (decNumberIsNaN(rhs)) { + decNaNs(res, rhs, nullptr, set, &status); + break; + } + + /* reduce result to the requested length and copy to result */ + decCopyFit(res, rhs, set, &residue, &status); /* copy & round */ + decFinish(res, set, &residue, &status); /* cleanup/set flags */ + decTrim(res, set, 1, 0, &dropped); /* normalize in place */ + /* [may clamp] */ + } while(0); /* end protected */ + + #if DECSUBSET + if (allocrhs !=nullptr) free(allocrhs); /* .. */ + #endif + if (status!=0) decStatus(res, status, set);/* then report status */ + return res; + } /* decNumberReduce */ + +/* ------------------------------------------------------------------ */ +/* decNumberRescale -- force exponent to requested value */ +/* */ +/* This computes C = op(A, B), where op adjusts the coefficient */ +/* of C (by rounding or shifting) such that the exponent (-scale) */ +/* of C has the value B. The numerical value of C will equal A, */ +/* except for the effects of any rounding that occurred. */ +/* */ +/* res is C, the result. C may be A or B */ +/* lhs is A, the number to adjust */ +/* rhs is B, the requested exponent */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Unless there is an error or the result is infinite, the exponent */ +/* after the operation is guaranteed to be equal to B. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberRescale(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decQuantizeOp(res, lhs, rhs, set, 0, &status); + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberRescale */ + +/* ------------------------------------------------------------------ */ +/* decNumberRemainder -- divide and return remainder */ +/* */ +/* This computes C = A % B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X%X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainder(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decDivideOp(res, lhs, rhs, set, REMAINDER, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberRemainder */ + +/* ------------------------------------------------------------------ */ +/* decNumberRemainderNear -- divide and return remainder from nearest */ +/* */ +/* This computes C = A % B, where % is the IEEE remainder operator */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X%X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainderNear(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + decDivideOp(res, lhs, rhs, set, REMNEAR, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberRemainderNear */ + +/* ------------------------------------------------------------------ */ +/* decNumberRotate -- rotate the coefficient of a Number left/right */ +/* */ +/* This computes C = A rot B (in base ten and rotating set->digits */ +/* digits). */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=XrotX) */ +/* lhs is A */ +/* rhs is B, the number of digits to rotate (-ve to right) */ +/* set is the context */ +/* */ +/* The digits of the coefficient of A are rotated to the left (if B */ +/* is positive) or to the right (if B is negative) without adjusting */ +/* the exponent or the sign of A. If lhs->digits is less than */ +/* set->digits the coefficient is padded with zeros on the left */ +/* before the rotate. Any leading zeros in the result are removed */ +/* as usual. */ +/* */ +/* B must be an integer (q=0) and in the range -set->digits through */ +/* +set->digits. */ +/* C must have space for set->digits digits. */ +/* NaNs are propagated as usual. Infinities are unaffected (but */ +/* B must be valid). No status is set unless B is invalid or an */ +/* operand is an sNaN. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberRotate(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + Int rotate; /* rhs as an Int */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + /* NaNs propagate as normal */ + if (decNumberIsNaN(lhs) || decNumberIsNaN(rhs)) + decNaNs(res, lhs, rhs, set, &status); + /* rhs must be an integer */ + else if (decNumberIsInfinite(rhs) || rhs->exponent!=0) + status=DEC_Invalid_operation; + else { /* both numeric, rhs is an integer */ + rotate=decGetInt(rhs); /* [cannot fail] */ + if (rotate==BADINT /* something bad .. */ + || rotate==BIGODD || rotate==BIGEVEN /* .. very big .. */ + || abs(rotate)>set->digits) /* .. or out of range */ + status=DEC_Invalid_operation; + else { /* rhs is OK */ + uprv_decNumberCopy(res, lhs); + /* convert -ve rotate to equivalent positive rotation */ + if (rotate<0) rotate=set->digits+rotate; + if (rotate!=0 && rotate!=set->digits /* zero or full rotation */ + && !decNumberIsInfinite(res)) { /* lhs was infinite */ + /* left-rotate to do; 0 < rotate < set->digits */ + uInt units, shift; /* work */ + uInt msudigits; /* digits in result msu */ + Unit *msu=res->lsu+D2U(res->digits)-1; /* current msu */ + Unit *msumax=res->lsu+D2U(set->digits)-1; /* rotation msu */ + for (msu++; msu<=msumax; msu++) *msu=0; /* ensure high units=0 */ + res->digits=set->digits; /* now full-length */ + msudigits=MSUDIGITS(res->digits); /* actual digits in msu */ + + /* rotation here is done in-place, in three steps */ + /* 1. shift all to least up to one unit to unit-align final */ + /* lsd [any digits shifted out are rotated to the left, */ + /* abutted to the original msd (which may require split)] */ + /* */ + /* [if there are no whole units left to rotate, the */ + /* rotation is now complete] */ + /* */ + /* 2. shift to least, from below the split point only, so that */ + /* the final msd is in the right place in its Unit [any */ + /* digits shifted out will fit exactly in the current msu, */ + /* left aligned, no split required] */ + /* */ + /* 3. rotate all the units by reversing left part, right */ + /* part, and then whole */ + /* */ + /* example: rotate right 8 digits (2 units + 2), DECDPUN=3. */ + /* */ + /* start: 00a bcd efg hij klm npq */ + /* */ + /* 1a 000 0ab cde fgh|ijk lmn [pq saved] */ + /* 1b 00p qab cde fgh|ijk lmn */ + /* */ + /* 2a 00p qab cde fgh|00i jkl [mn saved] */ + /* 2b mnp qab cde fgh|00i jkl */ + /* */ + /* 3a fgh cde qab mnp|00i jkl */ + /* 3b fgh cde qab mnp|jkl 00i */ + /* 3c 00i jkl mnp qab cde fgh */ + + /* Step 1: amount to shift is the partial right-rotate count */ + rotate=set->digits-rotate; /* make it right-rotate */ + units=rotate/DECDPUN; /* whole units to rotate */ + shift=rotate%DECDPUN; /* left-over digits count */ + if (shift>0) { /* not an exact number of units */ + uInt save=res->lsu[0]%powers[shift]; /* save low digit(s) */ + decShiftToLeast(res->lsu, D2U(res->digits), shift); + if (shift>msudigits) { /* msumax-1 needs >0 digits */ + uInt rem=save%powers[shift-msudigits];/* split save */ + *msumax=(Unit)(save/powers[shift-msudigits]); /* and insert */ + *(msumax-1)=*(msumax-1) + +(Unit)(rem*powers[DECDPUN-(shift-msudigits)]); /* .. */ + } + else { /* all fits in msumax */ + *msumax=*msumax+(Unit)(save*powers[msudigits-shift]); /* [maybe *1] */ + } + } /* digits shift needed */ + + /* If whole units to rotate... */ + if (units>0) { /* some to do */ + /* Step 2: the units to touch are the whole ones in rotate, */ + /* if any, and the shift is DECDPUN-msudigits (which may be */ + /* 0, again) */ + shift=DECDPUN-msudigits; + if (shift>0) { /* not an exact number of units */ + uInt save=res->lsu[0]%powers[shift]; /* save low digit(s) */ + decShiftToLeast(res->lsu, units, shift); + *msumax=*msumax+(Unit)(save*powers[msudigits]); + } /* partial shift needed */ + + /* Step 3: rotate the units array using triple reverse */ + /* (reversing is easy and fast) */ + decReverse(res->lsu+units, msumax); /* left part */ + decReverse(res->lsu, res->lsu+units-1); /* right part */ + decReverse(res->lsu, msumax); /* whole */ + } /* whole units to rotate */ + /* the rotation may have left an undetermined number of zeros */ + /* on the left, so true length needs to be calculated */ + res->digits=decGetDigits(res->lsu, static_cast(msumax-res->lsu+1)); + } /* rotate needed */ + } /* rhs OK */ + } /* numerics */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberRotate */ + +/* ------------------------------------------------------------------ */ +/* decNumberSameQuantum -- test for equal exponents */ +/* */ +/* res is the result number, which will contain either 0 or 1 */ +/* lhs is a number to test */ +/* rhs is the second (usually a pattern) */ +/* */ +/* No errors are possible and no context is needed. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberSameQuantum(decNumber *res, const decNumber *lhs, + const decNumber *rhs) { + Unit ret=0; /* return value */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, DECUNCONT)) return res; + #endif + + if (SPECIALARGS) { + if (decNumberIsNaN(lhs) && decNumberIsNaN(rhs)) ret=1; + else if (decNumberIsInfinite(lhs) && decNumberIsInfinite(rhs)) ret=1; + /* [anything else with a special gives 0] */ + } + else if (lhs->exponent==rhs->exponent) ret=1; + + uprv_decNumberZero(res); /* OK to overwrite an operand now */ + *res->lsu=ret; + return res; + } /* decNumberSameQuantum */ + +/* ------------------------------------------------------------------ */ +/* decNumberScaleB -- multiply by a power of 10 */ +/* */ +/* This computes C = A x 10**B where B is an integer (q=0) with */ +/* maximum magnitude 2*(emax+digits) */ +/* */ +/* res is C, the result. C may be A or B */ +/* lhs is A, the number to adjust */ +/* rhs is B, the requested power of ten to use */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* The result may underflow or overflow. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberScaleB(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + Int reqexp; /* requested exponent change [B] */ + uInt status=0; /* accumulator */ + Int residue; /* work */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + /* Handle special values except lhs infinite */ + if (decNumberIsNaN(lhs) || decNumberIsNaN(rhs)) + decNaNs(res, lhs, rhs, set, &status); + /* rhs must be an integer */ + else if (decNumberIsInfinite(rhs) || rhs->exponent!=0) + status=DEC_Invalid_operation; + else { + /* lhs is a number; rhs is a finite with q==0 */ + reqexp=decGetInt(rhs); /* [cannot fail] */ + if (reqexp==BADINT /* something bad .. */ + || reqexp==BIGODD || reqexp==BIGEVEN /* .. very big .. */ + || abs(reqexp)>(2*(set->digits+set->emax))) /* .. or out of range */ + status=DEC_Invalid_operation; + else { /* rhs is OK */ + uprv_decNumberCopy(res, lhs); /* all done if infinite lhs */ + if (!decNumberIsInfinite(res)) { /* prepare to scale */ + res->exponent+=reqexp; /* adjust the exponent */ + residue=0; + decFinalize(res, set, &residue, &status); /* .. and check */ + } /* finite LHS */ + } /* rhs OK */ + } /* rhs finite */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberScaleB */ + +/* ------------------------------------------------------------------ */ +/* decNumberShift -- shift the coefficient of a Number left or right */ +/* */ +/* This computes C = A << B or C = A >> -B (in base ten). */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X<digits through */ +/* +set->digits. */ +/* C must have space for set->digits digits. */ +/* NaNs are propagated as usual. Infinities are unaffected (but */ +/* B must be valid). No status is set unless B is invalid or an */ +/* operand is an sNaN. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberShift(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + Int shift; /* rhs as an Int */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + /* NaNs propagate as normal */ + if (decNumberIsNaN(lhs) || decNumberIsNaN(rhs)) + decNaNs(res, lhs, rhs, set, &status); + /* rhs must be an integer */ + else if (decNumberIsInfinite(rhs) || rhs->exponent!=0) + status=DEC_Invalid_operation; + else { /* both numeric, rhs is an integer */ + shift=decGetInt(rhs); /* [cannot fail] */ + if (shift==BADINT /* something bad .. */ + || shift==BIGODD || shift==BIGEVEN /* .. very big .. */ + || abs(shift)>set->digits) /* .. or out of range */ + status=DEC_Invalid_operation; + else { /* rhs is OK */ + uprv_decNumberCopy(res, lhs); + if (shift!=0 && !decNumberIsInfinite(res)) { /* something to do */ + if (shift>0) { /* to left */ + if (shift==set->digits) { /* removing all */ + *res->lsu=0; /* so place 0 */ + res->digits=1; /* .. */ + } + else { /* */ + /* first remove leading digits if necessary */ + if (res->digits+shift>set->digits) { + decDecap(res, res->digits+shift-set->digits); + /* that updated res->digits; may have gone to 1 (for a */ + /* single digit or for zero */ + } + if (res->digits>1 || *res->lsu) /* if non-zero.. */ + res->digits=decShiftToMost(res->lsu, res->digits, shift); + } /* partial left */ + } /* left */ + else { /* to right */ + if (-shift>=res->digits) { /* discarding all */ + *res->lsu=0; /* so place 0 */ + res->digits=1; /* .. */ + } + else { + decShiftToLeast(res->lsu, D2U(res->digits), -shift); + res->digits-=(-shift); + } + } /* to right */ + } /* non-0 non-Inf shift */ + } /* rhs OK */ + } /* numerics */ + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberShift */ + +/* ------------------------------------------------------------------ */ +/* decNumberSquareRoot -- square root operator */ +/* */ +/* This computes C = squareroot(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +/* This uses the following varying-precision algorithm in: */ +/* */ +/* Properly Rounded Variable Precision Square Root, T. E. Hull and */ +/* A. Abrham, ACM Transactions on Mathematical Software, Vol 11 #3, */ +/* pp229-237, ACM, September 1985. */ +/* */ +/* The square-root is calculated using Newton's method, after which */ +/* a check is made to ensure the result is correctly rounded. */ +/* */ +/* % [Reformatted original Numerical Turing source code follows.] */ +/* function sqrt(x : real) : real */ +/* % sqrt(x) returns the properly rounded approximation to the square */ +/* % root of x, in the precision of the calling environment, or it */ +/* % fails if x < 0. */ +/* % t e hull and a abrham, august, 1984 */ +/* if x <= 0 then */ +/* if x < 0 then */ +/* assert false */ +/* else */ +/* result 0 */ +/* end if */ +/* end if */ +/* var f := setexp(x, 0) % fraction part of x [0.1 <= x < 1] */ +/* var e := getexp(x) % exponent part of x */ +/* var approx : real */ +/* if e mod 2 = 0 then */ +/* approx := .259 + .819 * f % approx to root of f */ +/* else */ +/* f := f/l0 % adjustments */ +/* e := e + 1 % for odd */ +/* approx := .0819 + 2.59 * f % exponent */ +/* end if */ +/* */ +/* var p:= 3 */ +/* const maxp := currentprecision + 2 */ +/* loop */ +/* p := min(2*p - 2, maxp) % p = 4,6,10, . . . , maxp */ +/* precision p */ +/* approx := .5 * (approx + f/approx) */ +/* exit when p = maxp */ +/* end loop */ +/* */ +/* % approx is now within 1 ulp of the properly rounded square root */ +/* % of f; to ensure proper rounding, compare squares of (approx - */ +/* % l/2 ulp) and (approx + l/2 ulp) with f. */ +/* p := currentprecision */ +/* begin */ +/* precision p + 2 */ +/* const approxsubhalf := approx - setexp(.5, -p) */ +/* if mulru(approxsubhalf, approxsubhalf) > f then */ +/* approx := approx - setexp(.l, -p + 1) */ +/* else */ +/* const approxaddhalf := approx + setexp(.5, -p) */ +/* if mulrd(approxaddhalf, approxaddhalf) < f then */ +/* approx := approx + setexp(.l, -p + 1) */ +/* end if */ +/* end if */ +/* end */ +/* result setexp(approx, e div 2) % fix exponent */ +/* end sqrt */ +/* ------------------------------------------------------------------ */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif +U_CAPI decNumber * U_EXPORT2 uprv_decNumberSquareRoot(decNumber *res, const decNumber *rhs, + decContext *set) { + decContext workset, approxset; /* work contexts */ + decNumber dzero; /* used for constant zero */ + Int maxp; /* largest working precision */ + Int workp; /* working precision */ + Int residue=0; /* rounding residue */ + uInt status=0, ignore=0; /* status accumulators */ + uInt rstatus; /* .. */ + Int exp; /* working exponent */ + Int ideal; /* ideal (preferred) exponent */ + Int needbytes; /* work */ + Int dropped; /* .. */ + + #if DECSUBSET + decNumber *allocrhs=nullptr; /* non-nullptr if rounded rhs allocated */ + #endif + /* buffer for f [needs +1 in case DECBUFFER 0] */ + decNumber buff[D2N(DECBUFFER+1)]; + /* buffer for a [needs +2 to match likely maxp] */ + decNumber bufa[D2N(DECBUFFER+2)]; + /* buffer for temporary, b [must be same size as a] */ + decNumber bufb[D2N(DECBUFFER+2)]; + decNumber *allocbuff=nullptr; /* -> allocated buff, iff allocated */ + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *allocbufb=nullptr; /* -> allocated bufb, iff allocated */ + decNumber *f=buff; /* reduced fraction */ + decNumber *a=bufa; /* approximation to result */ + decNumber *b=bufb; /* intermediate result */ + /* buffer for temporary variable, up to 3 digits */ + decNumber buft[D2N(3)]; + decNumber *t=buft; /* up-to-3-digit constant or work */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operand and set lostDigits status, as needed */ + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, &status); + if (allocrhs==nullptr) break; + /* [Note: 'f' allocation below could reuse this buffer if */ + /* used, but as this is rare they are kept separate for clarity.] */ + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* handle infinities and NaNs */ + if (SPECIALARG) { + if (decNumberIsInfinite(rhs)) { /* an infinity */ + if (decNumberIsNegative(rhs)) status|=DEC_Invalid_operation; + else uprv_decNumberCopy(res, rhs); /* +Infinity */ + } + else decNaNs(res, rhs, nullptr, set, &status); /* a NaN */ + break; + } + + /* calculate the ideal (preferred) exponent [floor(exp/2)] */ + /* [It would be nicer to write: ideal=rhs->exponent>>1, but this */ + /* generates a compiler warning. Generated code is the same.] */ + ideal=(rhs->exponent&~1)/2; /* target */ + + /* handle zeros */ + if (ISZERO(rhs)) { + uprv_decNumberCopy(res, rhs); /* could be 0 or -0 */ + res->exponent=ideal; /* use the ideal [safe] */ + /* use decFinish to clamp any out-of-range exponent, etc. */ + decFinish(res, set, &residue, &status); + break; + } + + /* any other -x is an oops */ + if (decNumberIsNegative(rhs)) { + status|=DEC_Invalid_operation; + break; + } + + /* space is needed for three working variables */ + /* f -- the same precision as the RHS, reduced to 0.01->0.99... */ + /* a -- Hull's approximation -- precision, when assigned, is */ + /* currentprecision+1 or the input argument precision, */ + /* whichever is larger (+2 for use as temporary) */ + /* b -- intermediate temporary result (same size as a) */ + /* if any is too long for local storage, then allocate */ + workp=MAXI(set->digits+1, rhs->digits); /* actual rounding precision */ + workp=MAXI(workp, 7); /* at least 7 for low cases */ + maxp=workp+2; /* largest working precision */ + + needbytes=sizeof(decNumber)+(D2U(rhs->digits)-1)*sizeof(Unit); + if (needbytes>(Int)sizeof(buff)) { + allocbuff=(decNumber *)malloc(needbytes); + if (allocbuff==nullptr) { /* hopeless -- abandon */ + status|=DEC_Insufficient_storage; + break;} + f=allocbuff; /* use the allocated space */ + } + /* a and b both need to be able to hold a maxp-length number */ + needbytes=sizeof(decNumber)+(D2U(maxp)-1)*sizeof(Unit); + if (needbytes>(Int)sizeof(bufa)) { /* [same applies to b] */ + allocbufa=(decNumber *)malloc(needbytes); + allocbufb=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr || allocbufb==nullptr) { /* hopeless */ + status|=DEC_Insufficient_storage; + break;} + a=allocbufa; /* use the allocated spaces */ + b=allocbufb; /* .. */ + } + + /* copy rhs -> f, save exponent, and reduce so 0.1 <= f < 1 */ + uprv_decNumberCopy(f, rhs); + exp=f->exponent+f->digits; /* adjusted to Hull rules */ + f->exponent=-(f->digits); /* to range */ + + /* set up working context */ + uprv_decContextDefault(&workset, DEC_INIT_DECIMAL64); + workset.emax=DEC_MAX_EMAX; + workset.emin=DEC_MIN_EMIN; + + /* [Until further notice, no error is possible and status bits */ + /* (Rounded, etc.) should be ignored, not accumulated.] */ + + /* Calculate initial approximation, and allow for odd exponent */ + workset.digits=workp; /* p for initial calculation */ + t->bits=0; t->digits=3; + a->bits=0; a->digits=3; + if ((exp & 1)==0) { /* even exponent */ + /* Set t=0.259, a=0.819 */ + t->exponent=-3; + a->exponent=-3; + #if DECDPUN>=3 + t->lsu[0]=259; + a->lsu[0]=819; + #elif DECDPUN==2 + t->lsu[0]=59; t->lsu[1]=2; + a->lsu[0]=19; a->lsu[1]=8; + #else + t->lsu[0]=9; t->lsu[1]=5; t->lsu[2]=2; + a->lsu[0]=9; a->lsu[1]=1; a->lsu[2]=8; + #endif + } + else { /* odd exponent */ + /* Set t=0.0819, a=2.59 */ + f->exponent--; /* f=f/10 */ + exp++; /* e=e+1 */ + t->exponent=-4; + a->exponent=-2; + #if DECDPUN>=3 + t->lsu[0]=819; + a->lsu[0]=259; + #elif DECDPUN==2 + t->lsu[0]=19; t->lsu[1]=8; + a->lsu[0]=59; a->lsu[1]=2; + #else + t->lsu[0]=9; t->lsu[1]=1; t->lsu[2]=8; + a->lsu[0]=9; a->lsu[1]=5; a->lsu[2]=2; + #endif + } + + decMultiplyOp(a, a, f, &workset, &ignore); /* a=a*f */ + decAddOp(a, a, t, &workset, 0, &ignore); /* ..+t */ + /* [a is now the initial approximation for sqrt(f), calculated with */ + /* currentprecision, which is also a's precision.] */ + + /* the main calculation loop */ + uprv_decNumberZero(&dzero); /* make 0 */ + uprv_decNumberZero(t); /* set t = 0.5 */ + t->lsu[0]=5; /* .. */ + t->exponent=-1; /* .. */ + workset.digits=3; /* initial p */ + for (; workset.digitsexponent+=exp/2; /* set correct exponent */ + rstatus=0; /* clear status */ + residue=0; /* .. and accumulator */ + decCopyFit(a, a, &approxset, &residue, &rstatus); /* reduce (if needed) */ + decFinish(a, &approxset, &residue, &rstatus); /* clean and finalize */ + + /* Overflow was possible if the input exponent was out-of-range, */ + /* in which case quit */ + if (rstatus&DEC_Overflow) { + status=rstatus; /* use the status as-is */ + uprv_decNumberCopy(res, a); /* copy to result */ + break; + } + + /* Preserve status except Inexact/Rounded */ + status|=(rstatus & ~(DEC_Rounded|DEC_Inexact)); + + /* Carry out the Hull correction */ + a->exponent-=exp/2; /* back to 0.1->1 */ + + /* a is now at final precision and within 1 ulp of the properly */ + /* rounded square root of f; to ensure proper rounding, compare */ + /* squares of (a - l/2 ulp) and (a + l/2 ulp) with f. */ + /* Here workset.digits=maxp and t=0.5, and a->digits determines */ + /* the ulp */ + workset.digits--; /* maxp-1 is OK now */ + t->exponent=-a->digits-1; /* make 0.5 ulp */ + decAddOp(b, a, t, &workset, DECNEG, &ignore); /* b = a - 0.5 ulp */ + workset.round=DEC_ROUND_UP; + decMultiplyOp(b, b, b, &workset, &ignore); /* b = mulru(b, b) */ + decCompareOp(b, f, b, &workset, COMPARE, &ignore); /* b ? f, reversed */ + if (decNumberIsNegative(b)) { /* f < b [i.e., b > f] */ + /* this is the more common adjustment, though both are rare */ + t->exponent++; /* make 1.0 ulp */ + t->lsu[0]=1; /* .. */ + decAddOp(a, a, t, &workset, DECNEG, &ignore); /* a = a - 1 ulp */ + /* assign to approx [round to length] */ + approxset.emin-=exp/2; /* adjust to match a */ + approxset.emax-=exp/2; + decAddOp(a, &dzero, a, &approxset, 0, &ignore); + } + else { + decAddOp(b, a, t, &workset, 0, &ignore); /* b = a + 0.5 ulp */ + workset.round=DEC_ROUND_DOWN; + decMultiplyOp(b, b, b, &workset, &ignore); /* b = mulrd(b, b) */ + decCompareOp(b, b, f, &workset, COMPARE, &ignore); /* b ? f */ + if (decNumberIsNegative(b)) { /* b < f */ + t->exponent++; /* make 1.0 ulp */ + t->lsu[0]=1; /* .. */ + decAddOp(a, a, t, &workset, 0, &ignore); /* a = a + 1 ulp */ + /* assign to approx [round to length] */ + approxset.emin-=exp/2; /* adjust to match a */ + approxset.emax-=exp/2; + decAddOp(a, &dzero, a, &approxset, 0, &ignore); + } + } + /* [no errors are possible in the above, and rounding/inexact during */ + /* estimation are irrelevant, so status was not accumulated] */ + + /* Here, 0.1 <= a < 1 (still), so adjust back */ + a->exponent+=exp/2; /* set correct exponent */ + + /* count droppable zeros [after any subnormal rounding] by */ + /* trimming a copy */ + uprv_decNumberCopy(b, a); + decTrim(b, set, 1, 1, &dropped); /* [drops trailing zeros] */ + + /* Set Inexact and Rounded. The answer can only be exact if */ + /* it is short enough so that squaring it could fit in workp */ + /* digits, so this is the only (relatively rare) condition that */ + /* a careful check is needed */ + if (b->digits*2-1 > workp) { /* cannot fit */ + status|=DEC_Inexact|DEC_Rounded; + } + else { /* could be exact/unrounded */ + uInt mstatus=0; /* local status */ + decMultiplyOp(b, b, b, &workset, &mstatus); /* try the multiply */ + if (mstatus&DEC_Overflow) { /* result just won't fit */ + status|=DEC_Inexact|DEC_Rounded; + } + else { /* plausible */ + decCompareOp(t, b, rhs, &workset, COMPARE, &mstatus); /* b ? rhs */ + if (!ISZERO(t)) status|=DEC_Inexact|DEC_Rounded; /* not equal */ + else { /* is Exact */ + /* here, dropped is the count of trailing zeros in 'a' */ + /* use closest exponent to ideal... */ + Int todrop=ideal-a->exponent; /* most that can be dropped */ + if (todrop<0) status|=DEC_Rounded; /* ideally would add 0s */ + else { /* unrounded */ + /* there are some to drop, but emax may not allow all */ + Int maxexp=set->emax-set->digits+1; + Int maxdrop=maxexp-a->exponent; + if (todrop>maxdrop && set->clamp) { /* apply clamping */ + todrop=maxdrop; + status|=DEC_Clamped; + } + if (dropped0) { /* have some to drop */ + decShiftToLeast(a->lsu, D2U(a->digits), todrop); + a->exponent+=todrop; /* maintain numerical value */ + a->digits-=todrop; /* new length */ + } + } + } + } + } + + /* double-check Underflow, as perhaps the result could not have */ + /* been subnormal (initial argument too big), or it is now Exact */ + if (status&DEC_Underflow) { + Int ae=rhs->exponent+rhs->digits-1; /* adjusted exponent */ + /* check if truly subnormal */ + #if DECEXTFLAG /* DEC_Subnormal too */ + if (ae>=set->emin*2) status&=~(DEC_Subnormal|DEC_Underflow); + #else + if (ae>=set->emin*2) status&=~DEC_Underflow; + #endif + /* check if truly inexact */ + if (!(status&DEC_Inexact)) status&=~DEC_Underflow; + } + + uprv_decNumberCopy(res, a); /* a is now the result */ + } while(0); /* end protected */ + + if (allocbuff!=nullptr) free(allocbuff); /* drop any storage used */ + if (allocbufa!=nullptr) free(allocbufa); /* .. */ + if (allocbufb!=nullptr) free(allocbufb); /* .. */ + #if DECSUBSET + if (allocrhs !=nullptr) free(allocrhs); /* .. */ + #endif + if (status!=0) decStatus(res, status, set);/* then report status */ + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberSquareRoot */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic pop +#endif + +/* ------------------------------------------------------------------ */ +/* decNumberSubtract -- subtract two Numbers */ +/* */ +/* This computes C = A - B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X-X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* */ +/* C must have space for set->digits digits. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberSubtract(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + uInt status=0; /* accumulator */ + + decAddOp(res, lhs, rhs, set, DECNEG, &status); + if (status!=0) decStatus(res, status, set); + #if DECCHECK + decCheckInexact(res, set); + #endif + return res; + } /* decNumberSubtract */ + +/* ------------------------------------------------------------------ */ +/* decNumberToIntegralExact -- round-to-integral-value with InExact */ +/* decNumberToIntegralValue -- round-to-integral-value */ +/* */ +/* res is the result */ +/* rhs is input number */ +/* set is the context */ +/* */ +/* res must have space for any value of rhs. */ +/* */ +/* This implements the IEEE special operators and therefore treats */ +/* special values as valid. For finite numbers it returns */ +/* rescale(rhs, 0) if rhs->exponent is <0. */ +/* Otherwise the result is rhs (so no error is possible, except for */ +/* sNaN). */ +/* */ +/* The context is used for rounding mode and status after sNaN, but */ +/* the digits setting is ignored. The Exact version will signal */ +/* Inexact if the result differs numerically from rhs; the other */ +/* never signals Inexact. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralExact(decNumber *res, const decNumber *rhs, + decContext *set) { + decNumber dn; + decContext workset; /* working context */ + uInt status=0; /* accumulator */ + + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + /* handle infinities and NaNs */ + if (SPECIALARG) { + if (decNumberIsInfinite(rhs)) uprv_decNumberCopy(res, rhs); /* an Infinity */ + else decNaNs(res, rhs, nullptr, set, &status); /* a NaN */ + } + else { /* finite */ + /* have a finite number; no error possible (res must be big enough) */ + if (rhs->exponent>=0) return uprv_decNumberCopy(res, rhs); + /* that was easy, but if negative exponent there is work to do... */ + workset=*set; /* clone rounding, etc. */ + workset.digits=rhs->digits; /* no length rounding */ + workset.traps=0; /* no traps */ + uprv_decNumberZero(&dn); /* make a number with exponent 0 */ + uprv_decNumberQuantize(res, rhs, &dn, &workset); + status|=workset.status; + } + if (status!=0) decStatus(res, status, set); + return res; + } /* decNumberToIntegralExact */ + +U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralValue(decNumber *res, const decNumber *rhs, + decContext *set) { + decContext workset=*set; /* working context */ + workset.traps=0; /* no traps */ + uprv_decNumberToIntegralExact(res, rhs, &workset); + /* this never affects set, except for sNaNs; NaN will have been set */ + /* or propagated already, so no need to call decStatus */ + set->status|=workset.status&DEC_Invalid_operation; + return res; + } /* decNumberToIntegralValue */ + +/* ------------------------------------------------------------------ */ +/* decNumberXor -- XOR two Numbers, digitwise */ +/* */ +/* This computes C = A ^ B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X^X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context (used for result length and error report) */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Logical function restrictions apply (see above); a NaN is */ +/* returned with Invalid_operation if a restriction is violated. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberXor(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + const Unit *ua, *ub; /* -> operands */ + const Unit *msua, *msub; /* -> operand msus */ + Unit *uc, *msuc; /* -> result and its msu */ + Int msudigs; /* digits in res msu */ + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + if (lhs->exponent!=0 || decNumberIsSpecial(lhs) || decNumberIsNegative(lhs) + || rhs->exponent!=0 || decNumberIsSpecial(rhs) || decNumberIsNegative(rhs)) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + /* operands are valid */ + ua=lhs->lsu; /* bottom-up */ + ub=rhs->lsu; /* .. */ + uc=res->lsu; /* .. */ + msua=ua+D2U(lhs->digits)-1; /* -> msu of lhs */ + msub=ub+D2U(rhs->digits)-1; /* -> msu of rhs */ + msuc=uc+D2U(set->digits)-1; /* -> msu of result */ + msudigs=MSUDIGITS(set->digits); /* [faster than remainder] */ + for (; uc<=msuc; ua++, ub++, uc++) { /* Unit loop */ + Unit a, b; /* extract units */ + if (ua>msua) a=0; + else a=*ua; + if (ub>msub) b=0; + else b=*ub; + *uc=0; /* can now write back */ + if (a|b) { /* maybe 1 bits to examine */ + Int i, j; + /* This loop could be unrolled and/or use BIN2BCD tables */ + for (i=0; i1) { + decStatus(res, DEC_Invalid_operation, set); + return res; + } + if (uc==msuc && i==msudigs-1) break; /* just did final digit */ + } /* each digit */ + } /* non-zero */ + } /* each unit */ + /* [here uc-1 is the msu of the result] */ + res->digits=decGetDigits(res->lsu, static_cast(uc-res->lsu)); + res->exponent=0; /* integer */ + res->bits=0; /* sign=0 */ + return res; /* [no status to set] */ + } /* decNumberXor */ + + +/* ================================================================== */ +/* Utility routines */ +/* ================================================================== */ + +/* ------------------------------------------------------------------ */ +/* decNumberClass -- return the decClass of a decNumber */ +/* dn -- the decNumber to test */ +/* set -- the context to use for Emin */ +/* returns the decClass enum */ +/* ------------------------------------------------------------------ */ +enum decClass uprv_decNumberClass(const decNumber *dn, decContext *set) { + if (decNumberIsSpecial(dn)) { + if (decNumberIsQNaN(dn)) return DEC_CLASS_QNAN; + if (decNumberIsSNaN(dn)) return DEC_CLASS_SNAN; + /* must be an infinity */ + if (decNumberIsNegative(dn)) return DEC_CLASS_NEG_INF; + return DEC_CLASS_POS_INF; + } + /* is finite */ + if (uprv_decNumberIsNormal(dn, set)) { /* most common */ + if (decNumberIsNegative(dn)) return DEC_CLASS_NEG_NORMAL; + return DEC_CLASS_POS_NORMAL; + } + /* is subnormal or zero */ + if (decNumberIsZero(dn)) { /* most common */ + if (decNumberIsNegative(dn)) return DEC_CLASS_NEG_ZERO; + return DEC_CLASS_POS_ZERO; + } + if (decNumberIsNegative(dn)) return DEC_CLASS_NEG_SUBNORMAL; + return DEC_CLASS_POS_SUBNORMAL; + } /* decNumberClass */ + +/* ------------------------------------------------------------------ */ +/* decNumberClassToString -- convert decClass to a string */ +/* */ +/* eclass is a valid decClass */ +/* returns a constant string describing the class (max 13+1 chars) */ +/* ------------------------------------------------------------------ */ +const char *uprv_decNumberClassToString(enum decClass eclass) { + if (eclass==DEC_CLASS_POS_NORMAL) return DEC_ClassString_PN; + if (eclass==DEC_CLASS_NEG_NORMAL) return DEC_ClassString_NN; + if (eclass==DEC_CLASS_POS_ZERO) return DEC_ClassString_PZ; + if (eclass==DEC_CLASS_NEG_ZERO) return DEC_ClassString_NZ; + if (eclass==DEC_CLASS_POS_SUBNORMAL) return DEC_ClassString_PS; + if (eclass==DEC_CLASS_NEG_SUBNORMAL) return DEC_ClassString_NS; + if (eclass==DEC_CLASS_POS_INF) return DEC_ClassString_PI; + if (eclass==DEC_CLASS_NEG_INF) return DEC_ClassString_NI; + if (eclass==DEC_CLASS_QNAN) return DEC_ClassString_QN; + if (eclass==DEC_CLASS_SNAN) return DEC_ClassString_SN; + return DEC_ClassString_UN; /* Unknown */ + } /* decNumberClassToString */ + +/* ------------------------------------------------------------------ */ +/* decNumberCopy -- copy a number */ +/* */ +/* dest is the target decNumber */ +/* src is the source decNumber */ +/* returns dest */ +/* */ +/* (dest==src is allowed and is a no-op) */ +/* All fields are updated as required. This is a utility operation, */ +/* so special values are unchanged and no error is possible. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopy(decNumber *dest, const decNumber *src) { + + #if DECCHECK + if (src==nullptr) return uprv_decNumberZero(dest); + #endif + + if (dest==src) return dest; /* no copy required */ + + /* Use explicit assignments here as structure assignment could copy */ + /* more than just the lsu (for small DECDPUN). This would not affect */ + /* the value of the results, but could disturb test harness spill */ + /* checking. */ + dest->bits=src->bits; + dest->exponent=src->exponent; + dest->digits=src->digits; + dest->lsu[0]=src->lsu[0]; + if (src->digits>DECDPUN) { /* more Units to come */ + const Unit *smsup, *s; /* work */ + Unit *d; /* .. */ + /* memcpy for the remaining Units would be safe as they cannot */ + /* overlap. However, this explicit loop is faster in short cases. */ + d=dest->lsu+1; /* -> first destination */ + smsup=src->lsu+D2U(src->digits); /* -> source msu+1 */ + for (s=src->lsu+1; sdigits digits. */ +/* No exception or error can occur; this is a quiet bitwise operation.*/ +/* See also decNumberAbs for a checking version of this. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyAbs(decNumber *res, const decNumber *rhs) { + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, DECUNCONT)) return res; + #endif + uprv_decNumberCopy(res, rhs); + res->bits&=~DECNEG; /* turn off sign */ + return res; + } /* decNumberCopyAbs */ + +/* ------------------------------------------------------------------ */ +/* decNumberCopyNegate -- quiet negate value operator */ +/* */ +/* This sets C = negate(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* */ +/* C must have space for set->digits digits. */ +/* No exception or error can occur; this is a quiet bitwise operation.*/ +/* See also decNumberMinus for a checking version of this. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyNegate(decNumber *res, const decNumber *rhs) { + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, DECUNCONT)) return res; + #endif + uprv_decNumberCopy(res, rhs); + res->bits^=DECNEG; /* invert the sign */ + return res; + } /* decNumberCopyNegate */ + +/* ------------------------------------------------------------------ */ +/* decNumberCopySign -- quiet copy and set sign operator */ +/* */ +/* This sets C = A with the sign of B */ +/* */ +/* res is C, the result. C may be A */ +/* lhs is A */ +/* rhs is B */ +/* */ +/* C must have space for set->digits digits. */ +/* No exception or error can occur; this is a quiet bitwise operation.*/ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopySign(decNumber *res, const decNumber *lhs, + const decNumber *rhs) { + uByte sign; /* rhs sign */ + #if DECCHECK + if (decCheckOperands(res, DECUNUSED, rhs, DECUNCONT)) return res; + #endif + sign=rhs->bits & DECNEG; /* save sign bit */ + uprv_decNumberCopy(res, lhs); + res->bits&=~DECNEG; /* clear the sign */ + res->bits|=sign; /* set from rhs */ + return res; + } /* decNumberCopySign */ + +/* ------------------------------------------------------------------ */ +/* decNumberGetBCD -- get the coefficient in BCD8 */ +/* dn is the source decNumber */ +/* bcd is the uInt array that will receive dn->digits BCD bytes, */ +/* most-significant at offset 0 */ +/* returns bcd */ +/* */ +/* bcd must have at least dn->digits bytes. No error is possible; if */ +/* dn is a NaN or Infinite, digits must be 1 and the coefficient 0. */ +/* ------------------------------------------------------------------ */ +U_CAPI uByte * U_EXPORT2 uprv_decNumberGetBCD(const decNumber *dn, uByte *bcd) { + uByte *ub=bcd+dn->digits-1; /* -> lsd */ + const Unit *up=dn->lsu; /* Unit pointer, -> lsu */ + + #if DECDPUN==1 /* trivial simple copy */ + for (; ub>=bcd; ub--, up++) *ub=*up; + #else /* chopping needed */ + uInt u=*up; /* work */ + uInt cut=DECDPUN; /* downcounter through unit */ + for (; ub>=bcd; ub--) { + *ub=(uByte)(u%10); /* [*6554 trick inhibits, here] */ + u=u/10; + cut--; + if (cut>0) continue; /* more in this unit */ + up++; + u=*up; + cut=DECDPUN; + } + #endif + return bcd; + } /* decNumberGetBCD */ + +/* ------------------------------------------------------------------ */ +/* decNumberSetBCD -- set (replace) the coefficient from BCD8 */ +/* dn is the target decNumber */ +/* bcd is the uInt array that will source n BCD bytes, most- */ +/* significant at offset 0 */ +/* n is the number of digits in the source BCD array (bcd) */ +/* returns dn */ +/* */ +/* dn must have space for at least n digits. No error is possible; */ +/* if dn is a NaN, or Infinite, or is to become a zero, n must be 1 */ +/* and bcd[0] zero. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberSetBCD(decNumber *dn, const uByte *bcd, uInt n) { + Unit *up=dn->lsu+D2U(dn->digits)-1; /* -> msu [target pointer] */ + const uByte *ub=bcd; /* -> source msd */ + + #if DECDPUN==1 /* trivial simple copy */ + for (; ub=dn->lsu; up--) { /* each Unit from msu */ + *up=0; /* will take <=DECDPUN digits */ + for (; cut>0; ub++, cut--) *up=X10(*up)+*ub; + cut=DECDPUN; /* next Unit has all digits */ + } + #endif + dn->digits=n; /* set digit count */ + return dn; + } /* decNumberSetBCD */ + +/* ------------------------------------------------------------------ */ +/* decNumberIsNormal -- test normality of a decNumber */ +/* dn is the decNumber to test */ +/* set is the context to use for Emin */ +/* returns 1 if |dn| is finite and >=Nmin, 0 otherwise */ +/* ------------------------------------------------------------------ */ +Int uprv_decNumberIsNormal(const decNumber *dn, decContext *set) { + Int ae; /* adjusted exponent */ + #if DECCHECK + if (decCheckOperands(DECUNRESU, DECUNUSED, dn, set)) return 0; + #endif + + if (decNumberIsSpecial(dn)) return 0; /* not finite */ + if (decNumberIsZero(dn)) return 0; /* not non-zero */ + + ae=dn->exponent+dn->digits-1; /* adjusted exponent */ + if (aeemin) return 0; /* is subnormal */ + return 1; + } /* decNumberIsNormal */ + +/* ------------------------------------------------------------------ */ +/* decNumberIsSubnormal -- test subnormality of a decNumber */ +/* dn is the decNumber to test */ +/* set is the context to use for Emin */ +/* returns 1 if |dn| is finite, non-zero, and exponent+dn->digits-1; /* adjusted exponent */ + if (aeemin) return 1; /* is subnormal */ + return 0; + } /* decNumberIsSubnormal */ + +/* ------------------------------------------------------------------ */ +/* decNumberTrim -- remove insignificant zeros */ +/* */ +/* dn is the number to trim */ +/* returns dn */ +/* */ +/* All fields are updated as required. This is a utility operation, */ +/* so special values are unchanged and no error is possible. The */ +/* zeros are removed unconditionally. */ +/* ------------------------------------------------------------------ */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberTrim(decNumber *dn) { + Int dropped; /* work */ + decContext set; /* .. */ + #if DECCHECK + if (decCheckOperands(DECUNRESU, DECUNUSED, dn, DECUNCONT)) return dn; + #endif + uprv_decContextDefault(&set, DEC_INIT_BASE); /* clamp=0 */ + return decTrim(dn, &set, 0, 1, &dropped); + } /* decNumberTrim */ + +/* ------------------------------------------------------------------ */ +/* decNumberVersion -- return the name and version of this module */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +const char * uprv_decNumberVersion() { + return DECVERSION; + } /* decNumberVersion */ + +/* ------------------------------------------------------------------ */ +/* decNumberZero -- set a number to 0 */ +/* */ +/* dn is the number to set, with space for one digit */ +/* returns dn */ +/* */ +/* No error is possible. */ +/* ------------------------------------------------------------------ */ +/* Memset is not used as it is much slower in some environments. */ +U_CAPI decNumber * U_EXPORT2 uprv_decNumberZero(decNumber *dn) { + + #if DECCHECK + if (decCheckOperands(dn, DECUNUSED, DECUNUSED, DECUNCONT)) return dn; + #endif + + dn->bits=0; + dn->exponent=0; + dn->digits=1; + dn->lsu[0]=0; + return dn; + } /* decNumberZero */ + +/* ================================================================== */ +/* Local routines */ +/* ================================================================== */ + +/* ------------------------------------------------------------------ */ +/* decToString -- lay out a number into a string */ +/* */ +/* dn is the number to lay out */ +/* string is where to lay out the number */ +/* eng is 1 if Engineering, 0 if Scientific */ +/* */ +/* string must be at least dn->digits+14 characters long */ +/* No error is possible. */ +/* */ +/* Note that this routine can generate a -0 or 0.000. These are */ +/* never generated in subset to-number or arithmetic, but can occur */ +/* in non-subset arithmetic (e.g., -1*0 or 1.234-1.234). */ +/* ------------------------------------------------------------------ */ +/* If DECCHECK is enabled the string "?" is returned if a number is */ +/* invalid. */ +static void decToString(const decNumber *dn, char *string, Flag eng) { + Int exp=dn->exponent; /* local copy */ + Int e; /* E-part value */ + Int pre; /* digits before the '.' */ + Int cut; /* for counting digits in a Unit */ + char *c=string; /* work [output pointer] */ + const Unit *up=dn->lsu+D2U(dn->digits)-1; /* -> msu [input pointer] */ + uInt u, pow; /* work */ + + #if DECCHECK + if (decCheckOperands(DECUNRESU, dn, DECUNUSED, DECUNCONT)) { + strcpy(string, "?"); + return;} + #endif + + if (decNumberIsNegative(dn)) { /* Negatives get a minus */ + *c='-'; + c++; + } + if (dn->bits&DECSPECIAL) { /* Is a special value */ + if (decNumberIsInfinite(dn)) { + strcpy(c, "Inf"); + strcpy(c+3, "inity"); + return;} + /* a NaN */ + if (dn->bits&DECSNAN) { /* signalling NaN */ + *c='s'; + c++; + } + strcpy(c, "NaN"); + c+=3; /* step past */ + /* if not a clean non-zero coefficient, that's all there is in a */ + /* NaN string */ + if (exp!=0 || (*dn->lsu==0 && dn->digits==1)) return; + /* [drop through to add integer] */ + } + + /* calculate how many digits in msu, and hence first cut */ + cut=MSUDIGITS(dn->digits); /* [faster than remainder] */ + cut--; /* power of ten for digit */ + + if (exp==0) { /* simple integer [common fastpath] */ + for (;up>=dn->lsu; up--) { /* each Unit from msu */ + u=*up; /* contains DECDPUN digits to lay out */ + for (; cut>=0; c++, cut--) TODIGIT(u, cut, c, pow); + cut=DECDPUN-1; /* next Unit has all digits */ + } + *c='\0'; /* terminate the string */ + return;} + + /* non-0 exponent -- assume plain form */ + pre=dn->digits+exp; /* digits before '.' */ + e=0; /* no E */ + if ((exp>0) || (pre<-5)) { /* need exponential form */ + e=exp+dn->digits-1; /* calculate E value */ + pre=1; /* assume one digit before '.' */ + if (eng && (e!=0)) { /* engineering: may need to adjust */ + Int adj; /* adjustment */ + /* The C remainder operator is undefined for negative numbers, so */ + /* a positive remainder calculation must be used here */ + if (e<0) { + adj=(-e)%3; + if (adj!=0) adj=3-adj; + } + else { /* e>0 */ + adj=e%3; + } + e=e-adj; + /* if dealing with zero still produce an exponent which is a */ + /* multiple of three, as expected, but there will only be the */ + /* one zero before the E, still. Otherwise note the padding. */ + if (!ISZERO(dn)) pre+=adj; + else { /* is zero */ + if (adj!=0) { /* 0.00Esnn needed */ + e=e+3; + pre=-(2-adj); + } + } /* zero */ + } /* eng */ + } /* need exponent */ + + /* lay out the digits of the coefficient, adding 0s and . as needed */ + u=*up; + if (pre>0) { /* xxx.xxx or xx00 (engineering) form */ + Int n=pre; + for (; pre>0; pre--, c++, cut--) { + if (cut<0) { /* need new Unit */ + if (up==dn->lsu) break; /* out of input digits (pre>digits) */ + up--; + cut=DECDPUN-1; + u=*up; + } + TODIGIT(u, cut, c, pow); + } + if (ndigits) { /* more to come, after '.' */ + *c='.'; c++; + for (;; c++, cut--) { + if (cut<0) { /* need new Unit */ + if (up==dn->lsu) break; /* out of input digits */ + up--; + cut=DECDPUN-1; + u=*up; + } + TODIGIT(u, cut, c, pow); + } + } + else for (; pre>0; pre--, c++) *c='0'; /* 0 padding (for engineering) needed */ + } + else { /* 0.xxx or 0.000xxx form */ + *c='0'; c++; + *c='.'; c++; + for (; pre<0; pre++, c++) *c='0'; /* add any 0's after '.' */ + for (; ; c++, cut--) { + if (cut<0) { /* need new Unit */ + if (up==dn->lsu) break; /* out of input digits */ + up--; + cut=DECDPUN-1; + u=*up; + } + TODIGIT(u, cut, c, pow); + } + } + + /* Finally add the E-part, if needed. It will never be 0, has a + base maximum and minimum of +999999999 through -999999999, but + could range down to -1999999998 for abnormal numbers */ + if (e!=0) { + Flag had=0; /* 1=had non-zero */ + *c='E'; c++; + *c='+'; c++; /* assume positive */ + u=e; /* .. */ + if (e<0) { + *(c-1)='-'; /* oops, need - */ + u=-e; /* uInt, please */ + } + /* lay out the exponent [_itoa or equivalent is not ANSI C] */ + for (cut=9; cut>=0; cut--) { + TODIGIT(u, cut, c, pow); + if (*c=='0' && !had) continue; /* skip leading zeros */ + had=1; /* had non-0 */ + c++; /* step for next */ + } /* cut */ + } + *c='\0'; /* terminate the string (all paths) */ + return; + } /* decToString */ + +/* ------------------------------------------------------------------ */ +/* decAddOp -- add/subtract operation */ +/* */ +/* This computes C = A + B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X+X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* negate is DECNEG if rhs should be negated, or 0 otherwise */ +/* status accumulates status for the caller */ +/* */ +/* C must have space for set->digits digits. */ +/* Inexact in status must be 0 for correct Exact zero sign in result */ +/* ------------------------------------------------------------------ */ +/* If possible, the coefficient is calculated directly into C. */ +/* However, if: */ +/* -- a digits+1 calculation is needed because the numbers are */ +/* unaligned and span more than set->digits digits */ +/* -- a carry to digits+1 digits looks possible */ +/* -- C is the same as A or B, and the result would destructively */ +/* overlap the A or B coefficient */ +/* then the result must be calculated into a temporary buffer. In */ +/* this case a local (stack) buffer is used if possible, and only if */ +/* too long for that does malloc become the final resort. */ +/* */ +/* Misalignment is handled as follows: */ +/* Apad: (AExp>BExp) Swap operands and proceed as for BExp>AExp. */ +/* BPad: Apply the padding by a combination of shifting (whole */ +/* units) and multiplication (part units). */ +/* */ +/* Addition, especially x=x+1, is speed-critical. */ +/* The static buffer is larger than might be expected to allow for */ +/* calls from higher-level functions (notable exp). */ +/* ------------------------------------------------------------------ */ +static decNumber * decAddOp(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set, + uByte negate, uInt *status) { + #if DECSUBSET + decNumber *alloclhs=nullptr; /* non-nullptr if rounded lhs allocated */ + decNumber *allocrhs=nullptr; /* .., rhs */ + #endif + Int rhsshift; /* working shift (in Units) */ + Int maxdigits; /* longest logical length */ + Int mult; /* multiplier */ + Int residue; /* rounding accumulator */ + uByte bits; /* result bits */ + Flag diffsign; /* non-0 if arguments have different sign */ + Unit *acc; /* accumulator for result */ + Unit accbuff[SD2U(DECBUFFER*2+20)]; /* local buffer [*2+20 reduces many */ + /* allocations when called from */ + /* other operations, notable exp] */ + Unit *allocacc=nullptr; /* -> allocated acc buffer, iff allocated */ + Int reqdigits=set->digits; /* local copy; requested DIGITS */ + Int padding; /* work */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operands and set lostDigits status, as needed */ + if (lhs->digits>reqdigits) { + alloclhs=decRoundOperand(lhs, set, status); + if (alloclhs==nullptr) break; + lhs=alloclhs; + } + if (rhs->digits>reqdigits) { + allocrhs=decRoundOperand(rhs, set, status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* note whether signs differ [used all paths] */ + diffsign=(Flag)((lhs->bits^rhs->bits^negate)&DECNEG); + + /* handle infinities and NaNs */ + if (SPECIALARGS) { /* a special bit set */ + if (SPECIALARGS & (DECSNAN | DECNAN)) /* a NaN */ + decNaNs(res, lhs, rhs, set, status); + else { /* one or two infinities */ + if (decNumberIsInfinite(lhs)) { /* LHS is infinity */ + /* two infinities with different signs is invalid */ + if (decNumberIsInfinite(rhs) && diffsign) { + *status|=DEC_Invalid_operation; + break; + } + bits=lhs->bits & DECNEG; /* get sign from LHS */ + } + else bits=(rhs->bits^negate) & DECNEG;/* RHS must be Infinity */ + bits|=DECINF; + uprv_decNumberZero(res); + res->bits=bits; /* set +/- infinity */ + } /* an infinity */ + break; + } + + /* Quick exit for add 0s; return the non-0, modified as need be */ + if (ISZERO(lhs)) { + Int adjust; /* work */ + Int lexp=lhs->exponent; /* save in case LHS==RES */ + bits=lhs->bits; /* .. */ + residue=0; /* clear accumulator */ + decCopyFit(res, rhs, set, &residue, status); /* copy (as needed) */ + res->bits^=negate; /* flip if rhs was negated */ + #if DECSUBSET + if (set->extended) { /* exponents on zeros count */ + #endif + /* exponent will be the lower of the two */ + adjust=lexp-res->exponent; /* adjustment needed [if -ve] */ + if (ISZERO(res)) { /* both 0: special IEEE 754 rules */ + if (adjust<0) res->exponent=lexp; /* set exponent */ + /* 0-0 gives +0 unless rounding to -infinity, and -0-0 gives -0 */ + if (diffsign) { + if (set->round!=DEC_ROUND_FLOOR) res->bits=0; + else res->bits=DECNEG; /* preserve 0 sign */ + } + } + else { /* non-0 res */ + if (adjust<0) { /* 0-padding needed */ + if ((res->digits-adjust)>set->digits) { + adjust=res->digits-set->digits; /* to fit exactly */ + *status|=DEC_Rounded; /* [but exact] */ + } + res->digits=decShiftToMost(res->lsu, res->digits, -adjust); + res->exponent+=adjust; /* set the exponent. */ + } + } /* non-0 res */ + #if DECSUBSET + } /* extended */ + #endif + decFinish(res, set, &residue, status); /* clean and finalize */ + break;} + + if (ISZERO(rhs)) { /* [lhs is non-zero] */ + Int adjust; /* work */ + Int rexp=rhs->exponent; /* save in case RHS==RES */ + bits=rhs->bits; /* be clean */ + residue=0; /* clear accumulator */ + decCopyFit(res, lhs, set, &residue, status); /* copy (as needed) */ + #if DECSUBSET + if (set->extended) { /* exponents on zeros count */ + #endif + /* exponent will be the lower of the two */ + /* [0-0 case handled above] */ + adjust=rexp-res->exponent; /* adjustment needed [if -ve] */ + if (adjust<0) { /* 0-padding needed */ + if ((res->digits-adjust)>set->digits) { + adjust=res->digits-set->digits; /* to fit exactly */ + *status|=DEC_Rounded; /* [but exact] */ + } + res->digits=decShiftToMost(res->lsu, res->digits, -adjust); + res->exponent+=adjust; /* set the exponent. */ + } + #if DECSUBSET + } /* extended */ + #endif + decFinish(res, set, &residue, status); /* clean and finalize */ + break;} + + /* [NB: both fastpath and mainpath code below assume these cases */ + /* (notably 0-0) have already been handled] */ + + /* calculate the padding needed to align the operands */ + padding=rhs->exponent-lhs->exponent; + + /* Fastpath cases where the numbers are aligned and normal, the RHS */ + /* is all in one unit, no operand rounding is needed, and no carry, */ + /* lengthening, or borrow is needed */ + if (padding==0 + && rhs->digits<=DECDPUN + && rhs->exponent>=set->emin /* [some normals drop through] */ + && rhs->exponent<=set->emax-set->digits+1 /* [could clamp] */ + && rhs->digits<=reqdigits + && lhs->digits<=reqdigits) { + Int partial=*lhs->lsu; + if (!diffsign) { /* adding */ + partial+=*rhs->lsu; + if ((partial<=DECDPUNMAX) /* result fits in unit */ + && (lhs->digits>=DECDPUN || /* .. and no digits-count change */ + partial<(Int)powers[lhs->digits])) { /* .. */ + if (res!=lhs) uprv_decNumberCopy(res, lhs); /* not in place */ + *res->lsu=(Unit)partial; /* [copy could have overwritten RHS] */ + break; + } + /* else drop out for careful add */ + } + else { /* signs differ */ + partial-=*rhs->lsu; + if (partial>0) { /* no borrow needed, and non-0 result */ + if (res!=lhs) uprv_decNumberCopy(res, lhs); /* not in place */ + *res->lsu=(Unit)partial; + /* this could have reduced digits [but result>0] */ + res->digits=decGetDigits(res->lsu, D2U(res->digits)); + break; + } + /* else drop out for careful subtract */ + } + } + + /* Now align (pad) the lhs or rhs so they can be added or */ + /* subtracted, as necessary. If one number is much larger than */ + /* the other (that is, if in plain form there is a least one */ + /* digit between the lowest digit of one and the highest of the */ + /* other) padding with up to DIGITS-1 trailing zeros may be */ + /* needed; then apply rounding (as exotic rounding modes may be */ + /* affected by the residue). */ + rhsshift=0; /* rhs shift to left (padding) in Units */ + bits=lhs->bits; /* assume sign is that of LHS */ + mult=1; /* likely multiplier */ + + /* [if padding==0 the operands are aligned; no padding is needed] */ + if (padding!=0) { + /* some padding needed; always pad the RHS, as any required */ + /* padding can then be effected by a simple combination of */ + /* shifts and a multiply */ + Flag swapped=0; + if (padding<0) { /* LHS needs the padding */ + const decNumber *t; + padding=-padding; /* will be +ve */ + bits=(uByte)(rhs->bits^negate); /* assumed sign is now that of RHS */ + t=lhs; lhs=rhs; rhs=t; + swapped=1; + } + + /* If, after pad, rhs would be longer than lhs by digits+1 or */ + /* more then lhs cannot affect the answer, except as a residue, */ + /* so only need to pad up to a length of DIGITS+1. */ + if (rhs->digits+padding > lhs->digits+reqdigits+1) { + /* The RHS is sufficient */ + /* for residue use the relative sign indication... */ + Int shift=reqdigits-rhs->digits; /* left shift needed */ + residue=1; /* residue for rounding */ + if (diffsign) residue=-residue; /* signs differ */ + /* copy, shortening if necessary */ + decCopyFit(res, rhs, set, &residue, status); + /* if it was already shorter, then need to pad with zeros */ + if (shift>0) { + res->digits=decShiftToMost(res->lsu, res->digits, shift); + res->exponent-=shift; /* adjust the exponent. */ + } + /* flip the result sign if unswapped and rhs was negated */ + if (!swapped) res->bits^=negate; + decFinish(res, set, &residue, status); /* done */ + break;} + + /* LHS digits may affect result */ + rhsshift=D2U(padding+1)-1; /* this much by Unit shift .. */ + mult=powers[padding-(rhsshift*DECDPUN)]; /* .. this by multiplication */ + } /* padding needed */ + + if (diffsign) mult=-mult; /* signs differ */ + + /* determine the longer operand */ + maxdigits=rhs->digits+padding; /* virtual length of RHS */ + if (lhs->digits>maxdigits) maxdigits=lhs->digits; + + /* Decide on the result buffer to use; if possible place directly */ + /* into result. */ + acc=res->lsu; /* assume add direct to result */ + /* If destructive overlap, or the number is too long, or a carry or */ + /* borrow to DIGITS+1 might be possible, a buffer must be used. */ + /* [Might be worth more sophisticated tests when maxdigits==reqdigits] */ + if ((maxdigits>=reqdigits) /* is, or could be, too large */ + || (res==rhs && rhsshift>0)) { /* destructive overlap */ + /* buffer needed, choose it; units for maxdigits digits will be */ + /* needed, +1 Unit for carry or borrow */ + Int need=D2U(maxdigits)+1; + acc=accbuff; /* assume use local buffer */ + if (need*sizeof(Unit)>sizeof(accbuff)) { + /* printf("malloc add %ld %ld\n", need, sizeof(accbuff)); */ + allocacc=(Unit *)malloc(need*sizeof(Unit)); + if (allocacc==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + acc=allocacc; + } + } + + res->bits=(uByte)(bits&DECNEG); /* it's now safe to overwrite.. */ + res->exponent=lhs->exponent; /* .. operands (even if aliased) */ + + #if DECTRACE + decDumpAr('A', lhs->lsu, D2U(lhs->digits)); + decDumpAr('B', rhs->lsu, D2U(rhs->digits)); + printf(" :h: %ld %ld\n", rhsshift, mult); + #endif + + /* add [A+B*m] or subtract [A+B*(-m)] */ + U_ASSERT(rhs->digits > 0); + U_ASSERT(lhs->digits > 0); + res->digits=decUnitAddSub(lhs->lsu, D2U(lhs->digits), + rhs->lsu, D2U(rhs->digits), + rhsshift, acc, mult) + *DECDPUN; /* [units -> digits] */ + if (res->digits<0) { /* borrowed... */ + res->digits=-res->digits; + res->bits^=DECNEG; /* flip the sign */ + } + #if DECTRACE + decDumpAr('+', acc, D2U(res->digits)); + #endif + + /* If a buffer was used the result must be copied back, possibly */ + /* shortening. (If no buffer was used then the result must have */ + /* fit, so can't need rounding and residue must be 0.) */ + residue=0; /* clear accumulator */ + if (acc!=res->lsu) { + #if DECSUBSET + if (set->extended) { /* round from first significant digit */ + #endif + /* remove leading zeros that were added due to rounding up to */ + /* integral Units -- before the test for rounding. */ + if (res->digits>reqdigits) + res->digits=decGetDigits(acc, D2U(res->digits)); + decSetCoeff(res, set, acc, res->digits, &residue, status); + #if DECSUBSET + } + else { /* subset arithmetic rounds from original significant digit */ + /* May have an underestimate. This only occurs when both */ + /* numbers fit in DECDPUN digits and are padding with a */ + /* negative multiple (-10, -100...) and the top digit(s) become */ + /* 0. (This only matters when using X3.274 rules where the */ + /* leading zero could be included in the rounding.) */ + if (res->digitsdigits))=0; /* ensure leading 0 is there */ + res->digits=maxdigits; + } + else { + /* remove leading zeros that added due to rounding up to */ + /* integral Units (but only those in excess of the original */ + /* maxdigits length, unless extended) before test for rounding. */ + if (res->digits>reqdigits) { + res->digits=decGetDigits(acc, D2U(res->digits)); + if (res->digitsdigits=maxdigits; + } + } + decSetCoeff(res, set, acc, res->digits, &residue, status); + /* Now apply rounding if needed before removing leading zeros. */ + /* This is safe because subnormals are not a possibility */ + if (residue!=0) { + decApplyRound(res, set, residue, status); + residue=0; /* did what needed to be done */ + } + } /* subset */ + #endif + } /* used buffer */ + + /* strip leading zeros [these were left on in case of subset subtract] */ + res->digits=decGetDigits(res->lsu, D2U(res->digits)); + + /* apply checks and rounding */ + decFinish(res, set, &residue, status); + + /* "When the sum of two operands with opposite signs is exactly */ + /* zero, the sign of that sum shall be '+' in all rounding modes */ + /* except round toward -Infinity, in which mode that sign shall be */ + /* '-'." [Subset zeros also never have '-', set by decFinish.] */ + if (ISZERO(res) && diffsign + #if DECSUBSET + && set->extended + #endif + && (*status&DEC_Inexact)==0) { + if (set->round==DEC_ROUND_FLOOR) res->bits|=DECNEG; /* sign - */ + else res->bits&=~DECNEG; /* sign + */ + } + } while(0); /* end protected */ + + if (allocacc!=nullptr) free(allocacc); /* drop any storage used */ + #if DECSUBSET + if (allocrhs!=nullptr) free(allocrhs); /* .. */ + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + #endif + return res; + } /* decAddOp */ + +/* ------------------------------------------------------------------ */ +/* decDivideOp -- division operation */ +/* */ +/* This routine performs the calculations for all four division */ +/* operators (divide, divideInteger, remainder, remainderNear). */ +/* */ +/* C=A op B */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X/X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* op is DIVIDE, DIVIDEINT, REMAINDER, or REMNEAR respectively. */ +/* status is the usual accumulator */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* ------------------------------------------------------------------ */ +/* The underlying algorithm of this routine is the same as in the */ +/* 1981 S/370 implementation, that is, non-restoring long division */ +/* with bi-unit (rather than bi-digit) estimation for each unit */ +/* multiplier. In this pseudocode overview, complications for the */ +/* Remainder operators and division residues for exact rounding are */ +/* omitted for clarity. */ +/* */ +/* Prepare operands and handle special values */ +/* Test for x/0 and then 0/x */ +/* Exp =Exp1 - Exp2 */ +/* Exp =Exp +len(var1) -len(var2) */ +/* Sign=Sign1 * Sign2 */ +/* Pad accumulator (Var1) to double-length with 0's (pad1) */ +/* Pad Var2 to same length as Var1 */ +/* msu2pair/plus=1st 2 or 1 units of var2, +1 to allow for round */ +/* have=0 */ +/* Do until (have=digits+1 OR residue=0) */ +/* if exp<0 then if integer divide/residue then leave */ +/* this_unit=0 */ +/* Do forever */ +/* compare numbers */ +/* if <0 then leave inner_loop */ +/* if =0 then (* quick exit without subtract *) do */ +/* this_unit=this_unit+1; output this_unit */ +/* leave outer_loop; end */ +/* Compare lengths of numbers (mantissae): */ +/* If same then tops2=msu2pair -- {units 1&2 of var2} */ +/* else tops2=msu2plus -- {0, unit 1 of var2} */ +/* tops1=first_unit_of_Var1*10**DECDPUN +second_unit_of_var1 */ +/* mult=tops1/tops2 -- Good and safe guess at divisor */ +/* if mult=0 then mult=1 */ +/* this_unit=this_unit+mult */ +/* subtract */ +/* end inner_loop */ +/* if have\=0 | this_unit\=0 then do */ +/* output this_unit */ +/* have=have+1; end */ +/* var2=var2/10 */ +/* exp=exp-1 */ +/* end outer_loop */ +/* exp=exp+1 -- set the proper exponent */ +/* if have=0 then generate answer=0 */ +/* Return (Result is defined by Var1) */ +/* */ +/* ------------------------------------------------------------------ */ +/* Two working buffers are needed during the division; one (digits+ */ +/* 1) to accumulate the result, and the other (up to 2*digits+1) for */ +/* long subtractions. These are acc and var1 respectively. */ +/* var1 is a copy of the lhs coefficient, var2 is the rhs coefficient.*/ +/* The static buffers may be larger than might be expected to allow */ +/* for calls from higher-level functions (notable exp). */ +/* ------------------------------------------------------------------ */ +static decNumber * decDivideOp(decNumber *res, + const decNumber *lhs, const decNumber *rhs, + decContext *set, Flag op, uInt *status) { + #if DECSUBSET + decNumber *alloclhs=nullptr; /* non-nullptr if rounded lhs allocated */ + decNumber *allocrhs=nullptr; /* .., rhs */ + #endif + Unit accbuff[SD2U(DECBUFFER+DECDPUN+10)]; /* local buffer */ + Unit *acc=accbuff; /* -> accumulator array for result */ + Unit *allocacc=nullptr; /* -> allocated buffer, iff allocated */ + Unit *accnext; /* -> where next digit will go */ + Int acclength; /* length of acc needed [Units] */ + Int accunits; /* count of units accumulated */ + Int accdigits; /* count of digits accumulated */ + + Unit varbuff[SD2U(DECBUFFER*2+DECDPUN)]; /* buffer for var1 */ + Unit *var1=varbuff; /* -> var1 array for long subtraction */ + Unit *varalloc=nullptr; /* -> allocated buffer, iff used */ + Unit *msu1; /* -> msu of var1 */ + + const Unit *var2; /* -> var2 array */ + const Unit *msu2; /* -> msu of var2 */ + Int msu2plus; /* msu2 plus one [does not vary] */ + eInt msu2pair; /* msu2 pair plus one [does not vary] */ + + Int var1units, var2units; /* actual lengths */ + Int var2ulen; /* logical length (units) */ + Int var1initpad=0; /* var1 initial padding (digits) */ + Int maxdigits; /* longest LHS or required acc length */ + Int mult; /* multiplier for subtraction */ + Unit thisunit; /* current unit being accumulated */ + Int residue; /* for rounding */ + Int reqdigits=set->digits; /* requested DIGITS */ + Int exponent; /* working exponent */ + Int maxexponent=0; /* DIVIDE maximum exponent if unrounded */ + uByte bits; /* working sign */ + Unit *target; /* work */ + const Unit *source; /* .. */ + uInt const *pow; /* .. */ + Int shift, cut; /* .. */ + #if DECSUBSET + Int dropped; /* work */ + #endif + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operands and set lostDigits status, as needed */ + if (lhs->digits>reqdigits) { + alloclhs=decRoundOperand(lhs, set, status); + if (alloclhs==nullptr) break; + lhs=alloclhs; + } + if (rhs->digits>reqdigits) { + allocrhs=decRoundOperand(rhs, set, status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + bits=(lhs->bits^rhs->bits)&DECNEG; /* assumed sign for divisions */ + + /* handle infinities and NaNs */ + if (SPECIALARGS) { /* a special bit set */ + if (SPECIALARGS & (DECSNAN | DECNAN)) { /* one or two NaNs */ + decNaNs(res, lhs, rhs, set, status); + break; + } + /* one or two infinities */ + if (decNumberIsInfinite(lhs)) { /* LHS (dividend) is infinite */ + if (decNumberIsInfinite(rhs) || /* two infinities are invalid .. */ + op & (REMAINDER | REMNEAR)) { /* as is remainder of infinity */ + *status|=DEC_Invalid_operation; + break; + } + /* [Note that infinity/0 raises no exceptions] */ + uprv_decNumberZero(res); + res->bits=bits|DECINF; /* set +/- infinity */ + break; + } + else { /* RHS (divisor) is infinite */ + residue=0; + if (op&(REMAINDER|REMNEAR)) { + /* result is [finished clone of] lhs */ + decCopyFit(res, lhs, set, &residue, status); + } + else { /* a division */ + uprv_decNumberZero(res); + res->bits=bits; /* set +/- zero */ + /* for DIVIDEINT the exponent is always 0. For DIVIDE, result */ + /* is a 0 with infinitely negative exponent, clamped to minimum */ + if (op&DIVIDE) { + res->exponent=set->emin-set->digits+1; + *status|=DEC_Clamped; + } + } + decFinish(res, set, &residue, status); + break; + } + } + + /* handle 0 rhs (x/0) */ + if (ISZERO(rhs)) { /* x/0 is always exceptional */ + if (ISZERO(lhs)) { + uprv_decNumberZero(res); /* [after lhs test] */ + *status|=DEC_Division_undefined;/* 0/0 will become NaN */ + } + else { + uprv_decNumberZero(res); + if (op&(REMAINDER|REMNEAR)) *status|=DEC_Invalid_operation; + else { + *status|=DEC_Division_by_zero; /* x/0 */ + res->bits=bits|DECINF; /* .. is +/- Infinity */ + } + } + break;} + + /* handle 0 lhs (0/x) */ + if (ISZERO(lhs)) { /* 0/x [x!=0] */ + #if DECSUBSET + if (!set->extended) uprv_decNumberZero(res); + else { + #endif + if (op&DIVIDE) { + residue=0; + exponent=lhs->exponent-rhs->exponent; /* ideal exponent */ + uprv_decNumberCopy(res, lhs); /* [zeros always fit] */ + res->bits=bits; /* sign as computed */ + res->exponent=exponent; /* exponent, too */ + decFinalize(res, set, &residue, status); /* check exponent */ + } + else if (op&DIVIDEINT) { + uprv_decNumberZero(res); /* integer 0 */ + res->bits=bits; /* sign as computed */ + } + else { /* a remainder */ + exponent=rhs->exponent; /* [save in case overwrite] */ + uprv_decNumberCopy(res, lhs); /* [zeros always fit] */ + if (exponentexponent) res->exponent=exponent; /* use lower */ + } + #if DECSUBSET + } + #endif + break;} + + /* Precalculate exponent. This starts off adjusted (and hence fits */ + /* in 31 bits) and becomes the usual unadjusted exponent as the */ + /* division proceeds. The order of evaluation is important, here, */ + /* to avoid wrap. */ + exponent=(lhs->exponent+lhs->digits)-(rhs->exponent+rhs->digits); + + /* If the working exponent is -ve, then some quick exits are */ + /* possible because the quotient is known to be <1 */ + /* [for REMNEAR, it needs to be < -1, as -0.5 could need work] */ + if (exponent<0 && !(op==DIVIDE)) { + if (op&DIVIDEINT) { + uprv_decNumberZero(res); /* integer part is 0 */ + #if DECSUBSET + if (set->extended) + #endif + res->bits=bits; /* set +/- zero */ + break;} + /* fastpath remainders so long as the lhs has the smaller */ + /* (or equal) exponent */ + if (lhs->exponent<=rhs->exponent) { + if (op&REMAINDER || exponent<-1) { + /* It is REMAINDER or safe REMNEAR; result is [finished */ + /* clone of] lhs (r = x - 0*y) */ + residue=0; + decCopyFit(res, lhs, set, &residue, status); + decFinish(res, set, &residue, status); + break; + } + /* [unsafe REMNEAR drops through] */ + } + } /* fastpaths */ + + /* Long (slow) division is needed; roll up the sleeves... */ + + /* The accumulator will hold the quotient of the division. */ + /* If it needs to be too long for stack storage, then allocate. */ + acclength=D2U(reqdigits+DECDPUN); /* in Units */ + if (acclength*sizeof(Unit)>sizeof(accbuff)) { + /* printf("malloc dvacc %ld units\n", acclength); */ + allocacc=(Unit *)malloc(acclength*sizeof(Unit)); + if (allocacc==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + acc=allocacc; /* use the allocated space */ + } + + /* var1 is the padded LHS ready for subtractions. */ + /* If it needs to be too long for stack storage, then allocate. */ + /* The maximum units needed for var1 (long subtraction) is: */ + /* Enough for */ + /* (rhs->digits+reqdigits-1) -- to allow full slide to right */ + /* or (lhs->digits) -- to allow for long lhs */ + /* whichever is larger */ + /* +1 -- for rounding of slide to right */ + /* +1 -- for leading 0s */ + /* +1 -- for pre-adjust if a remainder or DIVIDEINT */ + /* [Note: unused units do not participate in decUnitAddSub data] */ + maxdigits=rhs->digits+reqdigits-1; + if (lhs->digits>maxdigits) maxdigits=lhs->digits; + var1units=D2U(maxdigits)+2; + /* allocate a guard unit above msu1 for REMAINDERNEAR */ + if (!(op&DIVIDE)) var1units++; + if ((var1units+1)*sizeof(Unit)>sizeof(varbuff)) { + /* printf("malloc dvvar %ld units\n", var1units+1); */ + varalloc=(Unit *)malloc((var1units+1)*sizeof(Unit)); + if (varalloc==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + var1=varalloc; /* use the allocated space */ + } + + /* Extend the lhs and rhs to full long subtraction length. The lhs */ + /* is truly extended into the var1 buffer, with 0 padding, so a */ + /* subtract in place is always possible. The rhs (var2) has */ + /* virtual padding (implemented by decUnitAddSub). */ + /* One guard unit was allocated above msu1 for rem=rem+rem in */ + /* REMAINDERNEAR. */ + msu1=var1+var1units-1; /* msu of var1 */ + source=lhs->lsu+D2U(lhs->digits)-1; /* msu of input array */ + for (target=msu1; source>=lhs->lsu; source--, target--) *target=*source; + for (; target>=var1; target--) *target=0; + + /* rhs (var2) is left-aligned with var1 at the start */ + var2ulen=var1units; /* rhs logical length (units) */ + var2units=D2U(rhs->digits); /* rhs actual length (units) */ + var2=rhs->lsu; /* -> rhs array */ + msu2=var2+var2units-1; /* -> msu of var2 [never changes] */ + /* now set up the variables which will be used for estimating the */ + /* multiplication factor. If these variables are not exact, add */ + /* 1 to make sure that the multiplier is never overestimated. */ + msu2plus=*msu2; /* it's value .. */ + if (var2units>1) msu2plus++; /* .. +1 if any more */ + msu2pair=(eInt)*msu2*(DECDPUNMAX+1);/* top two pair .. */ + if (var2units>1) { /* .. [else treat 2nd as 0] */ + msu2pair+=*(msu2-1); /* .. */ + if (var2units>2) msu2pair++; /* .. +1 if any more */ + } + + /* The calculation is working in units, which may have leading zeros, */ + /* but the exponent was calculated on the assumption that they are */ + /* both left-aligned. Adjust the exponent to compensate: add the */ + /* number of leading zeros in var1 msu and subtract those in var2 msu. */ + /* [This is actually done by counting the digits and negating, as */ + /* lead1=DECDPUN-digits1, and similarly for lead2.] */ + for (pow=&powers[1]; *msu1>=*pow; pow++) exponent--; + for (pow=&powers[1]; *msu2>=*pow; pow++) exponent++; + + /* Now, if doing an integer divide or remainder, ensure that */ + /* the result will be Unit-aligned. To do this, shift the var1 */ + /* accumulator towards least if need be. (It's much easier to */ + /* do this now than to reassemble the residue afterwards, if */ + /* doing a remainder.) Also ensure the exponent is not negative. */ + if (!(op&DIVIDE)) { + Unit *u; /* work */ + /* save the initial 'false' padding of var1, in digits */ + var1initpad=(var1units-D2U(lhs->digits))*DECDPUN; + /* Determine the shift to do. */ + if (exponent<0) cut=-exponent; + else cut=DECDPUN-exponent%DECDPUN; + decShiftToLeast(var1, var1units, cut); + exponent+=cut; /* maintain numerical value */ + var1initpad-=cut; /* .. and reduce padding */ + /* clean any most-significant units which were just emptied */ + for (u=msu1; cut>=DECDPUN; cut-=DECDPUN, u--) *u=0; + } /* align */ + else { /* is DIVIDE */ + maxexponent=lhs->exponent-rhs->exponent; /* save */ + /* optimization: if the first iteration will just produce 0, */ + /* preadjust to skip it [valid for DIVIDE only] */ + if (*msu1<*msu2) { + var2ulen--; /* shift down */ + exponent-=DECDPUN; /* update the exponent */ + } + } + + /* ---- start the long-division loops ------------------------------ */ + accunits=0; /* no units accumulated yet */ + accdigits=0; /* .. or digits */ + accnext=acc+acclength-1; /* -> msu of acc [NB: allows digits+1] */ + for (;;) { /* outer forever loop */ + thisunit=0; /* current unit assumed 0 */ + /* find the next unit */ + for (;;) { /* inner forever loop */ + /* strip leading zero units [from either pre-adjust or from */ + /* subtract last time around]. Leave at least one unit. */ + for (; *msu1==0 && msu1>var1; msu1--) var1units--; + + if (var1units msu */ + for (pv1=msu1; ; pv1--, pv2--) { + /* v1=*pv1 -- always OK */ + v2=0; /* assume in padding */ + if (pv2>=var2) v2=*pv2; /* in range */ + if (*pv1!=v2) break; /* no longer the same */ + if (pv1==var1) break; /* done; leave pv1 as is */ + } + /* here when all inspected or a difference seen */ + if (*pv1v2. Prepare for real subtraction; the lengths are equal */ + /* Estimate the multiplier (there's always a msu1-1)... */ + /* Bring in two units of var2 to provide a good estimate. */ + mult=(Int)(((eInt)*msu1*(DECDPUNMAX+1)+*(msu1-1))/msu2pair); + } /* lengths the same */ + else { /* var1units > var2ulen, so subtraction is safe */ + /* The var2 msu is one unit towards the lsu of the var1 msu, */ + /* so only one unit for var2 can be used. */ + mult=(Int)(((eInt)*msu1*(DECDPUNMAX+1)+*(msu1-1))/msu2plus); + } + if (mult==0) mult=1; /* must always be at least 1 */ + /* subtraction needed; var1 is > var2 */ + thisunit=(Unit)(thisunit+mult); /* accumulate */ + /* subtract var1-var2, into var1; only the overlap needs */ + /* processing, as this is an in-place calculation */ + shift=var2ulen-var2units; + #if DECTRACE + decDumpAr('1', &var1[shift], var1units-shift); + decDumpAr('2', var2, var2units); + printf("m=%ld\n", -mult); + #endif + decUnitAddSub(&var1[shift], var1units-shift, + var2, var2units, 0, + &var1[shift], -mult); + #if DECTRACE + decDumpAr('#', &var1[shift], var1units-shift); + #endif + /* var1 now probably has leading zeros; these are removed at the */ + /* top of the inner loop. */ + } /* inner loop */ + + /* The next unit has been calculated in full; unless it's a */ + /* leading zero, add to acc */ + if (accunits!=0 || thisunit!=0) { /* is first or non-zero */ + *accnext=thisunit; /* store in accumulator */ + /* account exactly for the new digits */ + if (accunits==0) { + accdigits++; /* at least one */ + for (pow=&powers[1]; thisunit>=*pow; pow++) accdigits++; + } + else accdigits+=DECDPUN; + accunits++; /* update count */ + accnext--; /* ready for next */ + if (accdigits>reqdigits) break; /* have enough digits */ + } + + /* if the residue is zero, the operation is done (unless divide */ + /* or divideInteger and still not enough digits yet) */ + if (*var1==0 && var1units==1) { /* residue is 0 */ + if (op&(REMAINDER|REMNEAR)) break; + if ((op&DIVIDE) && (exponent<=maxexponent)) break; + /* [drop through if divideInteger] */ + } + /* also done enough if calculating remainder or integer */ + /* divide and just did the last ('units') unit */ + if (exponent==0 && !(op&DIVIDE)) break; + + /* to get here, var1 is less than var2, so divide var2 by the per- */ + /* Unit power of ten and go for the next digit */ + var2ulen--; /* shift down */ + exponent-=DECDPUN; /* update the exponent */ + } /* outer loop */ + + /* ---- division is complete --------------------------------------- */ + /* here: acc has at least reqdigits+1 of good results (or fewer */ + /* if early stop), starting at accnext+1 (its lsu) */ + /* var1 has any residue at the stopping point */ + /* accunits is the number of digits collected in acc */ + if (accunits==0) { /* acc is 0 */ + accunits=1; /* show have a unit .. */ + accdigits=1; /* .. */ + *accnext=0; /* .. whose value is 0 */ + } + else accnext++; /* back to last placed */ + /* accnext now -> lowest unit of result */ + + residue=0; /* assume no residue */ + if (op&DIVIDE) { + /* record the presence of any residue, for rounding */ + if (*var1!=0 || var1units>1) residue=1; + else { /* no residue */ + /* Had an exact division; clean up spurious trailing 0s. */ + /* There will be at most DECDPUN-1, from the final multiply, */ + /* and then only if the result is non-0 (and even) and the */ + /* exponent is 'loose'. */ + #if DECDPUN>1 + Unit lsu=*accnext; + if (!(lsu&0x01) && (lsu!=0)) { + /* count the trailing zeros */ + Int drop=0; + for (;; drop++) { /* [will terminate because lsu!=0] */ + if (exponent>=maxexponent) break; /* don't chop real 0s */ + #if DECDPUN<=4 + if ((lsu-QUOT10(lsu, drop+1) + *powers[drop+1])!=0) break; /* found non-0 digit */ + #else + if (lsu%powers[drop+1]!=0) break; /* found non-0 digit */ + #endif + exponent++; + } + if (drop>0) { + accunits=decShiftToLeast(accnext, accunits, drop); + accdigits=decGetDigits(accnext, accunits); + accunits=D2U(accdigits); + /* [exponent was adjusted in the loop] */ + } + } /* neither odd nor 0 */ + #endif + } /* exact divide */ + } /* divide */ + else /* op!=DIVIDE */ { + /* check for coefficient overflow */ + if (accdigits+exponent>reqdigits) { + *status|=DEC_Division_impossible; + break; + } + if (op & (REMAINDER|REMNEAR)) { + /* [Here, the exponent will be 0, because var1 was adjusted */ + /* appropriately.] */ + Int postshift; /* work */ + Flag wasodd=0; /* integer was odd */ + Unit *quotlsu; /* for save */ + Int quotdigits; /* .. */ + + bits=lhs->bits; /* remainder sign is always as lhs */ + + /* Fastpath when residue is truly 0 is worthwhile [and */ + /* simplifies the code below] */ + if (*var1==0 && var1units==1) { /* residue is 0 */ + Int exp=lhs->exponent; /* save min(exponents) */ + if (rhs->exponentexponent; + uprv_decNumberZero(res); /* 0 coefficient */ + #if DECSUBSET + if (set->extended) + #endif + res->exponent=exp; /* .. with proper exponent */ + res->bits=(uByte)(bits&DECNEG); /* [cleaned] */ + decFinish(res, set, &residue, status); /* might clamp */ + break; + } + /* note if the quotient was odd */ + if (*accnext & 0x01) wasodd=1; /* acc is odd */ + quotlsu=accnext; /* save in case need to reinspect */ + quotdigits=accdigits; /* .. */ + + /* treat the residue, in var1, as the value to return, via acc */ + /* calculate the unused zero digits. This is the smaller of: */ + /* var1 initial padding (saved above) */ + /* var2 residual padding, which happens to be given by: */ + postshift=var1initpad+exponent-lhs->exponent+rhs->exponent; + /* [the 'exponent' term accounts for the shifts during divide] */ + if (var1initpadexponent; /* exponent is smaller of lhs & rhs */ + if (rhs->exponentexponent; + + /* Now correct the result if doing remainderNear; if it */ + /* (looking just at coefficients) is > rhs/2, or == rhs/2 and */ + /* the integer was odd then the result should be rem-rhs. */ + if (op&REMNEAR) { + Int compare, tarunits; /* work */ + Unit *up; /* .. */ + /* calculate remainder*2 into the var1 buffer (which has */ + /* 'headroom' of an extra unit and hence enough space) */ + /* [a dedicated 'double' loop would be faster, here] */ + tarunits=decUnitAddSub(accnext, accunits, accnext, accunits, + 0, accnext, 1); + /* decDumpAr('r', accnext, tarunits); */ + + /* Here, accnext (var1) holds tarunits Units with twice the */ + /* remainder's coefficient, which must now be compared to the */ + /* RHS. The remainder's exponent may be smaller than the RHS's. */ + compare=decUnitCompare(accnext, tarunits, rhs->lsu, D2U(rhs->digits), + rhs->exponent-exponent); + if (compare==BADINT) { /* deep trouble */ + *status|=DEC_Insufficient_storage; + break;} + + /* now restore the remainder by dividing by two; the lsu */ + /* is known to be even. */ + for (up=accnext; up0 || (compare==0 && wasodd)) { /* adjustment needed */ + Int exp, expunits, exprem; /* work */ + /* This is effectively causing round-up of the quotient, */ + /* so if it was the rare case where it was full and all */ + /* nines, it would overflow and hence division-impossible */ + /* should be raised */ + Flag allnines=0; /* 1 if quotient all nines */ + if (quotdigits==reqdigits) { /* could be borderline */ + for (up=quotlsu; ; up++) { + if (quotdigits>DECDPUN) { + if (*up!=DECDPUNMAX) break;/* non-nines */ + } + else { /* this is the last Unit */ + if (*up==powers[quotdigits]-1) allnines=1; + break; + } + quotdigits-=DECDPUN; /* checked those digits */ + } /* up */ + } /* borderline check */ + if (allnines) { + *status|=DEC_Division_impossible; + break;} + + /* rem-rhs is needed; the sign will invert. Again, var1 */ + /* can safely be used for the working Units array. */ + exp=rhs->exponent-exponent; /* RHS padding needed */ + /* Calculate units and remainder from exponent. */ + expunits=exp/DECDPUN; + exprem=exp%DECDPUN; + /* subtract [A+B*(-m)]; the result will always be negative */ + accunits=-decUnitAddSub(accnext, accunits, + rhs->lsu, D2U(rhs->digits), + expunits, accnext, -(Int)powers[exprem]); + accdigits=decGetDigits(accnext, accunits); /* count digits exactly */ + accunits=D2U(accdigits); /* and recalculate the units for copy */ + /* [exponent is as for original remainder] */ + bits^=DECNEG; /* flip the sign */ + } + } /* REMNEAR */ + } /* REMAINDER or REMNEAR */ + } /* not DIVIDE */ + + /* Set exponent and bits */ + res->exponent=exponent; + res->bits=(uByte)(bits&DECNEG); /* [cleaned] */ + + /* Now the coefficient. */ + decSetCoeff(res, set, accnext, accdigits, &residue, status); + + decFinish(res, set, &residue, status); /* final cleanup */ + + #if DECSUBSET + /* If a divide then strip trailing zeros if subset [after round] */ + if (!set->extended && (op==DIVIDE)) decTrim(res, set, 0, 1, &dropped); + #endif + } while(0); /* end protected */ + + if (varalloc!=nullptr) free(varalloc); /* drop any storage used */ + if (allocacc!=nullptr) free(allocacc); /* .. */ + #if DECSUBSET + if (allocrhs!=nullptr) free(allocrhs); /* .. */ + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + #endif + return res; + } /* decDivideOp */ + +/* ------------------------------------------------------------------ */ +/* decMultiplyOp -- multiplication operation */ +/* */ +/* This routine performs the multiplication C=A x B. */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X*X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* status is the usual accumulator */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* ------------------------------------------------------------------ */ +/* 'Classic' multiplication is used rather than Karatsuba, as the */ +/* latter would give only a minor improvement for the short numbers */ +/* expected to be handled most (and uses much more memory). */ +/* */ +/* There are two major paths here: the general-purpose ('old code') */ +/* path which handles all DECDPUN values, and a fastpath version */ +/* which is used if 64-bit ints are available, DECDPUN<=4, and more */ +/* than two calls to decUnitAddSub would be made. */ +/* */ +/* The fastpath version lumps units together into 8-digit or 9-digit */ +/* chunks, and also uses a lazy carry strategy to minimise expensive */ +/* 64-bit divisions. The chunks are then broken apart again into */ +/* units for continuing processing. Despite this overhead, the */ +/* fastpath can speed up some 16-digit operations by 10x (and much */ +/* more for higher-precision calculations). */ +/* */ +/* A buffer always has to be used for the accumulator; in the */ +/* fastpath, buffers are also always needed for the chunked copies of */ +/* of the operand coefficients. */ +/* Static buffers are larger than needed just for multiply, to allow */ +/* for calls from other operations (notably exp). */ +/* ------------------------------------------------------------------ */ +#define FASTMUL (DECUSE64 && DECDPUN<5) +static decNumber * decMultiplyOp(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set, + uInt *status) { + Int accunits; /* Units of accumulator in use */ + Int exponent; /* work */ + Int residue=0; /* rounding residue */ + uByte bits; /* result sign */ + Unit *acc; /* -> accumulator Unit array */ + Int needbytes; /* size calculator */ + void *allocacc=nullptr; /* -> allocated accumulator, iff allocated */ + Unit accbuff[SD2U(DECBUFFER*4+1)]; /* buffer (+1 for DECBUFFER==0, */ + /* *4 for calls from other operations) */ + const Unit *mer, *mermsup; /* work */ + Int madlength; /* Units in multiplicand */ + Int shift; /* Units to shift multiplicand by */ + + #if FASTMUL + /* if DECDPUN is 1 or 3 work in base 10**9, otherwise */ + /* (DECDPUN is 2 or 4) then work in base 10**8 */ + #if DECDPUN & 1 /* odd */ + #define FASTBASE 1000000000 /* base */ + #define FASTDIGS 9 /* digits in base */ + #define FASTLAZY 18 /* carry resolution point [1->18] */ + #else + #define FASTBASE 100000000 + #define FASTDIGS 8 + #define FASTLAZY 1844 /* carry resolution point [1->1844] */ + #endif + /* three buffers are used, two for chunked copies of the operands */ + /* (base 10**8 or base 10**9) and one base 2**64 accumulator with */ + /* lazy carry evaluation */ + uInt zlhibuff[(DECBUFFER*2+1)/8+1]; /* buffer (+1 for DECBUFFER==0) */ + uInt *zlhi=zlhibuff; /* -> lhs array */ + uInt *alloclhi=nullptr; /* -> allocated buffer, iff allocated */ + uInt zrhibuff[(DECBUFFER*2+1)/8+1]; /* buffer (+1 for DECBUFFER==0) */ + uInt *zrhi=zrhibuff; /* -> rhs array */ + uInt *allocrhi=nullptr; /* -> allocated buffer, iff allocated */ + uLong zaccbuff[(DECBUFFER*2+1)/4+2]; /* buffer (+1 for DECBUFFER==0) */ + /* [allocacc is shared for both paths, as only one will run] */ + uLong *zacc=zaccbuff; /* -> accumulator array for exact result */ + #if DECDPUN==1 + Int zoff; /* accumulator offset */ + #endif + uInt *lip, *rip; /* item pointers */ + uInt *lmsi, *rmsi; /* most significant items */ + Int ilhs, irhs, iacc; /* item counts in the arrays */ + Int lazy; /* lazy carry counter */ + uLong lcarry; /* uLong carry */ + uInt carry; /* carry (NB not uLong) */ + Int count; /* work */ + const Unit *cup; /* .. */ + Unit *up; /* .. */ + uLong *lp; /* .. */ + Int p; /* .. */ + #endif + + #if DECSUBSET + decNumber *alloclhs=nullptr; /* -> allocated buffer, iff allocated */ + decNumber *allocrhs=nullptr; /* -> allocated buffer, iff allocated */ + #endif + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + /* precalculate result sign */ + bits=(uByte)((lhs->bits^rhs->bits)&DECNEG); + + /* handle infinities and NaNs */ + if (SPECIALARGS) { /* a special bit set */ + if (SPECIALARGS & (DECSNAN | DECNAN)) { /* one or two NaNs */ + decNaNs(res, lhs, rhs, set, status); + return res;} + /* one or two infinities; Infinity * 0 is invalid */ + if (((lhs->bits & DECINF)==0 && ISZERO(lhs)) + ||((rhs->bits & DECINF)==0 && ISZERO(rhs))) { + *status|=DEC_Invalid_operation; + return res;} + uprv_decNumberZero(res); + res->bits=bits|DECINF; /* infinity */ + return res;} + + /* For best speed, as in DMSRCN [the original Rexx numerics */ + /* module], use the shorter number as the multiplier (rhs) and */ + /* the longer as the multiplicand (lhs) to minimise the number of */ + /* adds (partial products) */ + if (lhs->digitsdigits) { /* swap... */ + const decNumber *hold=lhs; + lhs=rhs; + rhs=hold; + } + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operands and set lostDigits status, as needed */ + if (lhs->digits>set->digits) { + alloclhs=decRoundOperand(lhs, set, status); + if (alloclhs==nullptr) break; + lhs=alloclhs; + } + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + #if FASTMUL /* fastpath can be used */ + /* use the fast path if there are enough digits in the shorter */ + /* operand to make the setup and takedown worthwhile */ + #define NEEDTWO (DECDPUN*2) /* within two decUnitAddSub calls */ + if (rhs->digits>NEEDTWO) { /* use fastpath... */ + /* calculate the number of elements in each array */ + ilhs=(lhs->digits+FASTDIGS-1)/FASTDIGS; /* [ceiling] */ + irhs=(rhs->digits+FASTDIGS-1)/FASTDIGS; /* .. */ + iacc=ilhs+irhs; + + /* allocate buffers if required, as usual */ + needbytes=ilhs*sizeof(uInt); + if (needbytes>(Int)sizeof(zlhibuff)) { + alloclhi=(uInt *)malloc(needbytes); + zlhi=alloclhi;} + needbytes=irhs*sizeof(uInt); + if (needbytes>(Int)sizeof(zrhibuff)) { + allocrhi=(uInt *)malloc(needbytes); + zrhi=allocrhi;} + + /* Allocating the accumulator space needs a special case when */ + /* DECDPUN=1 because when converting the accumulator to Units */ + /* after the multiplication each 8-byte item becomes 9 1-byte */ + /* units. Therefore iacc extra bytes are needed at the front */ + /* (rounded up to a multiple of 8 bytes), and the uLong */ + /* accumulator starts offset the appropriate number of units */ + /* to the right to avoid overwrite during the unchunking. */ + + /* Make sure no signed int overflow below. This is always true */ + /* if the given numbers have less digits than DEC_MAX_DIGITS. */ + U_ASSERT((uint32_t)iacc <= INT32_MAX/sizeof(uLong)); + needbytes=iacc*sizeof(uLong); + #if DECDPUN==1 + zoff=(iacc+7)/8; /* items to offset by */ + needbytes+=zoff*8; + #endif + if (needbytes>(Int)sizeof(zaccbuff)) { + allocacc=(uLong *)malloc(needbytes); + zacc=(uLong *)allocacc;} + if (zlhi==nullptr||zrhi==nullptr||zacc==nullptr) { + *status|=DEC_Insufficient_storage; + break;} + + acc=(Unit *)zacc; /* -> target Unit array */ + #if DECDPUN==1 + zacc+=zoff; /* start uLong accumulator to right */ + #endif + + /* assemble the chunked copies of the left and right sides */ + for (count=lhs->digits, cup=lhs->lsu, lip=zlhi; count>0; lip++) + for (p=0, *lip=0; p0; + p+=DECDPUN, cup++, count-=DECDPUN) + *lip+=*cup*powers[p]; + lmsi=lip-1; /* save -> msi */ + for (count=rhs->digits, cup=rhs->lsu, rip=zrhi; count>0; rip++) + for (p=0, *rip=0; p0; + p+=DECDPUN, cup++, count-=DECDPUN) + *rip+=*cup*powers[p]; + rmsi=rip-1; /* save -> msi */ + + /* zero the accumulator */ + for (lp=zacc; lp0 && rip!=rmsi) continue; + lazy=FASTLAZY; /* reset delay count */ + /* spin up the accumulator resolving overflows */ + for (lp=zacc; lp(up-acc); /* count of units */ + } + else { /* here to use units directly, without chunking ['old code'] */ + #endif + + /* if accumulator will be too long for local storage, then allocate */ + acc=accbuff; /* -> assume buffer for accumulator */ + needbytes=(D2U(lhs->digits)+D2U(rhs->digits))*sizeof(Unit); + if (needbytes>(Int)sizeof(accbuff)) { + allocacc=(Unit *)malloc(needbytes); + if (allocacc==nullptr) {*status|=DEC_Insufficient_storage; break;} + acc=(Unit *)allocacc; /* use the allocated space */ + } + + /* Now the main long multiplication loop */ + /* Unlike the equivalent in the IBM Java implementation, there */ + /* is no advantage in calculating from msu to lsu. So, do it */ + /* by the book, as it were. */ + /* Each iteration calculates ACC=ACC+MULTAND*MULT */ + accunits=1; /* accumulator starts at '0' */ + *acc=0; /* .. (lsu=0) */ + shift=0; /* no multiplicand shift at first */ + madlength=D2U(lhs->digits); /* this won't change */ + mermsup=rhs->lsu+D2U(rhs->digits); /* -> msu+1 of multiplier */ + + for (mer=rhs->lsu; merlsu, madlength, 0, + &acc[shift], *mer) + + shift; + else { /* extend acc with a 0; it will be used shortly */ + *(acc+accunits)=0; /* [this avoids length of <=0 later] */ + accunits++; + } + /* multiply multiplicand by 10**DECDPUN for next Unit to left */ + shift++; /* add this for 'logical length' */ + } /* n */ + #if FASTMUL + } /* unchunked units */ + #endif + /* common end-path */ + #if DECTRACE + decDumpAr('*', acc, accunits); /* Show exact result */ + #endif + + /* acc now contains the exact result of the multiplication, */ + /* possibly with a leading zero unit; build the decNumber from */ + /* it, noting if any residue */ + res->bits=bits; /* set sign */ + res->digits=decGetDigits(acc, accunits); /* count digits exactly */ + + /* There can be a 31-bit wrap in calculating the exponent. */ + /* This can only happen if both input exponents are negative and */ + /* both their magnitudes are large. If there was a wrap, set a */ + /* safe very negative exponent, from which decFinalize() will */ + /* raise a hard underflow shortly. */ + exponent=lhs->exponent+rhs->exponent; /* calculate exponent */ + if (lhs->exponent<0 && rhs->exponent<0 && exponent>0) + exponent=-2*DECNUMMAXE; /* force underflow */ + res->exponent=exponent; /* OK to overwrite now */ + + + /* Set the coefficient. If any rounding, residue records */ + decSetCoeff(res, set, acc, res->digits, &residue, status); + decFinish(res, set, &residue, status); /* final cleanup */ + } while(0); /* end protected */ + + if (allocacc!=nullptr) free(allocacc); /* drop any storage used */ + #if DECSUBSET + if (allocrhs!=nullptr) free(allocrhs); /* .. */ + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + #endif + #if FASTMUL + if (allocrhi!=nullptr) free(allocrhi); /* .. */ + if (alloclhi!=nullptr) free(alloclhi); /* .. */ + #endif + return res; + } /* decMultiplyOp */ + +/* ------------------------------------------------------------------ */ +/* decExpOp -- effect exponentiation */ +/* */ +/* This computes C = exp(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. status is updated but */ +/* not set. */ +/* */ +/* Restrictions: */ +/* */ +/* digits, emax, and -emin in the context must be less than */ +/* 2*DEC_MAX_MATH (1999998), and the rhs must be within these */ +/* bounds or a zero. This is an internal routine, so these */ +/* restrictions are contractual and not enforced. */ +/* */ +/* A finite result is rounded using DEC_ROUND_HALF_EVEN; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* */ +/* Finite results will always be full precision and Inexact, except */ +/* when A is a zero or -Infinity (giving 1 or 0 respectively). */ +/* ------------------------------------------------------------------ */ +/* This approach used here is similar to the algorithm described in */ +/* */ +/* Variable Precision Exponential Function, T. E. Hull and */ +/* A. Abrham, ACM Transactions on Mathematical Software, Vol 12 #2, */ +/* pp79-91, ACM, June 1986. */ +/* */ +/* with the main difference being that the iterations in the series */ +/* evaluation are terminated dynamically (which does not require the */ +/* extra variable-precision variables which are expensive in this */ +/* context). */ +/* */ +/* The error analysis in Hull & Abrham's paper applies except for the */ +/* round-off error accumulation during the series evaluation. This */ +/* code does not precalculate the number of iterations and so cannot */ +/* use Horner's scheme. Instead, the accumulation is done at double- */ +/* precision, which ensures that the additions of the terms are exact */ +/* and do not accumulate round-off (and any round-off errors in the */ +/* terms themselves move 'to the right' faster than they can */ +/* accumulate). This code also extends the calculation by allowing, */ +/* in the spirit of other decNumber operators, the input to be more */ +/* precise than the result (the precision used is based on the more */ +/* precise of the input or requested result). */ +/* */ +/* Implementation notes: */ +/* */ +/* 1. This is separated out as decExpOp so it can be called from */ +/* other Mathematical functions (notably Ln) with a wider range */ +/* than normal. In particular, it can handle the slightly wider */ +/* (double) range needed by Ln (which has to be able to calculate */ +/* exp(-x) where x can be the tiniest number (Ntiny). */ +/* */ +/* 2. Normalizing x to be <=0.1 (instead of <=1) reduces loop */ +/* iterations by approximately a third with additional (although */ +/* diminishing) returns as the range is reduced to even smaller */ +/* fractions. However, h (the power of 10 used to correct the */ +/* result at the end, see below) must be kept <=8 as otherwise */ +/* the final result cannot be computed. Hence the leverage is a */ +/* sliding value (8-h), where potentially the range is reduced */ +/* more for smaller values. */ +/* */ +/* The leverage that can be applied in this way is severely */ +/* limited by the cost of the raise-to-the power at the end, */ +/* which dominates when the number of iterations is small (less */ +/* than ten) or when rhs is short. As an example, the adjustment */ +/* x**10,000,000 needs 31 multiplications, all but one full-width. */ +/* */ +/* 3. The restrictions (especially precision) could be raised with */ +/* care, but the full decNumber range seems very hard within the */ +/* 32-bit limits. */ +/* */ +/* 4. The working precisions for the static buffers are twice the */ +/* obvious size to allow for calls from decNumberPower. */ +/* ------------------------------------------------------------------ */ +decNumber * decExpOp(decNumber *res, const decNumber *rhs, + decContext *set, uInt *status) { + uInt ignore=0; /* working status */ + Int h; /* adjusted exponent for 0.xxxx */ + Int p; /* working precision */ + Int residue; /* rounding residue */ + uInt needbytes; /* for space calculations */ + const decNumber *x=rhs; /* (may point to safe copy later) */ + decContext aset, tset, dset; /* working contexts */ + Int comp; /* work */ + + /* the argument is often copied to normalize it, so (unusually) it */ + /* is treated like other buffers, using DECBUFFER, +1 in case */ + /* DECBUFFER is 0 */ + decNumber bufr[D2N(DECBUFFER*2+1)]; + decNumber *allocrhs=nullptr; /* non-nullptr if rhs buffer allocated */ + + /* the working precision will be no more than set->digits+8+1 */ + /* so for on-stack buffers DECBUFFER+9 is used, +1 in case DECBUFFER */ + /* is 0 (and twice that for the accumulator) */ + + /* buffer for t, term (working precision plus) */ + decNumber buft[D2N(DECBUFFER*2+9+1)]; + decNumber *allocbuft=nullptr; /* -> allocated buft, iff allocated */ + decNumber *t=buft; /* term */ + /* buffer for a, accumulator (working precision * 2), at least 9 */ + decNumber bufa[D2N(DECBUFFER*4+18+1)]; + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *a=bufa; /* accumulator */ + /* decNumber for the divisor term; this needs at most 9 digits */ + /* and so can be fixed size [16 so can use standard context] */ + decNumber bufd[D2N(16)]; + decNumber *d=bufd; /* divisor */ + decNumber numone; /* constant 1 */ + + #if DECCHECK + Int iterations=0; /* for later sanity check */ + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + if (SPECIALARG) { /* handle infinities and NaNs */ + if (decNumberIsInfinite(rhs)) { /* an infinity */ + if (decNumberIsNegative(rhs)) /* -Infinity -> +0 */ + uprv_decNumberZero(res); + else uprv_decNumberCopy(res, rhs); /* +Infinity -> self */ + } + else decNaNs(res, rhs, nullptr, set, status); /* a NaN */ + break;} + + if (ISZERO(rhs)) { /* zeros -> exact 1 */ + uprv_decNumberZero(res); /* make clean 1 */ + *res->lsu=1; /* .. */ + break;} /* [no status to set] */ + + /* e**x when 0 < x < 0.66 is < 1+3x/2, hence can fast-path */ + /* positive and negative tiny cases which will result in inexact */ + /* 1. This also allows the later add-accumulate to always be */ + /* exact (because its length will never be more than twice the */ + /* working precision). */ + /* The comparator (tiny) needs just one digit, so use the */ + /* decNumber d for it (reused as the divisor, etc., below); its */ + /* exponent is such that if x is positive it will have */ + /* set->digits-1 zeros between the decimal point and the digit, */ + /* which is 4, and if x is negative one more zero there as the */ + /* more precise result will be of the form 0.9999999 rather than */ + /* 1.0000001. Hence, tiny will be 0.0000004 if digits=7 and x>0 */ + /* or 0.00000004 if digits=7 and x<0. If RHS not larger than */ + /* this then the result will be 1.000000 */ + uprv_decNumberZero(d); /* clean */ + *d->lsu=4; /* set 4 .. */ + d->exponent=-set->digits; /* * 10**(-d) */ + if (decNumberIsNegative(rhs)) d->exponent--; /* negative case */ + comp=decCompare(d, rhs, 1); /* signless compare */ + if (comp==BADINT) { + *status|=DEC_Insufficient_storage; + break;} + if (comp>=0) { /* rhs < d */ + Int shift=set->digits-1; + uprv_decNumberZero(res); /* set 1 */ + *res->lsu=1; /* .. */ + res->digits=decShiftToMost(res->lsu, 1, shift); + res->exponent=-shift; /* make 1.0000... */ + *status|=DEC_Inexact | DEC_Rounded; /* .. inexactly */ + break;} /* tiny */ + + /* set up the context to be used for calculating a, as this is */ + /* used on both paths below */ + uprv_decContextDefault(&aset, DEC_INIT_DECIMAL64); + /* accumulator bounds are as requested (could underflow) */ + aset.emax=set->emax; /* usual bounds */ + aset.emin=set->emin; /* .. */ + aset.clamp=0; /* and no concrete format */ + + /* calculate the adjusted (Hull & Abrham) exponent (where the */ + /* decimal point is just to the left of the coefficient msd) */ + h=rhs->exponent+rhs->digits; + /* if h>8 then 10**h cannot be calculated safely; however, when */ + /* h=8 then exp(|rhs|) will be at least exp(1E+7) which is at */ + /* least 6.59E+4342944, so (due to the restriction on Emax/Emin) */ + /* overflow (or underflow to 0) is guaranteed -- so this case can */ + /* be handled by simply forcing the appropriate excess */ + if (h>8) { /* overflow/underflow */ + /* set up here so Power call below will over or underflow to */ + /* zero; set accumulator to either 2 or 0.02 */ + /* [stack buffer for a is always big enough for this] */ + uprv_decNumberZero(a); + *a->lsu=2; /* not 1 but < exp(1) */ + if (decNumberIsNegative(rhs)) a->exponent=-2; /* make 0.02 */ + h=8; /* clamp so 10**h computable */ + p=9; /* set a working precision */ + } + else { /* h<=8 */ + Int maxlever=(rhs->digits>8?1:0); + /* [could/should increase this for precisions >40 or so, too] */ + + /* if h is 8, cannot normalize to a lower upper limit because */ + /* the final result will not be computable (see notes above), */ + /* but leverage can be applied whenever h is less than 8. */ + /* Apply as much as possible, up to a MAXLEVER digits, which */ + /* sets the tradeoff against the cost of the later a**(10**h). */ + /* As h is increased, the working precision below also */ + /* increases to compensate for the "constant digits at the */ + /* front" effect. */ + Int lever=MINI(8-h, maxlever); /* leverage attainable */ + Int use=-rhs->digits-lever; /* exponent to use for RHS */ + h+=lever; /* apply leverage selected */ + if (h<0) { /* clamp */ + use+=h; /* [may end up subnormal] */ + h=0; + } + /* Take a copy of RHS if it needs normalization (true whenever x>=1) */ + if (rhs->exponent!=use) { + decNumber *newrhs=bufr; /* assume will fit on stack */ + needbytes=sizeof(decNumber)+(D2U(rhs->digits)-1)*sizeof(Unit); + if (needbytes>sizeof(bufr)) { /* need malloc space */ + allocrhs=(decNumber *)malloc(needbytes); + if (allocrhs==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + newrhs=allocrhs; /* use the allocated space */ + } + uprv_decNumberCopy(newrhs, rhs); /* copy to safe space */ + newrhs->exponent=use; /* normalize; now <1 */ + x=newrhs; /* ready for use */ + /* decNumberShow(x); */ + } + + /* Now use the usual power series to evaluate exp(x). The */ + /* series starts as 1 + x + x^2/2 ... so prime ready for the */ + /* third term by setting the term variable t=x, the accumulator */ + /* a=1, and the divisor d=2. */ + + /* First determine the working precision. From Hull & Abrham */ + /* this is set->digits+h+2. However, if x is 'over-precise' we */ + /* need to allow for all its digits to potentially participate */ + /* (consider an x where all the excess digits are 9s) so in */ + /* this case use x->digits+h+2 */ + p=MAXI(x->digits, set->digits)+h+2; /* [h<=8] */ + + /* a and t are variable precision, and depend on p, so space */ + /* must be allocated for them if necessary */ + + /* the accumulator needs to be able to hold 2p digits so that */ + /* the additions on the second and subsequent iterations are */ + /* sufficiently exact. */ + needbytes=sizeof(decNumber)+(D2U(p*2)-1)*sizeof(Unit); + if (needbytes>sizeof(bufa)) { /* need malloc space */ + allocbufa=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + a=allocbufa; /* use the allocated space */ + } + /* the term needs to be able to hold p digits (which is */ + /* guaranteed to be larger than x->digits, so the initial copy */ + /* is safe); it may also be used for the raise-to-power */ + /* calculation below, which needs an extra two digits */ + needbytes=sizeof(decNumber)+(D2U(p+2)-1)*sizeof(Unit); + if (needbytes>sizeof(buft)) { /* need malloc space */ + allocbuft=(decNumber *)malloc(needbytes); + if (allocbuft==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + t=allocbuft; /* use the allocated space */ + } + + uprv_decNumberCopy(t, x); /* term=x */ + uprv_decNumberZero(a); *a->lsu=1; /* accumulator=1 */ + uprv_decNumberZero(d); *d->lsu=2; /* divisor=2 */ + uprv_decNumberZero(&numone); *numone.lsu=1; /* constant 1 for increment */ + + /* set up the contexts for calculating a, t, and d */ + uprv_decContextDefault(&tset, DEC_INIT_DECIMAL64); + dset=tset; + /* accumulator bounds are set above, set precision now */ + aset.digits=p*2; /* double */ + /* term bounds avoid any underflow or overflow */ + tset.digits=p; + tset.emin=DEC_MIN_EMIN; /* [emax is plenty] */ + /* [dset.digits=16, etc., are sufficient] */ + + /* finally ready to roll */ + for (;;) { + #if DECCHECK + iterations++; + #endif + /* only the status from the accumulation is interesting */ + /* [but it should remain unchanged after first add] */ + decAddOp(a, a, t, &aset, 0, status); /* a=a+t */ + decMultiplyOp(t, t, x, &tset, &ignore); /* t=t*x */ + decDivideOp(t, t, d, &tset, DIVIDE, &ignore); /* t=t/d */ + /* the iteration ends when the term cannot affect the result, */ + /* if rounded to p digits, which is when its value is smaller */ + /* than the accumulator by p+1 digits. There must also be */ + /* full precision in a. */ + if (((a->digits+a->exponent)>=(t->digits+t->exponent+p+1)) + && (a->digits>=p)) break; + decAddOp(d, d, &numone, &dset, 0, &ignore); /* d=d+1 */ + } /* iterate */ + + #if DECCHECK + /* just a sanity check; comment out test to show always */ + if (iterations>p+3) + printf("Exp iterations=%ld, status=%08lx, p=%ld, d=%ld\n", + (LI)iterations, (LI)*status, (LI)p, (LI)x->digits); + #endif + } /* h<=8 */ + + /* apply postconditioning: a=a**(10**h) -- this is calculated */ + /* at a slightly higher precision than Hull & Abrham suggest */ + if (h>0) { + Int seenbit=0; /* set once a 1-bit is seen */ + Int i; /* counter */ + Int n=powers[h]; /* always positive */ + aset.digits=p+2; /* sufficient precision */ + /* avoid the overhead and many extra digits of decNumberPower */ + /* as all that is needed is the short 'multipliers' loop; here */ + /* accumulate the answer into t */ + uprv_decNumberZero(t); *t->lsu=1; /* acc=1 */ + for (i=1;;i++){ /* for each bit [top bit ignored] */ + /* abandon if have had overflow or terminal underflow */ + if (*status & (DEC_Overflow|DEC_Underflow)) { /* interesting? */ + if (*status&DEC_Overflow || ISZERO(t)) break;} + n=n<<1; /* move next bit to testable position */ + if (n<0) { /* top bit is set */ + seenbit=1; /* OK, have a significant bit */ + decMultiplyOp(t, t, a, &aset, status); /* acc=acc*x */ + } + if (i==31) break; /* that was the last bit */ + if (!seenbit) continue; /* no need to square 1 */ + decMultiplyOp(t, t, t, &aset, status); /* acc=acc*acc [square] */ + } /*i*/ /* 32 bits */ + /* decNumberShow(t); */ + a=t; /* and carry on using t instead of a */ + } + + /* Copy and round the result to res */ + residue=1; /* indicate dirt to right .. */ + if (ISZERO(a)) residue=0; /* .. unless underflowed to 0 */ + aset.digits=set->digits; /* [use default rounding] */ + decCopyFit(res, a, &aset, &residue, status); /* copy & shorten */ + decFinish(res, set, &residue, status); /* cleanup/set flags */ + } while(0); /* end protected */ + + if (allocrhs !=nullptr) free(allocrhs); /* drop any storage used */ + if (allocbufa!=nullptr) free(allocbufa); /* .. */ + if (allocbuft!=nullptr) free(allocbuft); /* .. */ + /* [status is handled by caller] */ + return res; + } /* decExpOp */ + +/* ------------------------------------------------------------------ */ +/* Initial-estimate natural logarithm table */ +/* */ +/* LNnn -- 90-entry 16-bit table for values from .10 through .99. */ +/* The result is a 4-digit encode of the coefficient (c=the */ +/* top 14 bits encoding 0-9999) and a 2-digit encode of the */ +/* exponent (e=the bottom 2 bits encoding 0-3) */ +/* */ +/* The resulting value is given by: */ +/* */ +/* v = -c * 10**(-e-3) */ +/* */ +/* where e and c are extracted from entry k = LNnn[x-10] */ +/* where x is truncated (NB) into the range 10 through 99, */ +/* and then c = k>>2 and e = k&3. */ +/* ------------------------------------------------------------------ */ +static const uShort LNnn[90]={9016, 8652, 8316, 8008, 7724, 7456, 7208, + 6972, 6748, 6540, 6340, 6148, 5968, 5792, 5628, 5464, 5312, + 5164, 5020, 4884, 4748, 4620, 4496, 4376, 4256, 4144, 4032, + 39233, 38181, 37157, 36157, 35181, 34229, 33297, 32389, 31501, 30629, + 29777, 28945, 28129, 27329, 26545, 25777, 25021, 24281, 23553, 22837, + 22137, 21445, 20769, 20101, 19445, 18801, 18165, 17541, 16925, 16321, + 15721, 15133, 14553, 13985, 13421, 12865, 12317, 11777, 11241, 10717, + 10197, 9685, 9177, 8677, 8185, 7697, 7213, 6737, 6269, 5801, + 5341, 4889, 4437, 39930, 35534, 31186, 26886, 22630, 18418, 14254, + 10130, 6046, 20055}; + +/* ------------------------------------------------------------------ */ +/* decLnOp -- effect natural logarithm */ +/* */ +/* This computes C = ln(A) */ +/* */ +/* res is C, the result. C may be A */ +/* rhs is A */ +/* set is the context; note that rounding mode has no effect */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Notable cases: */ +/* A<0 -> Invalid */ +/* A=0 -> -Infinity (Exact) */ +/* A=+Infinity -> +Infinity (Exact) */ +/* A=1 exactly -> 0 (Exact) */ +/* */ +/* Restrictions (as for Exp): */ +/* */ +/* digits, emax, and -emin in the context must be less than */ +/* DEC_MAX_MATH+11 (1000010), and the rhs must be within these */ +/* bounds or a zero. This is an internal routine, so these */ +/* restrictions are contractual and not enforced. */ +/* */ +/* A finite result is rounded using DEC_ROUND_HALF_EVEN; it will */ +/* almost always be correctly rounded, but may be up to 1 ulp in */ +/* error in rare cases. */ +/* ------------------------------------------------------------------ */ +/* The result is calculated using Newton's method, with each */ +/* iteration calculating a' = a + x * exp(-a) - 1. See, for example, */ +/* Epperson 1989. */ +/* */ +/* The iteration ends when the adjustment x*exp(-a)-1 is tiny enough. */ +/* This has to be calculated at the sum of the precision of x and the */ +/* working precision. */ +/* */ +/* Implementation notes: */ +/* */ +/* 1. This is separated out as decLnOp so it can be called from */ +/* other Mathematical functions (e.g., Log 10) with a wider range */ +/* than normal. In particular, it can handle the slightly wider */ +/* (+9+2) range needed by a power function. */ +/* */ +/* 2. The speed of this function is about 10x slower than exp, as */ +/* it typically needs 4-6 iterations for short numbers, and the */ +/* extra precision needed adds a squaring effect, twice. */ +/* */ +/* 3. Fastpaths are included for ln(10) and ln(2), up to length 40, */ +/* as these are common requests. ln(10) is used by log10(x). */ +/* */ +/* 4. An iteration might be saved by widening the LNnn table, and */ +/* would certainly save at least one if it were made ten times */ +/* bigger, too (for truncated fractions 0.100 through 0.999). */ +/* However, for most practical evaluations, at least four or five */ +/* iterations will be needed -- so this would only speed up by */ +/* 20-25% and that probably does not justify increasing the table */ +/* size. */ +/* */ +/* 5. The static buffers are larger than might be expected to allow */ +/* for calls from decNumberPower. */ +/* ------------------------------------------------------------------ */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif +decNumber * decLnOp(decNumber *res, const decNumber *rhs, + decContext *set, uInt *status) { + uInt ignore=0; /* working status accumulator */ + uInt needbytes; /* for space calculations */ + Int residue; /* rounding residue */ + Int r; /* rhs=f*10**r [see below] */ + Int p; /* working precision */ + Int pp; /* precision for iteration */ + Int t; /* work */ + + /* buffers for a (accumulator, typically precision+2) and b */ + /* (adjustment calculator, same size) */ + decNumber bufa[D2N(DECBUFFER+12)]; + decNumber *allocbufa=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *a=bufa; /* accumulator/work */ + decNumber bufb[D2N(DECBUFFER*2+2)]; + decNumber *allocbufb=nullptr; /* -> allocated bufa, iff allocated */ + decNumber *b=bufb; /* adjustment/work */ + + decNumber numone; /* constant 1 */ + decNumber cmp; /* work */ + decContext aset, bset; /* working contexts */ + + #if DECCHECK + Int iterations=0; /* for later sanity check */ + if (decCheckOperands(res, DECUNUSED, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + if (SPECIALARG) { /* handle infinities and NaNs */ + if (decNumberIsInfinite(rhs)) { /* an infinity */ + if (decNumberIsNegative(rhs)) /* -Infinity -> error */ + *status|=DEC_Invalid_operation; + else uprv_decNumberCopy(res, rhs); /* +Infinity -> self */ + } + else decNaNs(res, rhs, nullptr, set, status); /* a NaN */ + break;} + + if (ISZERO(rhs)) { /* +/- zeros -> -Infinity */ + uprv_decNumberZero(res); /* make clean */ + res->bits=DECINF|DECNEG; /* set - infinity */ + break;} /* [no status to set] */ + + /* Non-zero negatives are bad... */ + if (decNumberIsNegative(rhs)) { /* -x -> error */ + *status|=DEC_Invalid_operation; + break;} + + /* Here, rhs is positive, finite, and in range */ + + /* lookaside fastpath code for ln(2) and ln(10) at common lengths */ + if (rhs->exponent==0 && set->digits<=40) { + #if DECDPUN==1 + if (rhs->lsu[0]==0 && rhs->lsu[1]==1 && rhs->digits==2) { /* ln(10) */ + #else + if (rhs->lsu[0]==10 && rhs->digits==2) { /* ln(10) */ + #endif + aset=*set; aset.round=DEC_ROUND_HALF_EVEN; + #define LN10 "2.302585092994045684017991454684364207601" + uprv_decNumberFromString(res, LN10, &aset); + *status|=(DEC_Inexact | DEC_Rounded); /* is inexact */ + break;} + if (rhs->lsu[0]==2 && rhs->digits==1) { /* ln(2) */ + aset=*set; aset.round=DEC_ROUND_HALF_EVEN; + #define LN2 "0.6931471805599453094172321214581765680755" + uprv_decNumberFromString(res, LN2, &aset); + *status|=(DEC_Inexact | DEC_Rounded); + break;} + } /* integer and short */ + + /* Determine the working precision. This is normally the */ + /* requested precision + 2, with a minimum of 9. However, if */ + /* the rhs is 'over-precise' then allow for all its digits to */ + /* potentially participate (consider an rhs where all the excess */ + /* digits are 9s) so in this case use rhs->digits+2. */ + p=MAXI(rhs->digits, MAXI(set->digits, 7))+2; + + /* Allocate space for the accumulator and the high-precision */ + /* adjustment calculator, if necessary. The accumulator must */ + /* be able to hold p digits, and the adjustment up to */ + /* rhs->digits+p digits. They are also made big enough for 16 */ + /* digits so that they can be used for calculating the initial */ + /* estimate. */ + needbytes=sizeof(decNumber)+(D2U(MAXI(p,16))-1)*sizeof(Unit); + if (needbytes>sizeof(bufa)) { /* need malloc space */ + allocbufa=(decNumber *)malloc(needbytes); + if (allocbufa==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + a=allocbufa; /* use the allocated space */ + } + pp=p+rhs->digits; + needbytes=sizeof(decNumber)+(D2U(MAXI(pp,16))-1)*sizeof(Unit); + if (needbytes>sizeof(bufb)) { /* need malloc space */ + allocbufb=(decNumber *)malloc(needbytes); + if (allocbufb==nullptr) { /* hopeless -- abandon */ + *status|=DEC_Insufficient_storage; + break;} + b=allocbufb; /* use the allocated space */ + } + + /* Prepare an initial estimate in acc. Calculate this by */ + /* considering the coefficient of x to be a normalized fraction, */ + /* f, with the decimal point at far left and multiplied by */ + /* 10**r. Then, rhs=f*10**r and 0.1<=f<1, and */ + /* ln(x) = ln(f) + ln(10)*r */ + /* Get the initial estimate for ln(f) from a small lookup */ + /* table (see above) indexed by the first two digits of f, */ + /* truncated. */ + + uprv_decContextDefault(&aset, DEC_INIT_DECIMAL64); /* 16-digit extended */ + r=rhs->exponent+rhs->digits; /* 'normalised' exponent */ + uprv_decNumberFromInt32(a, r); /* a=r */ + uprv_decNumberFromInt32(b, 2302585); /* b=ln(10) (2.302585) */ + b->exponent=-6; /* .. */ + decMultiplyOp(a, a, b, &aset, &ignore); /* a=a*b */ + /* now get top two digits of rhs into b by simple truncate and */ + /* force to integer */ + residue=0; /* (no residue) */ + aset.digits=2; aset.round=DEC_ROUND_DOWN; + decCopyFit(b, rhs, &aset, &residue, &ignore); /* copy & shorten */ + b->exponent=0; /* make integer */ + t=decGetInt(b); /* [cannot fail] */ + if (t<10) t=X10(t); /* adjust single-digit b */ + t=LNnn[t-10]; /* look up ln(b) */ + uprv_decNumberFromInt32(b, t>>2); /* b=ln(b) coefficient */ + b->exponent=-(t&3)-3; /* set exponent */ + b->bits=DECNEG; /* ln(0.10)->ln(0.99) always -ve */ + aset.digits=16; aset.round=DEC_ROUND_HALF_EVEN; /* restore */ + decAddOp(a, a, b, &aset, 0, &ignore); /* acc=a+b */ + /* the initial estimate is now in a, with up to 4 digits correct. */ + /* When rhs is at or near Nmax the estimate will be low, so we */ + /* will approach it from below, avoiding overflow when calling exp. */ + + uprv_decNumberZero(&numone); *numone.lsu=1; /* constant 1 for adjustment */ + + /* accumulator bounds are as requested (could underflow, but */ + /* cannot overflow) */ + aset.emax=set->emax; + aset.emin=set->emin; + aset.clamp=0; /* no concrete format */ + /* set up a context to be used for the multiply and subtract */ + bset=aset; + bset.emax=DEC_MAX_MATH*2; /* use double bounds for the */ + bset.emin=-DEC_MAX_MATH*2; /* adjustment calculation */ + /* [see decExpOp call below] */ + /* for each iteration double the number of digits to calculate, */ + /* up to a maximum of p */ + pp=9; /* initial precision */ + /* [initially 9 as then the sequence starts 7+2, 16+2, and */ + /* 34+2, which is ideal for standard-sized numbers] */ + aset.digits=pp; /* working context */ + bset.digits=pp+rhs->digits; /* wider context */ + for (;;) { /* iterate */ + #if DECCHECK + iterations++; + if (iterations>24) break; /* consider 9 * 2**24 */ + #endif + /* calculate the adjustment (exp(-a)*x-1) into b. This is a */ + /* catastrophic subtraction but it really is the difference */ + /* from 1 that is of interest. */ + /* Use the internal entry point to Exp as it allows the double */ + /* range for calculating exp(-a) when a is the tiniest subnormal. */ + a->bits^=DECNEG; /* make -a */ + decExpOp(b, a, &bset, &ignore); /* b=exp(-a) */ + a->bits^=DECNEG; /* restore sign of a */ + /* now multiply by rhs and subtract 1, at the wider precision */ + decMultiplyOp(b, b, rhs, &bset, &ignore); /* b=b*rhs */ + decAddOp(b, b, &numone, &bset, DECNEG, &ignore); /* b=b-1 */ + + /* the iteration ends when the adjustment cannot affect the */ + /* result by >=0.5 ulp (at the requested digits), which */ + /* is when its value is smaller than the accumulator by */ + /* set->digits+1 digits (or it is zero) -- this is a looser */ + /* requirement than for Exp because all that happens to the */ + /* accumulator after this is the final rounding (but note that */ + /* there must also be full precision in a, or a=0). */ + + if (decNumberIsZero(b) || + (a->digits+a->exponent)>=(b->digits+b->exponent+set->digits+1)) { + if (a->digits==p) break; + if (decNumberIsZero(a)) { + decCompareOp(&cmp, rhs, &numone, &aset, COMPARE, &ignore); /* rhs=1 ? */ + if (cmp.lsu[0]==0) a->exponent=0; /* yes, exact 0 */ + else *status|=(DEC_Inexact | DEC_Rounded); /* no, inexact */ + break; + } + /* force padding if adjustment has gone to 0 before full length */ + if (decNumberIsZero(b)) b->exponent=a->exponent-p; + } + + /* not done yet ... */ + decAddOp(a, a, b, &aset, 0, &ignore); /* a=a+b for next estimate */ + if (pp==p) continue; /* precision is at maximum */ + /* lengthen the next calculation */ + pp=pp*2; /* double precision */ + if (pp>p) pp=p; /* clamp to maximum */ + aset.digits=pp; /* working context */ + bset.digits=pp+rhs->digits; /* wider context */ + } /* Newton's iteration */ + + #if DECCHECK + /* just a sanity check; remove the test to show always */ + if (iterations>24) + printf("Ln iterations=%ld, status=%08lx, p=%ld, d=%ld\n", + (LI)iterations, (LI)*status, (LI)p, (LI)rhs->digits); + #endif + + /* Copy and round the result to res */ + residue=1; /* indicate dirt to right */ + if (ISZERO(a)) residue=0; /* .. unless underflowed to 0 */ + aset.digits=set->digits; /* [use default rounding] */ + decCopyFit(res, a, &aset, &residue, status); /* copy & shorten */ + decFinish(res, set, &residue, status); /* cleanup/set flags */ + } while(0); /* end protected */ + + if (allocbufa!=nullptr) free(allocbufa); /* drop any storage used */ + if (allocbufb!=nullptr) free(allocbufb); /* .. */ + /* [status is handled by caller] */ + return res; + } /* decLnOp */ +#if defined(__clang__) || U_GCC_MAJOR_MINOR >= 406 +#pragma GCC diagnostic pop +#endif + +/* ------------------------------------------------------------------ */ +/* decQuantizeOp -- force exponent to requested value */ +/* */ +/* This computes C = op(A, B), where op adjusts the coefficient */ +/* of C (by rounding or shifting) such that the exponent (-scale) */ +/* of C has the value B or matches the exponent of B. */ +/* The numerical value of C will equal A, except for the effects of */ +/* any rounding that occurred. */ +/* */ +/* res is C, the result. C may be A or B */ +/* lhs is A, the number to adjust */ +/* rhs is B, the requested exponent */ +/* set is the context */ +/* quant is 1 for quantize or 0 for rescale */ +/* status is the status accumulator (this can be called without */ +/* risk of control loss) */ +/* */ +/* C must have space for set->digits digits. */ +/* */ +/* Unless there is an error or the result is infinite, the exponent */ +/* after the operation is guaranteed to be that requested. */ +/* ------------------------------------------------------------------ */ +static decNumber * decQuantizeOp(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set, + Flag quant, uInt *status) { + #if DECSUBSET + decNumber *alloclhs=nullptr; /* non-nullptr if rounded lhs allocated */ + decNumber *allocrhs=nullptr; /* .., rhs */ + #endif + const decNumber *inrhs=rhs; /* save original rhs */ + Int reqdigits=set->digits; /* requested DIGITS */ + Int reqexp; /* requested exponent [-scale] */ + Int residue=0; /* rounding residue */ + Int etiny=set->emin-(reqdigits-1); + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operands and set lostDigits status, as needed */ + if (lhs->digits>reqdigits) { + alloclhs=decRoundOperand(lhs, set, status); + if (alloclhs==nullptr) break; + lhs=alloclhs; + } + if (rhs->digits>reqdigits) { /* [this only checks lostDigits] */ + allocrhs=decRoundOperand(rhs, set, status); + if (allocrhs==nullptr) break; + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* Handle special values */ + if (SPECIALARGS) { + /* NaNs get usual processing */ + if (SPECIALARGS & (DECSNAN | DECNAN)) + decNaNs(res, lhs, rhs, set, status); + /* one infinity but not both is bad */ + else if ((lhs->bits ^ rhs->bits) & DECINF) + *status|=DEC_Invalid_operation; + /* both infinity: return lhs */ + else uprv_decNumberCopy(res, lhs); /* [nop if in place] */ + break; + } + + /* set requested exponent */ + if (quant) reqexp=inrhs->exponent; /* quantize -- match exponents */ + else { /* rescale -- use value of rhs */ + /* Original rhs must be an integer that fits and is in range, */ + /* which could be from -1999999997 to +999999999, thanks to */ + /* subnormals */ + reqexp=decGetInt(inrhs); /* [cannot fail] */ + } + + #if DECSUBSET + if (!set->extended) etiny=set->emin; /* no subnormals */ + #endif + + if (reqexp==BADINT /* bad (rescale only) or .. */ + || reqexp==BIGODD || reqexp==BIGEVEN /* very big (ditto) or .. */ + || (reqexpset->emax)) { /* > emax */ + *status|=DEC_Invalid_operation; + break;} + + /* the RHS has been processed, so it can be overwritten now if necessary */ + if (ISZERO(lhs)) { /* zero coefficient unchanged */ + uprv_decNumberCopy(res, lhs); /* [nop if in place] */ + res->exponent=reqexp; /* .. just set exponent */ + #if DECSUBSET + if (!set->extended) res->bits=0; /* subset specification; no -0 */ + #endif + } + else { /* non-zero lhs */ + Int adjust=reqexp-lhs->exponent; /* digit adjustment needed */ + /* if adjusted coefficient will definitely not fit, give up now */ + if ((lhs->digits-adjust)>reqdigits) { + *status|=DEC_Invalid_operation; + break; + } + + if (adjust>0) { /* increasing exponent */ + /* this will decrease the length of the coefficient by adjust */ + /* digits, and must round as it does so */ + decContext workset; /* work */ + workset=*set; /* clone rounding, etc. */ + workset.digits=lhs->digits-adjust; /* set requested length */ + /* [note that the latter can be <1, here] */ + decCopyFit(res, lhs, &workset, &residue, status); /* fit to result */ + decApplyRound(res, &workset, residue, status); /* .. and round */ + residue=0; /* [used] */ + /* If just rounded a 999s case, exponent will be off by one; */ + /* adjust back (after checking space), if so. */ + if (res->exponent>reqexp) { + /* re-check needed, e.g., for quantize(0.9999, 0.001) under */ + /* set->digits==3 */ + if (res->digits==reqdigits) { /* cannot shift by 1 */ + *status&=~(DEC_Inexact | DEC_Rounded); /* [clean these] */ + *status|=DEC_Invalid_operation; + break; + } + res->digits=decShiftToMost(res->lsu, res->digits, 1); /* shift */ + res->exponent--; /* (re)adjust the exponent. */ + } + #if DECSUBSET + if (ISZERO(res) && !set->extended) res->bits=0; /* subset; no -0 */ + #endif + } /* increase */ + else /* adjust<=0 */ { /* decreasing or = exponent */ + /* this will increase the length of the coefficient by -adjust */ + /* digits, by adding zero or more trailing zeros; this is */ + /* already checked for fit, above */ + uprv_decNumberCopy(res, lhs); /* [it will fit] */ + /* if padding needed (adjust<0), add it now... */ + if (adjust<0) { + res->digits=decShiftToMost(res->lsu, res->digits, -adjust); + res->exponent+=adjust; /* adjust the exponent */ + } + } /* decrease */ + } /* non-zero */ + + /* Check for overflow [do not use Finalize in this case, as an */ + /* overflow here is a "don't fit" situation] */ + if (res->exponent>set->emax-res->digits+1) { /* too big */ + *status|=DEC_Invalid_operation; + break; + } + else { + decFinalize(res, set, &residue, status); /* set subnormal flags */ + *status&=~DEC_Underflow; /* suppress Underflow [as per 754] */ + } + } while(0); /* end protected */ + + #if DECSUBSET + if (allocrhs!=nullptr) free(allocrhs); /* drop any storage used */ + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + #endif + return res; + } /* decQuantizeOp */ + +/* ------------------------------------------------------------------ */ +/* decCompareOp -- compare, min, or max two Numbers */ +/* */ +/* This computes C = A ? B and carries out one of four operations: */ +/* COMPARE -- returns the signum (as a number) giving the */ +/* result of a comparison unless one or both */ +/* operands is a NaN (in which case a NaN results) */ +/* COMPSIG -- as COMPARE except that a quiet NaN raises */ +/* Invalid operation. */ +/* COMPMAX -- returns the larger of the operands, using the */ +/* 754 maxnum operation */ +/* COMPMAXMAG -- ditto, comparing absolute values */ +/* COMPMIN -- the 754 minnum operation */ +/* COMPMINMAG -- ditto, comparing absolute values */ +/* COMTOTAL -- returns the signum (as a number) giving the */ +/* result of a comparison using 754 total ordering */ +/* */ +/* res is C, the result. C may be A and/or B (e.g., X=X?X) */ +/* lhs is A */ +/* rhs is B */ +/* set is the context */ +/* op is the operation flag */ +/* status is the usual accumulator */ +/* */ +/* C must have space for one digit for COMPARE or set->digits for */ +/* COMPMAX, COMPMIN, COMPMAXMAG, or COMPMINMAG. */ +/* ------------------------------------------------------------------ */ +/* The emphasis here is on speed for common cases, and avoiding */ +/* coefficient comparison if possible. */ +/* ------------------------------------------------------------------ */ +static decNumber * decCompareOp(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set, + Flag op, uInt *status) { + #if DECSUBSET + decNumber *alloclhs=nullptr; /* non-nullptr if rounded lhs allocated */ + decNumber *allocrhs=nullptr; /* .., rhs */ + #endif + Int result=0; /* default result value */ + uByte merged; /* work */ + + #if DECCHECK + if (decCheckOperands(res, lhs, rhs, set)) return res; + #endif + + do { /* protect allocated storage */ + #if DECSUBSET + if (!set->extended) { + /* reduce operands and set lostDigits status, as needed */ + if (lhs->digits>set->digits) { + alloclhs=decRoundOperand(lhs, set, status); + if (alloclhs==nullptr) {result=BADINT; break;} + lhs=alloclhs; + } + if (rhs->digits>set->digits) { + allocrhs=decRoundOperand(rhs, set, status); + if (allocrhs==nullptr) {result=BADINT; break;} + rhs=allocrhs; + } + } + #endif + /* [following code does not require input rounding] */ + + /* If total ordering then handle differing signs 'up front' */ + if (op==COMPTOTAL) { /* total ordering */ + if (decNumberIsNegative(lhs) && !decNumberIsNegative(rhs)) { + result=-1; + break; + } + if (!decNumberIsNegative(lhs) && decNumberIsNegative(rhs)) { + result=+1; + break; + } + } + + /* handle NaNs specially; let infinities drop through */ + /* This assumes sNaN (even just one) leads to NaN. */ + merged=(lhs->bits | rhs->bits) & (DECSNAN | DECNAN); + if (merged) { /* a NaN bit set */ + if (op==COMPARE); /* result will be NaN */ + else if (op==COMPSIG) /* treat qNaN as sNaN */ + *status|=DEC_Invalid_operation | DEC_sNaN; + else if (op==COMPTOTAL) { /* total ordering, always finite */ + /* signs are known to be the same; compute the ordering here */ + /* as if the signs are both positive, then invert for negatives */ + if (!decNumberIsNaN(lhs)) result=-1; + else if (!decNumberIsNaN(rhs)) result=+1; + /* here if both NaNs */ + else if (decNumberIsSNaN(lhs) && decNumberIsQNaN(rhs)) result=-1; + else if (decNumberIsQNaN(lhs) && decNumberIsSNaN(rhs)) result=+1; + else { /* both NaN or both sNaN */ + /* now it just depends on the payload */ + result=decUnitCompare(lhs->lsu, D2U(lhs->digits), + rhs->lsu, D2U(rhs->digits), 0); + /* [Error not possible, as these are 'aligned'] */ + } /* both same NaNs */ + if (decNumberIsNegative(lhs)) result=-result; + break; + } /* total order */ + + else if (merged & DECSNAN); /* sNaN -> qNaN */ + else { /* here if MIN or MAX and one or two quiet NaNs */ + /* min or max -- 754 rules ignore single NaN */ + if (!decNumberIsNaN(lhs) || !decNumberIsNaN(rhs)) { + /* just one NaN; force choice to be the non-NaN operand */ + op=COMPMAX; + if (lhs->bits & DECNAN) result=-1; /* pick rhs */ + else result=+1; /* pick lhs */ + break; + } + } /* max or min */ + op=COMPNAN; /* use special path */ + decNaNs(res, lhs, rhs, set, status); /* propagate NaN */ + break; + } + /* have numbers */ + if (op==COMPMAXMAG || op==COMPMINMAG) result=decCompare(lhs, rhs, 1); + else result=decCompare(lhs, rhs, 0); /* sign matters */ + } while(0); /* end protected */ + + if (result==BADINT) *status|=DEC_Insufficient_storage; /* rare */ + else { + if (op==COMPARE || op==COMPSIG ||op==COMPTOTAL) { /* returning signum */ + if (op==COMPTOTAL && result==0) { + /* operands are numerically equal or same NaN (and same sign, */ + /* tested first); if identical, leave result 0 */ + if (lhs->exponent!=rhs->exponent) { + if (lhs->exponentexponent) result=-1; + else result=+1; + if (decNumberIsNegative(lhs)) result=-result; + } /* lexp!=rexp */ + } /* total-order by exponent */ + uprv_decNumberZero(res); /* [always a valid result] */ + if (result!=0) { /* must be -1 or +1 */ + *res->lsu=1; + if (result<0) res->bits=DECNEG; + } + } + else if (op==COMPNAN); /* special, drop through */ + else { /* MAX or MIN, non-NaN result */ + Int residue=0; /* rounding accumulator */ + /* choose the operand for the result */ + const decNumber *choice; + if (result==0) { /* operands are numerically equal */ + /* choose according to sign then exponent (see 754) */ + uByte slhs=(lhs->bits & DECNEG); + uByte srhs=(rhs->bits & DECNEG); + #if DECSUBSET + if (!set->extended) { /* subset: force left-hand */ + op=COMPMAX; + result=+1; + } + else + #endif + if (slhs!=srhs) { /* signs differ */ + if (slhs) result=-1; /* rhs is max */ + else result=+1; /* lhs is max */ + } + else if (slhs && srhs) { /* both negative */ + if (lhs->exponentexponent) result=+1; + else result=-1; + /* [if equal, use lhs, technically identical] */ + } + else { /* both positive */ + if (lhs->exponent>rhs->exponent) result=+1; + else result=-1; + /* [ditto] */ + } + } /* numerically equal */ + /* here result will be non-0; reverse if looking for MIN */ + if (op==COMPMIN || op==COMPMINMAG) result=-result; + choice=(result>0 ? lhs : rhs); /* choose */ + /* copy chosen to result, rounding if need be */ + decCopyFit(res, choice, set, &residue, status); + decFinish(res, set, &residue, status); + } + } + #if DECSUBSET + if (allocrhs!=nullptr) free(allocrhs); /* free any storage used */ + if (alloclhs!=nullptr) free(alloclhs); /* .. */ + #endif + return res; + } /* decCompareOp */ + +/* ------------------------------------------------------------------ */ +/* decCompare -- compare two decNumbers by numerical value */ +/* */ +/* This routine compares A ? B without altering them. */ +/* */ +/* Arg1 is A, a decNumber which is not a NaN */ +/* Arg2 is B, a decNumber which is not a NaN */ +/* Arg3 is 1 for a sign-independent compare, 0 otherwise */ +/* */ +/* returns -1, 0, or 1 for AB, or BADINT if failure */ +/* (the only possible failure is an allocation error) */ +/* ------------------------------------------------------------------ */ +static Int decCompare(const decNumber *lhs, const decNumber *rhs, + Flag abs_c) { + Int result; /* result value */ + Int sigr; /* rhs signum */ + Int compare; /* work */ + + result=1; /* assume signum(lhs) */ + if (ISZERO(lhs)) result=0; + if (abs_c) { + if (ISZERO(rhs)) return result; /* LHS wins or both 0 */ + /* RHS is non-zero */ + if (result==0) return -1; /* LHS is 0; RHS wins */ + /* [here, both non-zero, result=1] */ + } + else { /* signs matter */ + if (result && decNumberIsNegative(lhs)) result=-1; + sigr=1; /* compute signum(rhs) */ + if (ISZERO(rhs)) sigr=0; + else if (decNumberIsNegative(rhs)) sigr=-1; + if (result > sigr) return +1; /* L > R, return 1 */ + if (result < sigr) return -1; /* L < R, return -1 */ + if (result==0) return 0; /* both 0 */ + } + + /* signums are the same; both are non-zero */ + if ((lhs->bits | rhs->bits) & DECINF) { /* one or more infinities */ + if (decNumberIsInfinite(rhs)) { + if (decNumberIsInfinite(lhs)) result=0;/* both infinite */ + else result=-result; /* only rhs infinite */ + } + return result; + } + /* must compare the coefficients, allowing for exponents */ + if (lhs->exponent>rhs->exponent) { /* LHS exponent larger */ + /* swap sides, and sign */ + const decNumber *temp=lhs; + lhs=rhs; + rhs=temp; + result=-result; + } + compare=decUnitCompare(lhs->lsu, D2U(lhs->digits), + rhs->lsu, D2U(rhs->digits), + rhs->exponent-lhs->exponent); + if (compare!=BADINT) compare*=result; /* comparison succeeded */ + return compare; + } /* decCompare */ + +/* ------------------------------------------------------------------ */ +/* decUnitCompare -- compare two >=0 integers in Unit arrays */ +/* */ +/* This routine compares A ? B*10**E where A and B are unit arrays */ +/* A is a plain integer */ +/* B has an exponent of E (which must be non-negative) */ +/* */ +/* Arg1 is A first Unit (lsu) */ +/* Arg2 is A length in Units */ +/* Arg3 is B first Unit (lsu) */ +/* Arg4 is B length in Units */ +/* Arg5 is E (0 if the units are aligned) */ +/* */ +/* returns -1, 0, or 1 for AB, or BADINT if failure */ +/* (the only possible failure is an allocation error, which can */ +/* only occur if E!=0) */ +/* ------------------------------------------------------------------ */ +static Int decUnitCompare(const Unit *a, Int alength, + const Unit *b, Int blength, Int exp) { + Unit *acc; /* accumulator for result */ + Unit accbuff[SD2U(DECBUFFER*2+1)]; /* local buffer */ + Unit *allocacc=nullptr; /* -> allocated acc buffer, iff allocated */ + Int accunits, need; /* units in use or needed for acc */ + const Unit *l, *r, *u; /* work */ + Int expunits, exprem, result; /* .. */ + + if (exp==0) { /* aligned; fastpath */ + if (alength>blength) return 1; + if (alength=a; l--, r--) { + if (*l>*r) return 1; + if (*l<*r) return -1; + } + return 0; /* all units match */ + } /* aligned */ + + /* Unaligned. If one is >1 unit longer than the other, padded */ + /* approximately, then can return easily */ + if (alength>blength+(Int)D2U(exp)) return 1; + if (alength+1sizeof(accbuff)) { + allocacc=(Unit *)malloc(need*sizeof(Unit)); + if (allocacc==nullptr) return BADINT; /* hopeless -- abandon */ + acc=allocacc; + } + /* Calculate units and remainder from exponent. */ + expunits=exp/DECDPUN; + exprem=exp%DECDPUN; + /* subtract [A+B*(-m)] */ + accunits=decUnitAddSub(a, alength, b, blength, expunits, acc, + -(Int)powers[exprem]); + /* [UnitAddSub result may have leading zeros, even on zero] */ + if (accunits<0) result=-1; /* negative result */ + else { /* non-negative result */ + /* check units of the result before freeing any storage */ + for (u=acc; u=0 integers in Unit arrays */ +/* */ +/* This routine performs the calculation: */ +/* */ +/* C=A+(B*M) */ +/* */ +/* Where M is in the range -DECDPUNMAX through +DECDPUNMAX. */ +/* */ +/* A may be shorter or longer than B. */ +/* */ +/* Leading zeros are not removed after a calculation. The result is */ +/* either the same length as the longer of A and B (adding any */ +/* shift), or one Unit longer than that (if a Unit carry occurred). */ +/* */ +/* A and B content are not altered unless C is also A or B. */ +/* C may be the same array as A or B, but only if no zero padding is */ +/* requested (that is, C may be B only if bshift==0). */ +/* C is filled from the lsu; only those units necessary to complete */ +/* the calculation are referenced. */ +/* */ +/* Arg1 is A first Unit (lsu) */ +/* Arg2 is A length in Units */ +/* Arg3 is B first Unit (lsu) */ +/* Arg4 is B length in Units */ +/* Arg5 is B shift in Units (>=0; pads with 0 units if positive) */ +/* Arg6 is C first Unit (lsu) */ +/* Arg7 is M, the multiplier */ +/* */ +/* returns the count of Units written to C, which will be non-zero */ +/* and negated if the result is negative. That is, the sign of the */ +/* returned Int is the sign of the result (positive for zero) and */ +/* the absolute value of the Int is the count of Units. */ +/* */ +/* It is the caller's responsibility to make sure that C size is */ +/* safe, allowing space if necessary for a one-Unit carry. */ +/* */ +/* This routine is severely performance-critical; *any* change here */ +/* must be measured (timed) to assure no performance degradation. */ +/* In particular, trickery here tends to be counter-productive, as */ +/* increased complexity of code hurts register optimizations on */ +/* register-poor architectures. Avoiding divisions is nearly */ +/* always a Good Idea, however. */ +/* */ +/* Special thanks to Rick McGuire (IBM Cambridge, MA) and Dave Clark */ +/* (IBM Warwick, UK) for some of the ideas used in this routine. */ +/* ------------------------------------------------------------------ */ +static Int decUnitAddSub(const Unit *a, Int alength, + const Unit *b, Int blength, Int bshift, + Unit *c, Int m) { + const Unit *alsu=a; /* A lsu [need to remember it] */ + Unit *clsu=c; /* C ditto */ + Unit *minC; /* low water mark for C */ + Unit *maxC; /* high water mark for C */ + eInt carry=0; /* carry integer (could be Long) */ + Int add; /* work */ + #if DECDPUN<=4 /* myriadal, millenary, etc. */ + Int est; /* estimated quotient */ + #endif + + #if DECTRACE + if (alength<1 || blength<1) + printf("decUnitAddSub: alen blen m %ld %ld [%ld]\n", alength, blength, m); + #endif + + maxC=c+alength; /* A is usually the longer */ + minC=c+blength; /* .. and B the shorter */ + if (bshift!=0) { /* B is shifted; low As copy across */ + minC+=bshift; + /* if in place [common], skip copy unless there's a gap [rare] */ + if (a==c && bshift<=alength) { + c+=bshift; + a+=bshift; + } + else for (; cmaxC) { /* swap */ + Unit *hold=minC; + minC=maxC; + maxC=hold; + } + + /* For speed, do the addition as two loops; the first where both A */ + /* and B contribute, and the second (if necessary) where only one or */ + /* other of the numbers contribute. */ + /* Carry handling is the same (i.e., duplicated) in each case. */ + for (; c=0) { + est=(((ueInt)carry>>11)*53687)>>18; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* likely quotient [89%] */ + if (*c>11)*53687)>>18; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + if (*c=0) { + est=(((ueInt)carry>>3)*16777)>>21; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* likely quotient [99%] */ + if (*c>3)*16777)>>21; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + if (*c=0) { + est=QUOT10(carry, DECDPUN); + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* quotient */ + continue; + } + /* negative case */ + carry=carry+(eInt)(DECDPUNMAX+1)*(DECDPUNMAX+1); /* make positive */ + est=QUOT10(carry, DECDPUN); + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + #else + /* remainder operator is undefined if negative, so must test */ + if ((ueInt)carry<(DECDPUNMAX+1)*2) { /* fastpath carry +1 */ + *c=(Unit)(carry-(DECDPUNMAX+1)); /* [helps additions] */ + carry=1; + continue; + } + if (carry>=0) { + *c=(Unit)(carry%(DECDPUNMAX+1)); + carry=carry/(DECDPUNMAX+1); + continue; + } + /* negative case */ + carry=carry+(eInt)(DECDPUNMAX+1)*(DECDPUNMAX+1); /* make positive */ + *c=(Unit)(carry%(DECDPUNMAX+1)); + carry=carry/(DECDPUNMAX+1)-(DECDPUNMAX+1); + #endif + } /* c */ + + /* now may have one or other to complete */ + /* [pretest to avoid loop setup/shutdown] */ + if (cDECDPUNMAX */ + #if DECDPUN==4 /* use divide-by-multiply */ + if (carry>=0) { + est=(((ueInt)carry>>11)*53687)>>18; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* likely quotient [79.7%] */ + if (*c>11)*53687)>>18; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + if (*c=0) { + est=(((ueInt)carry>>3)*16777)>>21; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* likely quotient [99%] */ + if (*c>3)*16777)>>21; + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + if (*c=0) { + est=QUOT10(carry, DECDPUN); + *c=(Unit)(carry-est*(DECDPUNMAX+1)); /* remainder */ + carry=est; /* quotient */ + continue; + } + /* negative case */ + carry=carry+(eInt)(DECDPUNMAX+1)*(DECDPUNMAX+1); /* make positive */ + est=QUOT10(carry, DECDPUN); + *c=(Unit)(carry-est*(DECDPUNMAX+1)); + carry=est-(DECDPUNMAX+1); /* correctly negative */ + #else + if ((ueInt)carry<(DECDPUNMAX+1)*2){ /* fastpath carry 1 */ + *c=(Unit)(carry-(DECDPUNMAX+1)); + carry=1; + continue; + } + /* remainder operator is undefined if negative, so must test */ + if (carry>=0) { + *c=(Unit)(carry%(DECDPUNMAX+1)); + carry=carry/(DECDPUNMAX+1); + continue; + } + /* negative case */ + carry=carry+(eInt)(DECDPUNMAX+1)*(DECDPUNMAX+1); /* make positive */ + *c=(Unit)(carry%(DECDPUNMAX+1)); + carry=carry/(DECDPUNMAX+1)-(DECDPUNMAX+1); + #endif + } /* c */ + + /* OK, all A and B processed; might still have carry or borrow */ + /* return number of Units in the result, negated if a borrow */ + if (carry==0) return static_cast(c-clsu); /* no carry, so no more to do */ + if (carry>0) { /* positive carry */ + *c=(Unit)carry; /* place as new unit */ + c++; /* .. */ + return static_cast(c-clsu); + } + /* -ve carry: it's a borrow; complement needed */ + add=1; /* temporary carry... */ + for (c=clsu; c(clsu-c); /* -ve result indicates borrowed */ + } /* decUnitAddSub */ + +/* ------------------------------------------------------------------ */ +/* decTrim -- trim trailing zeros or normalize */ +/* */ +/* dn is the number to trim or normalize */ +/* set is the context to use to check for clamp */ +/* all is 1 to remove all trailing zeros, 0 for just fraction ones */ +/* noclamp is 1 to unconditional (unclamped) trim */ +/* dropped returns the number of discarded trailing zeros */ +/* returns dn */ +/* */ +/* If clamp is set in the context then the number of zeros trimmed */ +/* may be limited if the exponent is high. */ +/* All fields are updated as required. This is a utility operation, */ +/* so special values are unchanged and no error is possible. */ +/* ------------------------------------------------------------------ */ +static decNumber * decTrim(decNumber *dn, decContext *set, Flag all, + Flag noclamp, Int *dropped) { + Int d, exp; /* work */ + uInt cut; /* .. */ + Unit *up; /* -> current Unit */ + + #if DECCHECK + if (decCheckOperands(dn, DECUNUSED, DECUNUSED, DECUNCONT)) return dn; + #endif + + *dropped=0; /* assume no zeros dropped */ + if ((dn->bits & DECSPECIAL) /* fast exit if special .. */ + || (*dn->lsu & 0x01)) return dn; /* .. or odd */ + if (ISZERO(dn)) { /* .. or 0 */ + dn->exponent=0; /* (sign is preserved) */ + return dn; + } + + /* have a finite number which is even */ + exp=dn->exponent; + cut=1; /* digit (1-DECDPUN) in Unit */ + up=dn->lsu; /* -> current Unit */ + for (d=0; ddigits-1; d++) { /* [don't strip the final digit] */ + /* slice by powers */ + #if DECDPUN<=4 + uInt quot=QUOT10(*up, cut); + if ((*up-quot*powers[cut])!=0) break; /* found non-0 digit */ + #else + if (*up%powers[cut]!=0) break; /* found non-0 digit */ + #endif + /* have a trailing 0 */ + if (!all) { /* trimming */ + /* [if exp>0 then all trailing 0s are significant for trim] */ + if (exp<=0) { /* if digit might be significant */ + if (exp==0) break; /* then quit */ + exp++; /* next digit might be significant */ + } + } + cut++; /* next power */ + if (cut>DECDPUN) { /* need new Unit */ + up++; + cut=1; + } + } /* d */ + if (d==0) return dn; /* none to drop */ + + /* may need to limit drop if clamping */ + if (set->clamp && !noclamp) { + Int maxd=set->emax-set->digits+1-dn->exponent; + if (maxd<=0) return dn; /* nothing possible */ + if (d>maxd) d=maxd; + } + + /* effect the drop */ + decShiftToLeast(dn->lsu, D2U(dn->digits), d); + dn->exponent+=d; /* maintain numerical value */ + dn->digits-=d; /* new length */ + *dropped=d; /* report the count */ + return dn; + } /* decTrim */ + +/* ------------------------------------------------------------------ */ +/* decReverse -- reverse a Unit array in place */ +/* */ +/* ulo is the start of the array */ +/* uhi is the end of the array (highest Unit to include) */ +/* */ +/* The units ulo through uhi are reversed in place (if the number */ +/* of units is odd, the middle one is untouched). Note that the */ +/* digit(s) in each unit are unaffected. */ +/* ------------------------------------------------------------------ */ +static void decReverse(Unit *ulo, Unit *uhi) { + Unit temp; + for (; ulo=uar; source--, target--) *target=*source; + } + else { + first=uar+D2U(digits+shift)-1; /* where msu of source will end up */ + for (; source>=uar; source--, target--) { + /* split the source Unit and accumulate remainder for next */ + #if DECDPUN<=4 + uInt quot=QUOT10(*source, cut); + uInt rem=*source-quot*powers[cut]; + next+=quot; + #else + uInt rem=*source%powers[cut]; + next+=*source/powers[cut]; + #endif + if (target<=first) *target=(Unit)next; /* write to target iff valid */ + next=rem*powers[DECDPUN-cut]; /* save remainder for next Unit */ + } + } /* shift-move */ + + /* propagate any partial unit to one below and clear the rest */ + for (; target>=uar; target--) { + *target=(Unit)next; + next=0; + } + return digits+shift; + } /* decShiftToMost */ + +/* ------------------------------------------------------------------ */ +/* decShiftToLeast -- shift digits in array towards least significant */ +/* */ +/* uar is the array */ +/* units is length of the array, in units */ +/* shift is the number of digits to remove from the lsu end; it */ +/* must be zero or positive and <= than units*DECDPUN. */ +/* */ +/* returns the new length of the integer in the array, in units */ +/* */ +/* Removed digits are discarded (lost). Units not required to hold */ +/* the final result are unchanged. */ +/* ------------------------------------------------------------------ */ +static Int decShiftToLeast(Unit *uar, Int units, Int shift) { + Unit *target, *up; /* work */ + Int cut, count; /* work */ + Int quot, rem; /* for division */ + + if (shift==0) return units; /* [fastpath] nothing to do */ + if (shift==units*DECDPUN) { /* [fastpath] little to do */ + *uar=0; /* all digits cleared gives zero */ + return 1; /* leaves just the one */ + } + + target=uar; /* both paths */ + cut=MSUDIGITS(shift); + if (cut==DECDPUN) { /* unit-boundary case; easy */ + up=uar+D2U(shift); + for (; up(target-uar); + } + + /* messier */ + up=uar+D2U(shift-cut); /* source; correct to whole Units */ + count=units*DECDPUN-shift; /* the maximum new length */ + #if DECDPUN<=4 + quot=QUOT10(*up, cut); + #else + quot=*up/powers[cut]; + #endif + for (; ; target++) { + *target=(Unit)quot; + count-=(DECDPUN-cut); + if (count<=0) break; + up++; + quot=*up; + #if DECDPUN<=4 + quot=QUOT10(quot, cut); + rem=*up-quot*powers[cut]; + #else + rem=quot%powers[cut]; + quot=quot/powers[cut]; + #endif + *target=(Unit)(*target+rem*powers[DECDPUN-cut]); + count-=cut; + if (count<=0) break; + } + return static_cast(target-uar+1); + } /* decShiftToLeast */ + +#if DECSUBSET +/* ------------------------------------------------------------------ */ +/* decRoundOperand -- round an operand [used for subset only] */ +/* */ +/* dn is the number to round (dn->digits is > set->digits) */ +/* set is the relevant context */ +/* status is the status accumulator */ +/* */ +/* returns an allocated decNumber with the rounded result. */ +/* */ +/* lostDigits and other status may be set by this. */ +/* */ +/* Since the input is an operand, it must not be modified. */ +/* Instead, return an allocated decNumber, rounded as required. */ +/* It is the caller's responsibility to free the allocated storage. */ +/* */ +/* If no storage is available then the result cannot be used, so nullptr */ +/* is returned. */ +/* ------------------------------------------------------------------ */ +static decNumber *decRoundOperand(const decNumber *dn, decContext *set, + uInt *status) { + decNumber *res; /* result structure */ + uInt newstatus=0; /* status from round */ + Int residue=0; /* rounding accumulator */ + + /* Allocate storage for the returned decNumber, big enough for the */ + /* length specified by the context */ + res=(decNumber *)malloc(sizeof(decNumber) + +(D2U(set->digits)-1)*sizeof(Unit)); + if (res==nullptr) { + *status|=DEC_Insufficient_storage; + return nullptr; + } + decCopyFit(res, dn, set, &residue, &newstatus); + decApplyRound(res, set, residue, &newstatus); + + /* If that set Inexact then "lost digits" is raised... */ + if (newstatus & DEC_Inexact) newstatus|=DEC_Lost_digits; + *status|=newstatus; + return res; + } /* decRoundOperand */ +#endif + +/* ------------------------------------------------------------------ */ +/* decCopyFit -- copy a number, truncating the coefficient if needed */ +/* */ +/* dest is the target decNumber */ +/* src is the source decNumber */ +/* set is the context [used for length (digits) and rounding mode] */ +/* residue is the residue accumulator */ +/* status contains the current status to be updated */ +/* */ +/* (dest==src is allowed and will be a no-op if fits) */ +/* All fields are updated as required. */ +/* ------------------------------------------------------------------ */ +static void decCopyFit(decNumber *dest, const decNumber *src, + decContext *set, Int *residue, uInt *status) { + dest->bits=src->bits; + dest->exponent=src->exponent; + decSetCoeff(dest, set, src->lsu, src->digits, residue, status); + } /* decCopyFit */ + +/* ------------------------------------------------------------------ */ +/* decSetCoeff -- set the coefficient of a number */ +/* */ +/* dn is the number whose coefficient array is to be set. */ +/* It must have space for set->digits digits */ +/* set is the context [for size] */ +/* lsu -> lsu of the source coefficient [may be dn->lsu] */ +/* len is digits in the source coefficient [may be dn->digits] */ +/* residue is the residue accumulator. This has values as in */ +/* decApplyRound, and will be unchanged unless the */ +/* target size is less than len. In this case, the */ +/* coefficient is truncated and the residue is updated to */ +/* reflect the previous residue and the dropped digits. */ +/* status is the status accumulator, as usual */ +/* */ +/* The coefficient may already be in the number, or it can be an */ +/* external intermediate array. If it is in the number, lsu must == */ +/* dn->lsu and len must == dn->digits. */ +/* */ +/* Note that the coefficient length (len) may be < set->digits, and */ +/* in this case this merely copies the coefficient (or is a no-op */ +/* if dn->lsu==lsu). */ +/* */ +/* Note also that (only internally, from decQuantizeOp and */ +/* decSetSubnormal) the value of set->digits may be less than one, */ +/* indicating a round to left. This routine handles that case */ +/* correctly; caller ensures space. */ +/* */ +/* dn->digits, dn->lsu (and as required), and dn->exponent are */ +/* updated as necessary. dn->bits (sign) is unchanged. */ +/* */ +/* DEC_Rounded status is set if any digits are discarded. */ +/* DEC_Inexact status is set if any non-zero digits are discarded, or */ +/* incoming residue was non-0 (implies rounded) */ +/* ------------------------------------------------------------------ */ +/* mapping array: maps 0-9 to canonical residues, so that a residue */ +/* can be adjusted in the range [-1, +1] and achieve correct rounding */ +/* 0 1 2 3 4 5 6 7 8 9 */ +static const uByte resmap[10]={0, 3, 3, 3, 3, 5, 7, 7, 7, 7}; +static void decSetCoeff(decNumber *dn, decContext *set, const Unit *lsu, + Int len, Int *residue, uInt *status) { + Int discard; /* number of digits to discard */ + uInt cut; /* cut point in Unit */ + const Unit *up; /* work */ + Unit *target; /* .. */ + Int count; /* .. */ + #if DECDPUN<=4 + uInt temp; /* .. */ + #endif + + discard=len-set->digits; /* digits to discard */ + if (discard<=0) { /* no digits are being discarded */ + if (dn->lsu!=lsu) { /* copy needed */ + /* copy the coefficient array to the result number; no shift needed */ + count=len; /* avoids D2U */ + up=lsu; + for (target=dn->lsu; count>0; target++, up++, count-=DECDPUN) + *target=*up; + dn->digits=len; /* set the new length */ + } + /* dn->exponent and residue are unchanged, record any inexactitude */ + if (*residue!=0) *status|=(DEC_Inexact | DEC_Rounded); + return; + } + + /* some digits must be discarded ... */ + dn->exponent+=discard; /* maintain numerical value */ + *status|=DEC_Rounded; /* accumulate Rounded status */ + if (*residue>1) *residue=1; /* previous residue now to right, so reduce */ + + if (discard>len) { /* everything, +1, is being discarded */ + /* guard digit is 0 */ + /* residue is all the number [NB could be all 0s] */ + if (*residue<=0) { /* not already positive */ + count=len; /* avoids D2U */ + for (up=lsu; count>0; up++, count-=DECDPUN) if (*up!=0) { /* found non-0 */ + *residue=1; + break; /* no need to check any others */ + } + } + if (*residue!=0) *status|=DEC_Inexact; /* record inexactitude */ + *dn->lsu=0; /* coefficient will now be 0 */ + dn->digits=1; /* .. */ + return; + } /* total discard */ + + /* partial discard [most common case] */ + /* here, at least the first (most significant) discarded digit exists */ + + /* spin up the number, noting residue during the spin, until get to */ + /* the Unit with the first discarded digit. When reach it, extract */ + /* it and remember its position */ + count=0; + for (up=lsu;; up++) { + count+=DECDPUN; + if (count>=discard) break; /* full ones all checked */ + if (*up!=0) *residue=1; + } /* up */ + + /* here up -> Unit with first discarded digit */ + cut=discard-(count-DECDPUN)-1; + if (cut==DECDPUN-1) { /* unit-boundary case (fast) */ + Unit half=(Unit)powers[DECDPUN]>>1; + /* set residue directly */ + if (*up>=half) { + if (*up>half) *residue=7; + else *residue+=5; /* add sticky bit */ + } + else { /* digits<=0) { /* special for Quantize/Subnormal :-( */ + *dn->lsu=0; /* .. result is 0 */ + dn->digits=1; /* .. */ + } + else { /* shift to least */ + count=set->digits; /* now digits to end up with */ + dn->digits=count; /* set the new length */ + up++; /* move to next */ + /* on unit boundary, so shift-down copy loop is simple */ + for (target=dn->lsu; count>0; target++, up++, count-=DECDPUN) + *target=*up; + } + } /* unit-boundary case */ + + else { /* discard digit is in low digit(s), and not top digit */ + uInt discard1; /* first discarded digit */ + uInt quot, rem; /* for divisions */ + if (cut==0) quot=*up; /* is at bottom of unit */ + else /* cut>0 */ { /* it's not at bottom of unit */ + #if DECDPUN<=4 + U_ASSERT(/* cut >= 0 &&*/ cut <= 4); + quot=QUOT10(*up, cut); + rem=*up-quot*powers[cut]; + #else + rem=*up%powers[cut]; + quot=*up/powers[cut]; + #endif + if (rem!=0) *residue=1; + } + /* discard digit is now at bottom of quot */ + #if DECDPUN<=4 + temp=(quot*6554)>>16; /* fast /10 */ + /* Vowels algorithm here not a win (9 instructions) */ + discard1=quot-X10(temp); + quot=temp; + #else + discard1=quot%10; + quot=quot/10; + #endif + /* here, discard1 is the guard digit, and residue is everything */ + /* else [use mapping array to accumulate residue safely] */ + *residue+=resmap[discard1]; + cut++; /* update cut */ + /* here: up -> Unit of the array with bottom digit */ + /* cut is the division point for each Unit */ + /* quot holds the uncut high-order digits for the current unit */ + if (set->digits<=0) { /* special for Quantize/Subnormal :-( */ + *dn->lsu=0; /* .. result is 0 */ + dn->digits=1; /* .. */ + } + else { /* shift to least needed */ + count=set->digits; /* now digits to end up with */ + dn->digits=count; /* set the new length */ + /* shift-copy the coefficient array to the result number */ + for (target=dn->lsu; ; target++) { + *target=(Unit)quot; + count-=(DECDPUN-cut); + if (count<=0) break; + up++; + quot=*up; + #if DECDPUN<=4 + quot=QUOT10(quot, cut); + rem=*up-quot*powers[cut]; + #else + rem=quot%powers[cut]; + quot=quot/powers[cut]; + #endif + *target=(Unit)(*target+rem*powers[DECDPUN-cut]); + count-=cut; + if (count<=0) break; + } /* shift-copy loop */ + } /* shift to least */ + } /* not unit boundary */ + + if (*residue!=0) *status|=DEC_Inexact; /* record inexactitude */ + return; + } /* decSetCoeff */ + +/* ------------------------------------------------------------------ */ +/* decApplyRound -- apply pending rounding to a number */ +/* */ +/* dn is the number, with space for set->digits digits */ +/* set is the context [for size and rounding mode] */ +/* residue indicates pending rounding, being any accumulated */ +/* guard and sticky information. It may be: */ +/* 6-9: rounding digit is >5 */ +/* 5: rounding digit is exactly half-way */ +/* 1-4: rounding digit is <5 and >0 */ +/* 0: the coefficient is exact */ +/* -1: as 1, but the hidden digits are subtractive, that */ +/* is, of the opposite sign to dn. In this case the */ +/* coefficient must be non-0. This case occurs when */ +/* subtracting a small number (which can be reduced to */ +/* a sticky bit); see decAddOp. */ +/* status is the status accumulator, as usual */ +/* */ +/* This routine applies rounding while keeping the length of the */ +/* coefficient constant. The exponent and status are unchanged */ +/* except if: */ +/* */ +/* -- the coefficient was increased and is all nines (in which */ +/* case Overflow could occur, and is handled directly here so */ +/* the caller does not need to re-test for overflow) */ +/* */ +/* -- the coefficient was decreased and becomes all nines (in which */ +/* case Underflow could occur, and is also handled directly). */ +/* */ +/* All fields in dn are updated as required. */ +/* */ +/* ------------------------------------------------------------------ */ +static void decApplyRound(decNumber *dn, decContext *set, Int residue, + uInt *status) { + Int bump; /* 1 if coefficient needs to be incremented */ + /* -1 if coefficient needs to be decremented */ + + if (residue==0) return; /* nothing to apply */ + + bump=0; /* assume a smooth ride */ + + /* now decide whether, and how, to round, depending on mode */ + switch (set->round) { + case DEC_ROUND_05UP: { /* round zero or five up (for reround) */ + /* This is the same as DEC_ROUND_DOWN unless there is a */ + /* positive residue and the lsd of dn is 0 or 5, in which case */ + /* it is bumped; when residue is <0, the number is therefore */ + /* bumped down unless the final digit was 1 or 6 (in which */ + /* case it is bumped down and then up -- a no-op) */ + Int lsd5=*dn->lsu%5; /* get lsd and quintate */ + if (residue<0 && lsd5!=1) bump=-1; + else if (residue>0 && lsd5==0) bump=1; + /* [bump==1 could be applied directly; use common path for clarity] */ + break;} /* r-05 */ + + case DEC_ROUND_DOWN: { + /* no change, except if negative residue */ + if (residue<0) bump=-1; + break;} /* r-d */ + + case DEC_ROUND_HALF_DOWN: { + if (residue>5) bump=1; + break;} /* r-h-d */ + + case DEC_ROUND_HALF_EVEN: { + if (residue>5) bump=1; /* >0.5 goes up */ + else if (residue==5) { /* exactly 0.5000... */ + /* 0.5 goes up iff [new] lsd is odd */ + if (*dn->lsu & 0x01) bump=1; + } + break;} /* r-h-e */ + + case DEC_ROUND_HALF_UP: { + if (residue>=5) bump=1; + break;} /* r-h-u */ + + case DEC_ROUND_UP: { + if (residue>0) bump=1; + break;} /* r-u */ + + case DEC_ROUND_CEILING: { + /* same as _UP for positive numbers, and as _DOWN for negatives */ + /* [negative residue cannot occur on 0] */ + if (decNumberIsNegative(dn)) { + if (residue<0) bump=-1; + } + else { + if (residue>0) bump=1; + } + break;} /* r-c */ + + case DEC_ROUND_FLOOR: { + /* same as _UP for negative numbers, and as _DOWN for positive */ + /* [negative residue cannot occur on 0] */ + if (!decNumberIsNegative(dn)) { + if (residue<0) bump=-1; + } + else { + if (residue>0) bump=1; + } + break;} /* r-f */ + + default: { /* e.g., DEC_ROUND_MAX */ + *status|=DEC_Invalid_context; + #if DECTRACE || (DECCHECK && DECVERB) + printf("Unknown rounding mode: %d\n", set->round); + #endif + break;} + } /* switch */ + + /* now bump the number, up or down, if need be */ + if (bump==0) return; /* no action required */ + + /* Simply use decUnitAddSub unless bumping up and the number is */ + /* all nines. In this special case set to 100... explicitly */ + /* and adjust the exponent by one (as otherwise could overflow */ + /* the array) */ + /* Similarly handle all-nines result if bumping down. */ + if (bump>0) { + Unit *up; /* work */ + uInt count=dn->digits; /* digits to be checked */ + for (up=dn->lsu; ; up++) { + if (count<=DECDPUN) { + /* this is the last Unit (the msu) */ + if (*up!=powers[count]-1) break; /* not still 9s */ + /* here if it, too, is all nines */ + *up=(Unit)powers[count-1]; /* here 999 -> 100 etc. */ + for (up=up-1; up>=dn->lsu; up--) *up=0; /* others all to 0 */ + dn->exponent++; /* and bump exponent */ + /* [which, very rarely, could cause Overflow...] */ + if ((dn->exponent+dn->digits)>set->emax+1) { + decSetOverflow(dn, set, status); + } + return; /* done */ + } + /* a full unit to check, with more to come */ + if (*up!=DECDPUNMAX) break; /* not still 9s */ + count-=DECDPUN; + } /* up */ + } /* bump>0 */ + else { /* -1 */ + /* here checking for a pre-bump of 1000... (leading 1, all */ + /* other digits zero) */ + Unit *up, *sup; /* work */ + uInt count=dn->digits; /* digits to be checked */ + for (up=dn->lsu; ; up++) { + if (count<=DECDPUN) { + /* this is the last Unit (the msu) */ + if (*up!=powers[count-1]) break; /* not 100.. */ + /* here if have the 1000... case */ + sup=up; /* save msu pointer */ + *up=(Unit)powers[count]-1; /* here 100 in msu -> 999 */ + /* others all to all-nines, too */ + for (up=up-1; up>=dn->lsu; up--) *up=(Unit)powers[DECDPUN]-1; + dn->exponent--; /* and bump exponent */ + + /* iff the number was at the subnormal boundary (exponent=etiny) */ + /* then the exponent is now out of range, so it will in fact get */ + /* clamped to etiny and the final 9 dropped. */ + /* printf(">> emin=%d exp=%d sdig=%d\n", set->emin, */ + /* dn->exponent, set->digits); */ + if (dn->exponent+1==set->emin-set->digits+1) { + if (count==1 && dn->digits==1) *sup=0; /* here 9 -> 0[.9] */ + else { + *sup=(Unit)powers[count-1]-1; /* here 999.. in msu -> 99.. */ + dn->digits--; + } + dn->exponent++; + *status|=DEC_Underflow | DEC_Subnormal | DEC_Inexact | DEC_Rounded; + } + return; /* done */ + } + + /* a full unit to check, with more to come */ + if (*up!=0) break; /* not still 0s */ + count-=DECDPUN; + } /* up */ + + } /* bump<0 */ + + /* Actual bump needed. Do it. */ + decUnitAddSub(dn->lsu, D2U(dn->digits), uarrone, 1, 0, dn->lsu, bump); + } /* decApplyRound */ + +#if DECSUBSET +/* ------------------------------------------------------------------ */ +/* decFinish -- finish processing a number */ +/* */ +/* dn is the number */ +/* set is the context */ +/* residue is the rounding accumulator (as in decApplyRound) */ +/* status is the accumulator */ +/* */ +/* This finishes off the current number by: */ +/* 1. If not extended: */ +/* a. Converting a zero result to clean '0' */ +/* b. Reducing positive exponents to 0, if would fit in digits */ +/* 2. Checking for overflow and subnormals (always) */ +/* Note this is just Finalize when no subset arithmetic. */ +/* All fields are updated as required. */ +/* ------------------------------------------------------------------ */ +static void decFinish(decNumber *dn, decContext *set, Int *residue, + uInt *status) { + if (!set->extended) { + if ISZERO(dn) { /* value is zero */ + dn->exponent=0; /* clean exponent .. */ + dn->bits=0; /* .. and sign */ + return; /* no error possible */ + } + if (dn->exponent>=0) { /* non-negative exponent */ + /* >0; reduce to integer if possible */ + if (set->digits >= (dn->exponent+dn->digits)) { + dn->digits=decShiftToMost(dn->lsu, dn->digits, dn->exponent); + dn->exponent=0; + } + } + } /* !extended */ + + decFinalize(dn, set, residue, status); + } /* decFinish */ +#endif + +/* ------------------------------------------------------------------ */ +/* decFinalize -- final check, clamp, and round of a number */ +/* */ +/* dn is the number */ +/* set is the context */ +/* residue is the rounding accumulator (as in decApplyRound) */ +/* status is the status accumulator */ +/* */ +/* This finishes off the current number by checking for subnormal */ +/* results, applying any pending rounding, checking for overflow, */ +/* and applying any clamping. */ +/* Underflow and overflow conditions are raised as appropriate. */ +/* All fields are updated as required. */ +/* ------------------------------------------------------------------ */ +static void decFinalize(decNumber *dn, decContext *set, Int *residue, + uInt *status) { + Int shift; /* shift needed if clamping */ + Int tinyexp=set->emin-dn->digits+1; /* precalculate subnormal boundary */ + + /* Must be careful, here, when checking the exponent as the */ + /* adjusted exponent could overflow 31 bits [because it may already */ + /* be up to twice the expected]. */ + + /* First test for subnormal. This must be done before any final */ + /* round as the result could be rounded to Nmin or 0. */ + if (dn->exponent<=tinyexp) { /* prefilter */ + Int comp; + decNumber nmin; + /* A very nasty case here is dn == Nmin and residue<0 */ + if (dn->exponentemin; + comp=decCompare(dn, &nmin, 1); /* (signless compare) */ + if (comp==BADINT) { /* oops */ + *status|=DEC_Insufficient_storage; /* abandon... */ + return; + } + if (*residue<0 && comp==0) { /* neg residue and dn==Nmin */ + decApplyRound(dn, set, *residue, status); /* might force down */ + decSetSubnormal(dn, set, residue, status); + return; + } + } + + /* now apply any pending round (this could raise overflow). */ + if (*residue!=0) decApplyRound(dn, set, *residue, status); + + /* Check for overflow [redundant in the 'rare' case] or clamp */ + if (dn->exponent<=set->emax-set->digits+1) return; /* neither needed */ + + + /* here when might have an overflow or clamp to do */ + if (dn->exponent>set->emax-dn->digits+1) { /* too big */ + decSetOverflow(dn, set, status); + return; + } + /* here when the result is normal but in clamp range */ + if (!set->clamp) return; + + /* here when need to apply the IEEE exponent clamp (fold-down) */ + shift=dn->exponent-(set->emax-set->digits+1); + + /* shift coefficient (if non-zero) */ + if (!ISZERO(dn)) { + dn->digits=decShiftToMost(dn->lsu, dn->digits, shift); + } + dn->exponent-=shift; /* adjust the exponent to match */ + *status|=DEC_Clamped; /* and record the dirty deed */ + return; + } /* decFinalize */ + +/* ------------------------------------------------------------------ */ +/* decSetOverflow -- set number to proper overflow value */ +/* */ +/* dn is the number (used for sign [only] and result) */ +/* set is the context [used for the rounding mode, etc.] */ +/* status contains the current status to be updated */ +/* */ +/* This sets the sign of a number and sets its value to either */ +/* Infinity or the maximum finite value, depending on the sign of */ +/* dn and the rounding mode, following IEEE 754 rules. */ +/* ------------------------------------------------------------------ */ +static void decSetOverflow(decNumber *dn, decContext *set, uInt *status) { + Flag needmax=0; /* result is maximum finite value */ + uByte sign=dn->bits&DECNEG; /* clean and save sign bit */ + + if (ISZERO(dn)) { /* zero does not overflow magnitude */ + Int emax=set->emax; /* limit value */ + if (set->clamp) emax-=set->digits-1; /* lower if clamping */ + if (dn->exponent>emax) { /* clamp required */ + dn->exponent=emax; + *status|=DEC_Clamped; + } + return; + } + + uprv_decNumberZero(dn); + switch (set->round) { + case DEC_ROUND_DOWN: { + needmax=1; /* never Infinity */ + break;} /* r-d */ + case DEC_ROUND_05UP: { + needmax=1; /* never Infinity */ + break;} /* r-05 */ + case DEC_ROUND_CEILING: { + if (sign) needmax=1; /* Infinity if non-negative */ + break;} /* r-c */ + case DEC_ROUND_FLOOR: { + if (!sign) needmax=1; /* Infinity if negative */ + break;} /* r-f */ + default: break; /* Infinity in all other cases */ + } + if (needmax) { + decSetMaxValue(dn, set); + dn->bits=sign; /* set sign */ + } + else dn->bits=sign|DECINF; /* Value is +/-Infinity */ + *status|=DEC_Overflow | DEC_Inexact | DEC_Rounded; + } /* decSetOverflow */ + +/* ------------------------------------------------------------------ */ +/* decSetMaxValue -- set number to +Nmax (maximum normal value) */ +/* */ +/* dn is the number to set */ +/* set is the context [used for digits and emax] */ +/* */ +/* This sets the number to the maximum positive value. */ +/* ------------------------------------------------------------------ */ +static void decSetMaxValue(decNumber *dn, decContext *set) { + Unit *up; /* work */ + Int count=set->digits; /* nines to add */ + dn->digits=count; + /* fill in all nines to set maximum value */ + for (up=dn->lsu; ; up++) { + if (count>DECDPUN) *up=DECDPUNMAX; /* unit full o'nines */ + else { /* this is the msu */ + *up=(Unit)(powers[count]-1); + break; + } + count-=DECDPUN; /* filled those digits */ + } /* up */ + dn->bits=0; /* + sign */ + dn->exponent=set->emax-set->digits+1; + } /* decSetMaxValue */ + +/* ------------------------------------------------------------------ */ +/* decSetSubnormal -- process value whose exponent is extended) { + uprv_decNumberZero(dn); + /* always full overflow */ + *status|=DEC_Underflow | DEC_Subnormal | DEC_Inexact | DEC_Rounded; + return; + } + #endif + + /* Full arithmetic -- allow subnormals, rounded to minimum exponent */ + /* (Etiny) if needed */ + etiny=set->emin-(set->digits-1); /* smallest allowed exponent */ + + if ISZERO(dn) { /* value is zero */ + /* residue can never be non-zero here */ + #if DECCHECK + if (*residue!=0) { + printf("++ Subnormal 0 residue %ld\n", (LI)*residue); + *status|=DEC_Invalid_operation; + } + #endif + if (dn->exponentexponent=etiny; + *status|=DEC_Clamped; + } + return; + } + + *status|=DEC_Subnormal; /* have a non-zero subnormal */ + adjust=etiny-dn->exponent; /* calculate digits to remove */ + if (adjust<=0) { /* not out of range; unrounded */ + /* residue can never be non-zero here, except in the Nmin-residue */ + /* case (which is a subnormal result), so can take fast-path here */ + /* it may already be inexact (from setting the coefficient) */ + if (*status&DEC_Inexact) *status|=DEC_Underflow; + return; + } + + /* adjust>0, so need to rescale the result so exponent becomes Etiny */ + /* [this code is similar to that in rescale] */ + workset=*set; /* clone rounding, etc. */ + workset.digits=dn->digits-adjust; /* set requested length */ + workset.emin-=adjust; /* and adjust emin to match */ + /* [note that the latter can be <1, here, similar to Rescale case] */ + decSetCoeff(dn, &workset, dn->lsu, dn->digits, residue, status); + decApplyRound(dn, &workset, *residue, status); + + /* Use 754 default rule: Underflow is set iff Inexact */ + /* [independent of whether trapped] */ + if (*status&DEC_Inexact) *status|=DEC_Underflow; + + /* if rounded up a 999s case, exponent will be off by one; adjust */ + /* back if so [it will fit, because it was shortened earlier] */ + if (dn->exponent>etiny) { + dn->digits=decShiftToMost(dn->lsu, dn->digits, 1); + dn->exponent--; /* (re)adjust the exponent. */ + } + + /* if rounded to zero, it is by definition clamped... */ + if (ISZERO(dn)) *status|=DEC_Clamped; + } /* decSetSubnormal */ + +/* ------------------------------------------------------------------ */ +/* decCheckMath - check entry conditions for a math function */ +/* */ +/* This checks the context and the operand */ +/* */ +/* rhs is the operand to check */ +/* set is the context to check */ +/* status is unchanged if both are good */ +/* */ +/* returns non-zero if status is changed, 0 otherwise */ +/* */ +/* Restrictions enforced: */ +/* */ +/* digits, emax, and -emin in the context must be less than */ +/* DEC_MAX_MATH (999999), and A must be within these bounds if */ +/* non-zero. Invalid_operation is set in the status if a */ +/* restriction is violated. */ +/* ------------------------------------------------------------------ */ +static uInt decCheckMath(const decNumber *rhs, decContext *set, + uInt *status) { + uInt save=*status; /* record */ + if (set->digits>DEC_MAX_MATH + || set->emax>DEC_MAX_MATH + || -set->emin>DEC_MAX_MATH) *status|=DEC_Invalid_context; + else if ((rhs->digits>DEC_MAX_MATH + || rhs->exponent+rhs->digits>DEC_MAX_MATH+1 + || rhs->exponent+rhs->digits<2*(1-DEC_MAX_MATH)) + && !ISZERO(rhs)) *status|=DEC_Invalid_operation; + return (*status!=save); + } /* decCheckMath */ + +/* ------------------------------------------------------------------ */ +/* decGetInt -- get integer from a number */ +/* */ +/* dn is the number [which will not be altered] */ +/* */ +/* returns one of: */ +/* BADINT if there is a non-zero fraction */ +/* the converted integer */ +/* BIGEVEN if the integer is even and magnitude > 2*10**9 */ +/* BIGODD if the integer is odd and magnitude > 2*10**9 */ +/* */ +/* This checks and gets a whole number from the input decNumber. */ +/* The sign can be determined from dn by the caller when BIGEVEN or */ +/* BIGODD is returned. */ +/* ------------------------------------------------------------------ */ +static Int decGetInt(const decNumber *dn) { + Int theInt; /* result accumulator */ + const Unit *up; /* work */ + Int got; /* digits (real or not) processed */ + Int ilength=dn->digits+dn->exponent; /* integral length */ + Flag neg=decNumberIsNegative(dn); /* 1 if -ve */ + + /* The number must be an integer that fits in 10 digits */ + /* Assert, here, that 10 is enough for any rescale Etiny */ + #if DEC_MAX_EMAX > 999999999 + #error GetInt may need updating [for Emax] + #endif + #if DEC_MIN_EMIN < -999999999 + #error GetInt may need updating [for Emin] + #endif + if (ISZERO(dn)) return 0; /* zeros are OK, with any exponent */ + + up=dn->lsu; /* ready for lsu */ + theInt=0; /* ready to accumulate */ + if (dn->exponent>=0) { /* relatively easy */ + /* no fractional part [usual]; allow for positive exponent */ + got=dn->exponent; + } + else { /* -ve exponent; some fractional part to check and discard */ + Int count=-dn->exponent; /* digits to discard */ + /* spin up whole units until reach the Unit with the unit digit */ + for (; count>=DECDPUN; up++) { + if (*up!=0) return BADINT; /* non-zero Unit to discard */ + count-=DECDPUN; + } + if (count==0) got=0; /* [a multiple of DECDPUN] */ + else { /* [not multiple of DECDPUN] */ + Int rem; /* work */ + /* slice off fraction digits and check for non-zero */ + #if DECDPUN<=4 + theInt=QUOT10(*up, count); + rem=*up-theInt*powers[count]; + #else + rem=*up%powers[count]; /* slice off discards */ + theInt=*up/powers[count]; + #endif + if (rem!=0) return BADINT; /* non-zero fraction */ + /* it looks good */ + got=DECDPUN-count; /* number of digits so far */ + up++; /* ready for next */ + } + } + /* now it's known there's no fractional part */ + + /* tricky code now, to accumulate up to 9.3 digits */ + if (got==0) {theInt=*up; got+=DECDPUN; up++;} /* ensure lsu is there */ + + if (ilength<11) { + Int save=theInt; + /* collect any remaining unit(s) */ + for (; got1999999997) ilength=11; + else if (!neg && theInt>999999999) ilength=11; + if (ilength==11) theInt=save; /* restore correct low bit */ + } + } + + if (ilength>10) { /* too big */ + if (theInt&1) return BIGODD; /* bottom bit 1 */ + return BIGEVEN; /* bottom bit 0 */ + } + + if (neg) theInt=-theInt; /* apply sign */ + return theInt; + } /* decGetInt */ + +/* ------------------------------------------------------------------ */ +/* decDecap -- decapitate the coefficient of a number */ +/* */ +/* dn is the number to be decapitated */ +/* drop is the number of digits to be removed from the left of dn; */ +/* this must be <= dn->digits (if equal, the coefficient is */ +/* set to 0) */ +/* */ +/* Returns dn; dn->digits will be <= the initial digits less drop */ +/* (after removing drop digits there may be leading zero digits */ +/* which will also be removed). Only dn->lsu and dn->digits change. */ +/* ------------------------------------------------------------------ */ +static decNumber *decDecap(decNumber *dn, Int drop) { + Unit *msu; /* -> target cut point */ + Int cut; /* work */ + if (drop>=dn->digits) { /* losing the whole thing */ + #if DECCHECK + if (drop>dn->digits) + printf("decDecap called with drop>digits [%ld>%ld]\n", + (LI)drop, (LI)dn->digits); + #endif + dn->lsu[0]=0; + dn->digits=1; + return dn; + } + msu=dn->lsu+D2U(dn->digits-drop)-1; /* -> likely msu */ + cut=MSUDIGITS(dn->digits-drop); /* digits to be in use in msu */ + if (cut!=DECDPUN) *msu%=powers[cut]; /* clear left digits */ + /* that may have left leading zero digits, so do a proper count... */ + dn->digits=decGetDigits(dn->lsu, static_cast(msu-dn->lsu+1)); + return dn; + } /* decDecap */ + +/* ------------------------------------------------------------------ */ +/* decBiStr -- compare string with pairwise options */ +/* */ +/* targ is the string to compare */ +/* str1 is one of the strings to compare against (length may be 0) */ +/* str2 is the other; it must be the same length as str1 */ +/* */ +/* returns 1 if strings compare equal, (that is, it is the same */ +/* length as str1 and str2, and each character of targ is in either */ +/* str1 or str2 in the corresponding position), or 0 otherwise */ +/* */ +/* This is used for generic caseless compare, including the awkward */ +/* case of the Turkish dotted and dotless Is. Use as (for example): */ +/* if (decBiStr(test, "mike", "MIKE")) ... */ +/* ------------------------------------------------------------------ */ +static Flag decBiStr(const char *targ, const char *str1, const char *str2) { + for (;;targ++, str1++, str2++) { + if (*targ!=*str1 && *targ!=*str2) return 0; + /* *targ has a match in one (or both, if terminator) */ + if (*targ=='\0') break; + } /* forever */ + return 1; + } /* decBiStr */ + +/* ------------------------------------------------------------------ */ +/* decNaNs -- handle NaN operand or operands */ +/* */ +/* res is the result number */ +/* lhs is the first operand */ +/* rhs is the second operand, or nullptr if none */ +/* context is used to limit payload length */ +/* status contains the current status */ +/* returns res in case convenient */ +/* */ +/* Called when one or both operands is a NaN, and propagates the */ +/* appropriate result to res. When an sNaN is found, it is changed */ +/* to a qNaN and Invalid operation is set. */ +/* ------------------------------------------------------------------ */ +static decNumber * decNaNs(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set, + uInt *status) { + /* This decision tree ends up with LHS being the source pointer, */ + /* and status updated if need be */ + if (lhs->bits & DECSNAN) + *status|=DEC_Invalid_operation | DEC_sNaN; + else if (rhs==nullptr); + else if (rhs->bits & DECSNAN) { + lhs=rhs; + *status|=DEC_Invalid_operation | DEC_sNaN; + } + else if (lhs->bits & DECNAN); + else lhs=rhs; + + /* propagate the payload */ + if (lhs->digits<=set->digits) uprv_decNumberCopy(res, lhs); /* easy */ + else { /* too long */ + const Unit *ul; + Unit *ur, *uresp1; + /* copy safe number of units, then decapitate */ + res->bits=lhs->bits; /* need sign etc. */ + uresp1=res->lsu+D2U(set->digits); + for (ur=res->lsu, ul=lhs->lsu; urdigits=D2U(set->digits)*DECDPUN; + /* maybe still too long */ + if (res->digits>set->digits) decDecap(res, res->digits-set->digits); + } + + res->bits&=~DECSNAN; /* convert any sNaN to NaN, while */ + res->bits|=DECNAN; /* .. preserving sign */ + res->exponent=0; /* clean exponent */ + /* [coefficient was copied/decapitated] */ + return res; + } /* decNaNs */ + +/* ------------------------------------------------------------------ */ +/* decStatus -- apply non-zero status */ +/* */ +/* dn is the number to set if error */ +/* status contains the current status (not yet in context) */ +/* set is the context */ +/* */ +/* If the status is an error status, the number is set to a NaN, */ +/* unless the error was an overflow, divide-by-zero, or underflow, */ +/* in which case the number will have already been set. */ +/* */ +/* The context status is then updated with the new status. Note that */ +/* this may raise a signal, so control may never return from this */ +/* routine (hence resources must be recovered before it is called). */ +/* ------------------------------------------------------------------ */ +static void decStatus(decNumber *dn, uInt status, decContext *set) { + if (status & DEC_NaNs) { /* error status -> NaN */ + /* if cause was an sNaN, clear and propagate [NaN is already set up] */ + if (status & DEC_sNaN) status&=~DEC_sNaN; + else { + uprv_decNumberZero(dn); /* other error: clean throughout */ + dn->bits=DECNAN; /* and make a quiet NaN */ + } + } + uprv_decContextSetStatus(set, status); /* [may not return] */ + return; + } /* decStatus */ + +/* ------------------------------------------------------------------ */ +/* decGetDigits -- count digits in a Units array */ +/* */ +/* uar is the Unit array holding the number (this is often an */ +/* accumulator of some sort) */ +/* len is the length of the array in units [>=1] */ +/* */ +/* returns the number of (significant) digits in the array */ +/* */ +/* All leading zeros are excluded, except the last if the array has */ +/* only zero Units. */ +/* ------------------------------------------------------------------ */ +/* This may be called twice during some operations. */ +static Int decGetDigits(Unit *uar, Int len) { + Unit *up=uar+(len-1); /* -> msu */ + Int digits=(len-1)*DECDPUN+1; /* possible digits excluding msu */ + #if DECDPUN>4 + uInt const *pow; /* work */ + #endif + /* (at least 1 in final msu) */ + #if DECCHECK + if (len<1) printf("decGetDigits called with len<1 [%ld]\n", (LI)len); + #endif + + for (; up>=uar; up--) { + if (*up==0) { /* unit is all 0s */ + if (digits==1) break; /* a zero has one digit */ + digits-=DECDPUN; /* adjust for 0 unit */ + continue;} + /* found the first (most significant) non-zero Unit */ + #if DECDPUN>1 /* not done yet */ + if (*up<10) break; /* is 1-9 */ + digits++; + #if DECDPUN>2 /* not done yet */ + if (*up<100) break; /* is 10-99 */ + digits++; + #if DECDPUN>3 /* not done yet */ + if (*up<1000) break; /* is 100-999 */ + digits++; + #if DECDPUN>4 /* count the rest ... */ + for (pow=&powers[4]; *up>=*pow; pow++) digits++; + #endif + #endif + #endif + #endif + break; + } /* up */ + return digits; + } /* decGetDigits */ + +#if DECTRACE | DECCHECK +/* ------------------------------------------------------------------ */ +/* decNumberShow -- display a number [debug aid] */ +/* dn is the number to show */ +/* */ +/* Shows: sign, exponent, coefficient (msu first), digits */ +/* or: sign, special-value */ +/* ------------------------------------------------------------------ */ +/* this is public so other modules can use it */ +void uprv_decNumberShow(const decNumber *dn) { + const Unit *up; /* work */ + uInt u, d; /* .. */ + Int cut; /* .. */ + char isign='+'; /* main sign */ + if (dn==nullptr) { + printf("nullptr\n"); + return;} + if (decNumberIsNegative(dn)) isign='-'; + printf(" >> %c ", isign); + if (dn->bits&DECSPECIAL) { /* Is a special value */ + if (decNumberIsInfinite(dn)) printf("Infinity"); + else { /* a NaN */ + if (dn->bits&DECSNAN) printf("sNaN"); /* signalling NaN */ + else printf("NaN"); + } + /* if coefficient and exponent are 0, no more to do */ + if (dn->exponent==0 && dn->digits==1 && *dn->lsu==0) { + printf("\n"); + return;} + /* drop through to report other information */ + printf(" "); + } + + /* now carefully display the coefficient */ + up=dn->lsu+D2U(dn->digits)-1; /* msu */ + printf("%ld", (LI)*up); + for (up=up-1; up>=dn->lsu; up--) { + u=*up; + printf(":"); + for (cut=DECDPUN-1; cut>=0; cut--) { + d=u/powers[cut]; + u-=d*powers[cut]; + printf("%ld", (LI)d); + } /* cut */ + } /* up */ + if (dn->exponent!=0) { + char esign='+'; + if (dn->exponent<0) esign='-'; + printf(" E%c%ld", esign, (LI)abs(dn->exponent)); + } + printf(" [%ld]\n", (LI)dn->digits); + } /* decNumberShow */ +#endif + +#if DECTRACE || DECCHECK +/* ------------------------------------------------------------------ */ +/* decDumpAr -- display a unit array [debug/check aid] */ +/* name is a single-character tag name */ +/* ar is the array to display */ +/* len is the length of the array in Units */ +/* ------------------------------------------------------------------ */ +static void decDumpAr(char name, const Unit *ar, Int len) { + Int i; + const char *spec; + #if DECDPUN==9 + spec="%09d "; + #elif DECDPUN==8 + spec="%08d "; + #elif DECDPUN==7 + spec="%07d "; + #elif DECDPUN==6 + spec="%06d "; + #elif DECDPUN==5 + spec="%05d "; + #elif DECDPUN==4 + spec="%04d "; + #elif DECDPUN==3 + spec="%03d "; + #elif DECDPUN==2 + spec="%02d "; + #else + spec="%d "; + #endif + printf(" :%c: ", name); + for (i=len-1; i>=0; i--) { + if (i==len-1) printf("%ld ", (LI)ar[i]); + else printf(spec, ar[i]); + } + printf("\n"); + return;} +#endif + +#if DECCHECK +/* ------------------------------------------------------------------ */ +/* decCheckOperands -- check operand(s) to a routine */ +/* res is the result structure (not checked; it will be set to */ +/* quiet NaN if error found (and it is not nullptr)) */ +/* lhs is the first operand (may be DECUNRESU) */ +/* rhs is the second (may be DECUNUSED) */ +/* set is the context (may be DECUNCONT) */ +/* returns 0 if both operands, and the context are clean, or 1 */ +/* otherwise (in which case the context will show an error, */ +/* unless nullptr). Note that res is not cleaned; caller should */ +/* handle this so res=nullptr case is safe. */ +/* The caller is expected to abandon immediately if 1 is returned. */ +/* ------------------------------------------------------------------ */ +static Flag decCheckOperands(decNumber *res, const decNumber *lhs, + const decNumber *rhs, decContext *set) { + Flag bad=0; + if (set==nullptr) { /* oops; hopeless */ + #if DECTRACE || DECVERB + printf("Reference to context is nullptr.\n"); + #endif + bad=1; + return 1;} + else if (set!=DECUNCONT + && (set->digits<1 || set->round>=DEC_ROUND_MAX)) { + bad=1; + #if DECTRACE || DECVERB + printf("Bad context [digits=%ld round=%ld].\n", + (LI)set->digits, (LI)set->round); + #endif + } + else { + if (res==nullptr) { + bad=1; + #if DECTRACE + /* this one not DECVERB as standard tests include nullptr */ + printf("Reference to result is nullptr.\n"); + #endif + } + if (!bad && lhs!=DECUNUSED) bad=(decCheckNumber(lhs)); + if (!bad && rhs!=DECUNUSED) bad=(decCheckNumber(rhs)); + } + if (bad) { + if (set!=DECUNCONT) uprv_decContextSetStatus(set, DEC_Invalid_operation); + if (res!=DECUNRESU && res!=nullptr) { + uprv_decNumberZero(res); + res->bits=DECNAN; /* qNaN */ + } + } + return bad; + } /* decCheckOperands */ + +/* ------------------------------------------------------------------ */ +/* decCheckNumber -- check a number */ +/* dn is the number to check */ +/* returns 0 if the number is clean, or 1 otherwise */ +/* */ +/* The number is considered valid if it could be a result from some */ +/* operation in some valid context. */ +/* ------------------------------------------------------------------ */ +static Flag decCheckNumber(const decNumber *dn) { + const Unit *up; /* work */ + uInt maxuint; /* .. */ + Int ae, d, digits; /* .. */ + Int emin, emax; /* .. */ + + if (dn==nullptr) { /* hopeless */ + #if DECTRACE + /* this one not DECVERB as standard tests include nullptr */ + printf("Reference to decNumber is nullptr.\n"); + #endif + return 1;} + + /* check special values */ + if (dn->bits & DECSPECIAL) { + if (dn->exponent!=0) { + #if DECTRACE || DECVERB + printf("Exponent %ld (not 0) for a special value [%02x].\n", + (LI)dn->exponent, dn->bits); + #endif + return 1;} + + /* 2003.09.08: NaNs may now have coefficients, so next tests Inf only */ + if (decNumberIsInfinite(dn)) { + if (dn->digits!=1) { + #if DECTRACE || DECVERB + printf("Digits %ld (not 1) for an infinity.\n", (LI)dn->digits); + #endif + return 1;} + if (*dn->lsu!=0) { + #if DECTRACE || DECVERB + printf("LSU %ld (not 0) for an infinity.\n", (LI)*dn->lsu); + #endif + decDumpAr('I', dn->lsu, D2U(dn->digits)); + return 1;} + } /* Inf */ + /* 2002.12.26: negative NaNs can now appear through proposed IEEE */ + /* concrete formats (decimal64, etc.). */ + return 0; + } + + /* check the coefficient */ + if (dn->digits<1 || dn->digits>DECNUMMAXP) { + #if DECTRACE || DECVERB + printf("Digits %ld in number.\n", (LI)dn->digits); + #endif + return 1;} + + d=dn->digits; + + for (up=dn->lsu; d>0; up++) { + if (d>DECDPUN) maxuint=DECDPUNMAX; + else { /* reached the msu */ + maxuint=powers[d]-1; + if (dn->digits>1 && *upmaxuint) { + #if DECTRACE || DECVERB + printf("Bad Unit [%08lx] in %ld-digit number at offset %ld [maxuint %ld].\n", + (LI)*up, (LI)dn->digits, (LI)(up-dn->lsu), (LI)maxuint); + #endif + return 1;} + d-=DECDPUN; + } + + /* check the exponent. Note that input operands can have exponents */ + /* which are out of the set->emin/set->emax and set->digits range */ + /* (just as they can have more digits than set->digits). */ + ae=dn->exponent+dn->digits-1; /* adjusted exponent */ + emax=DECNUMMAXE; + emin=DECNUMMINE; + digits=DECNUMMAXP; + if (ae+emax) { + #if DECTRACE || DECVERB + printf("Adjusted exponent overflow [%ld].\n", (LI)ae); + uprv_decNumberShow(dn); + #endif + return 1;} + + return 0; /* it's OK */ + } /* decCheckNumber */ + +/* ------------------------------------------------------------------ */ +/* decCheckInexact -- check a normal finite inexact result has digits */ +/* dn is the number to check */ +/* set is the context (for status and precision) */ +/* sets Invalid operation, etc., if some digits are missing */ +/* [this check is not made for DECSUBSET compilation or when */ +/* subnormal is not set] */ +/* ------------------------------------------------------------------ */ +static void decCheckInexact(const decNumber *dn, decContext *set) { + #if !DECSUBSET && DECEXTFLAG + if ((set->status & (DEC_Inexact|DEC_Subnormal))==DEC_Inexact + && (set->digits!=dn->digits) && !(dn->bits & DECSPECIAL)) { + #if DECTRACE || DECVERB + printf("Insufficient digits [%ld] on normal Inexact result.\n", + (LI)dn->digits); + uprv_decNumberShow(dn); + #endif + uprv_decContextSetStatus(set, DEC_Invalid_operation); + } + #else + /* next is a noop for quiet compiler */ + if (dn!=nullptr && dn->digits==0) set->status|=DEC_Invalid_operation; + #endif + return; + } /* decCheckInexact */ +#endif + +#if DECALLOC +#undef malloc +#undef free +/* ------------------------------------------------------------------ */ +/* decMalloc -- accountable allocation routine */ +/* n is the number of bytes to allocate */ +/* */ +/* Semantics is the same as the stdlib malloc routine, but bytes */ +/* allocated are accounted for globally, and corruption fences are */ +/* added before and after the 'actual' storage. */ +/* ------------------------------------------------------------------ */ +/* This routine allocates storage with an extra twelve bytes; 8 are */ +/* at the start and hold: */ +/* 0-3 the original length requested */ +/* 4-7 buffer corruption detection fence (DECFENCE, x4) */ +/* The 4 bytes at the end also hold a corruption fence (DECFENCE, x4) */ +/* ------------------------------------------------------------------ */ +static void *decMalloc(size_t n) { + uInt size=n+12; /* true size */ + void *alloc; /* -> allocated storage */ + uByte *b, *b0; /* work */ + uInt uiwork; /* for macros */ + + alloc=malloc(size); /* -> allocated storage */ + if (alloc==nullptr) return nullptr; /* out of strorage */ + b0=(uByte *)alloc; /* as bytes */ + decAllocBytes+=n; /* account for storage */ + UBFROMUI(alloc, n); /* save n */ + /* printf(" alloc ++ dAB: %ld (%ld)\n", (LI)decAllocBytes, (LI)n); */ + for (b=b0+4; b play area */ + } /* decMalloc */ + +/* ------------------------------------------------------------------ */ +/* decFree -- accountable free routine */ +/* alloc is the storage to free */ +/* */ +/* Semantics is the same as the stdlib malloc routine, except that */ +/* the global storage accounting is updated and the fences are */ +/* checked to ensure that no routine has written 'out of bounds'. */ +/* ------------------------------------------------------------------ */ +/* This routine first checks that the fences have not been corrupted. */ +/* It then frees the storage using the 'truw' storage address (that */ +/* is, offset by 8). */ +/* ------------------------------------------------------------------ */ +static void decFree(void *alloc) { + uInt n; /* original length */ + uByte *b, *b0; /* work */ + uInt uiwork; /* for macros */ + + if (alloc==nullptr) return; /* allowed; it's a nop */ + b0=(uByte *)alloc; /* as bytes */ + b0-=8; /* -> true start of storage */ + n=UBTOUI(b0); /* lift length */ + for (b=b0+4; b0 */ + /* and <10; 3 or powers of 2 are best]. */ + + /* DECNUMDIGITS is the default number of digits that can be held in */ + /* the structure. If undefined, 1 is assumed and it is assumed */ + /* that the structure will be immediately followed by extra space, */ + /* as required. DECNUMDIGITS is always >0. */ + #if !defined(DECNUMDIGITS) + #define DECNUMDIGITS 1 + #endif + + /* The size (integer data type) of each unit is determined by the */ + /* number of digits it will hold. */ + #if DECDPUN<=2 + #define decNumberUnit uint8_t + #elif DECDPUN<=4 + #define decNumberUnit uint16_t + #else + #define decNumberUnit uint32_t + #endif + /* The number of units needed is ceil(DECNUMDIGITS/DECDPUN) */ + #define DECNUMUNITS ((DECNUMDIGITS+DECDPUN-1)/DECDPUN) + + /* The data structure... */ + typedef struct { + int32_t digits; /* Count of digits in the coefficient; >0 */ + int32_t exponent; /* Unadjusted exponent, unbiased, in */ + /* range: -1999999997 through 999999999 */ + uint8_t bits; /* Indicator bits (see above) */ + /* Coefficient, from least significant unit */ + decNumberUnit lsu[DECNUMUNITS]; + } decNumber; + + /* Notes: */ + /* 1. If digits is > DECDPUN then there will one or more */ + /* decNumberUnits immediately following the first element of lsu.*/ + /* These contain the remaining (more significant) digits of the */ + /* number, and may be in the lsu array, or may be guaranteed by */ + /* some other mechanism (such as being contained in another */ + /* structure, or being overlaid on dynamically allocated */ + /* storage). */ + /* */ + /* Each integer of the coefficient (except potentially the last) */ + /* contains DECDPUN digits (e.g., a value in the range 0 through */ + /* 99999999 if DECDPUN is 8, or 0 through 999 if DECDPUN is 3). */ + /* */ + /* 2. A decNumber converted to a string may need up to digits+14 */ + /* characters. The worst cases (non-exponential and exponential */ + /* formats) are -0.00000{9...}# and -9.{9...}E+999999999# */ + /* (where # is '\0') */ + + + /* ---------------------------------------------------------------- */ + /* decNumber public functions and macros */ + /* ---------------------------------------------------------------- */ + /* Conversions */ + U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromInt32(decNumber *, int32_t); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromUInt32(decNumber *, uint32_t); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromString(decNumber *, const char *, decContext *); + U_CAPI char * U_EXPORT2 uprv_decNumberToString(const decNumber *, char *); + U_CAPI char * U_EXPORT2 uprv_decNumberToEngString(const decNumber *, char *); + U_CAPI uint32_t U_EXPORT2 uprv_decNumberToUInt32(const decNumber *, decContext *); + U_CAPI int32_t U_EXPORT2 uprv_decNumberToInt32(const decNumber *, decContext *); + U_CAPI uint8_t * U_EXPORT2 uprv_decNumberGetBCD(const decNumber *, uint8_t *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberSetBCD(decNumber *, const uint8_t *, uint32_t); + + /* Operators and elementary functions */ + U_CAPI decNumber * U_EXPORT2 uprv_decNumberAbs(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberAdd(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberAnd(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompare(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareSignal(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotal(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotalMag(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivide(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivideInteger(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberExp(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberFMA(decNumber *, const decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberInvert(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberLn(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberLogB(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberLog10(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMax(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMaxMag(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMin(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinMag(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinus(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberMultiply(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberNormalize(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberOr(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberPlus(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberPower(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberQuantize(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberReduce(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainder(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainderNear(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberRescale(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberRotate(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberSameQuantum(decNumber *, const decNumber *, const decNumber *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberScaleB(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberShift(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberSquareRoot(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberSubtract(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralExact(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralValue(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberXor(decNumber *, const decNumber *, const decNumber *, decContext *); + + /* Utilities */ + enum decClass uprv_decNumberClass(const decNumber *, decContext *); + U_CAPI const char * U_EXPORT2 uprv_decNumberClassToString(enum decClass); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopy(decNumber *, const decNumber *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyAbs(decNumber *, const decNumber *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyNegate(decNumber *, const decNumber *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopySign(decNumber *, const decNumber *, const decNumber *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextMinus(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextPlus(decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextToward(decNumber *, const decNumber *, const decNumber *, decContext *); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberTrim(decNumber *); + U_CAPI const char * U_EXPORT2 uprv_decNumberVersion(); + U_CAPI decNumber * U_EXPORT2 uprv_decNumberZero(decNumber *); + + /* Functions for testing decNumbers (normality depends on context) */ + U_CAPI int32_t U_EXPORT2 uprv_decNumberIsNormal(const decNumber *, decContext *); + U_CAPI int32_t U_EXPORT2 uprv_decNumberIsSubnormal(const decNumber *, decContext *); + + /* Macros for testing decNumber *dn */ + #define decNumberIsCanonical(dn) (1) /* All decNumbers are saintly */ + #define decNumberIsFinite(dn) (((dn)->bits&DECSPECIAL)==0) + #define decNumberIsInfinite(dn) (((dn)->bits&DECINF)!=0) + #define decNumberIsNaN(dn) (((dn)->bits&(DECNAN|DECSNAN))!=0) + #define decNumberIsNegative(dn) (((dn)->bits&DECNEG)!=0) + #define decNumberIsQNaN(dn) (((dn)->bits&(DECNAN))!=0) + #define decNumberIsSNaN(dn) (((dn)->bits&(DECSNAN))!=0) + #define decNumberIsSpecial(dn) (((dn)->bits&DECSPECIAL)!=0) + #define decNumberIsZero(dn) (*(dn)->lsu==0 \ + && (dn)->digits==1 \ + && (((dn)->bits&DECSPECIAL)==0)) + #define decNumberRadix(dn) (10) + +#endif diff --git a/intl/icu/source/i18n/decNumberLocal.h b/intl/icu/source/i18n/decNumberLocal.h new file mode 100644 index 0000000000..1c5a79b702 --- /dev/null +++ b/intl/icu/source/i18n/decNumberLocal.h @@ -0,0 +1,728 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* ------------------------------------------------------------------ */ +/* decNumber package local type, tuning, and macro definitions */ +/* ------------------------------------------------------------------ */ +/* Copyright (c) IBM Corporation, 2000-2016. All rights reserved. */ +/* */ +/* This software is made available under the terms of the */ +/* ICU License -- ICU 1.8.1 and later. */ +/* */ +/* The description and User's Guide ("The decNumber C Library") for */ +/* this software is called decNumber.pdf. This document is */ +/* available, together with arithmetic and format specifications, */ +/* testcases, and Web links, on the General Decimal Arithmetic page. */ +/* */ +/* Please send comments, suggestions, and corrections to the author: */ +/* mfc@uk.ibm.com */ +/* Mike Cowlishaw, IBM Fellow */ +/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */ +/* ------------------------------------------------------------------ */ +/* This header file is included by all modules in the decNumber */ +/* library, and contains local type definitions, tuning parameters, */ +/* etc. It should not need to be used by application programs. */ +/* decNumber.h or one of decDouble (etc.) must be included first. */ +/* ------------------------------------------------------------------ */ + +#if !defined(DECNUMBERLOC) + #define DECNUMBERLOC + #define DECVERSION "decNumber 3.61" /* Package Version [16 max.] */ + #define DECNLAUTHOR "Mike Cowlishaw" /* Who to blame */ + + #include /* for abs */ + #include /* for memset, strcpy */ + #include "decContext.h" + + /* Conditional code flag -- set this to match hardware platform */ + #if !defined(DECLITEND) + #define DECLITEND 1 /* 1=little-endian, 0=big-endian */ + #endif + + /* Conditional code flag -- set this to 1 for best performance */ + #if !defined(DECUSE64) + #define DECUSE64 1 /* 1=use int64s, 0=int32 & smaller only */ + #endif + + /* Conditional check flags -- set these to 0 for best performance */ + #if !defined(DECCHECK) + #define DECCHECK 0 /* 1 to enable robust checking */ + #endif + #if !defined(DECALLOC) + #define DECALLOC 0 /* 1 to enable memory accounting */ + #endif + #if !defined(DECTRACE) + #define DECTRACE 0 /* 1 to trace certain internals, etc. */ + #endif + + /* Tuning parameter for decNumber (arbitrary precision) module */ + #if !defined(DECBUFFER) + #define DECBUFFER 36 /* Size basis for local buffers. This */ + /* should be a common maximum precision */ + /* rounded up to a multiple of 4; must */ + /* be zero or positive. */ + #endif + + /* ---------------------------------------------------------------- */ + /* Definitions for all modules (general-purpose) */ + /* ---------------------------------------------------------------- */ + + /* Local names for common types -- for safety, decNumber modules do */ + /* not use int or long directly. */ + #define Flag uint8_t + #define Byte int8_t + #define uByte uint8_t + #define Short int16_t + #define uShort uint16_t + #define Int int32_t + #define uInt uint32_t + #define Unit decNumberUnit + #if DECUSE64 + #define Long int64_t + #define uLong uint64_t + #endif + + /* Development-use definitions */ + typedef long int LI; /* for printf arguments only */ + #define DECNOINT 0 /* 1 to check no internal use of 'int' */ + /* or stdint types */ + #if DECNOINT + /* if these interfere with your C includes, do not set DECNOINT */ + #define int ? /* enable to ensure that plain C 'int' */ + #define long ?? /* .. or 'long' types are not used */ + #endif + + /* LONGMUL32HI -- set w=(u*v)>>32, where w, u, and v are uInts */ + /* (that is, sets w to be the high-order word of the 64-bit result; */ + /* the low-order word is simply u*v.) */ + /* This version is derived from Knuth via Hacker's Delight; */ + /* it seems to optimize better than some others tried */ + #define LONGMUL32HI(w, u, v) { \ + uInt u0, u1, v0, v1, w0, w1, w2, t; \ + u0=u & 0xffff; u1=u>>16; \ + v0=v & 0xffff; v1=v>>16; \ + w0=u0*v0; \ + t=u1*v0 + (w0>>16); \ + w1=t & 0xffff; w2=t>>16; \ + w1=u0*v1 + w1; \ + (w)=u1*v1 + w2 + (w1>>16);} + + /* ROUNDUP -- round an integer up to a multiple of n */ + #define ROUNDUP(i, n) ((((i)+(n)-1)/n)*n) + #define ROUNDUP4(i) (((i)+3)&~3) /* special for n=4 */ + + /* ROUNDDOWN -- round an integer down to a multiple of n */ + #define ROUNDDOWN(i, n) (((i)/n)*n) + #define ROUNDDOWN4(i) ((i)&~3) /* special for n=4 */ + + /* References to multi-byte sequences under different sizes; these */ + /* require locally declared variables, but do not violate strict */ + /* aliasing or alignment (as did the UINTAT simple cast to uInt). */ + /* Variables needed are uswork, uiwork, etc. [so do not use at same */ + /* level in an expression, e.g., UBTOUI(x)==UBTOUI(y) may fail]. */ + + /* Return a uInt, etc., from bytes starting at a char* or uByte* */ + #define UBTOUS(b) (memcpy((void *)&uswork, b, 2), uswork) + #define UBTOUI(b) (memcpy((void *)&uiwork, b, 4), uiwork) + + /* Store a uInt, etc., into bytes starting at a char* or uByte*. */ + /* Returns i, evaluated, for convenience; has to use uiwork because */ + /* i may be an expression. */ + #define UBFROMUS(b, i) (uswork=(i), memcpy(b, (void *)&uswork, 2), uswork) + #define UBFROMUI(b, i) (uiwork=(i), memcpy(b, (void *)&uiwork, 4), uiwork) + + /* X10 and X100 -- multiply integer i by 10 or 100 */ + /* [shifts are usually faster than multiply; could be conditional] */ + #define X10(i) (((i)<<1)+((i)<<3)) + #define X100(i) (((i)<<2)+((i)<<5)+((i)<<6)) + + /* MAXI and MINI -- general max & min (not in ANSI) for integers */ + #define MAXI(x,y) ((x)<(y)?(y):(x)) + #define MINI(x,y) ((x)>(y)?(y):(x)) + + /* Useful constants */ + #define BILLION 1000000000 /* 10**9 */ + /* CHARMASK: 0x30303030 for ASCII/UTF8; 0xF0F0F0F0 for EBCDIC */ + #define CHARMASK ((((((((uInt)'0')<<8)+'0')<<8)+'0')<<8)+'0') + + + /* ---------------------------------------------------------------- */ + /* Definitions for arbitrary-precision modules (only valid after */ + /* decNumber.h has been included) */ + /* ---------------------------------------------------------------- */ + + /* Limits and constants */ + #define DECNUMMAXP 999999999 /* maximum precision code can handle */ + #define DECNUMMAXE 999999999 /* maximum adjusted exponent ditto */ + #define DECNUMMINE -999999999 /* minimum adjusted exponent ditto */ + #if (DECNUMMAXP != DEC_MAX_DIGITS) + #error Maximum digits mismatch + #endif + #if (DECNUMMAXE != DEC_MAX_EMAX) + #error Maximum exponent mismatch + #endif + #if (DECNUMMINE != DEC_MIN_EMIN) + #error Minimum exponent mismatch + #endif + + /* Set DECDPUNMAX -- the maximum integer that fits in DECDPUN */ + /* digits, and D2UTABLE -- the initializer for the D2U table */ + #ifndef DECDPUN + // no-op + #elif DECDPUN==1 + #define DECDPUNMAX 9 + #define D2UTABLE {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17, \ + 18,19,20,21,22,23,24,25,26,27,28,29,30,31,32, \ + 33,34,35,36,37,38,39,40,41,42,43,44,45,46,47, \ + 48,49} + #elif DECDPUN==2 + #define DECDPUNMAX 99 + #define D2UTABLE {0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10, \ + 11,11,12,12,13,13,14,14,15,15,16,16,17,17,18, \ + 18,19,19,20,20,21,21,22,22,23,23,24,24,25} + #elif DECDPUN==3 + #define DECDPUNMAX 999 + #define D2UTABLE {0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7, \ + 8,8,8,9,9,9,10,10,10,11,11,11,12,12,12,13,13, \ + 13,14,14,14,15,15,15,16,16,16,17} + #elif DECDPUN==4 + #define DECDPUNMAX 9999 + #define D2UTABLE {0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6, \ + 6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11, \ + 11,11,11,12,12,12,12,13} + #elif DECDPUN==5 + #define DECDPUNMAX 99999 + #define D2UTABLE {0,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,5, \ + 5,5,5,5,6,6,6,6,6,7,7,7,7,7,8,8,8,8,8,9,9,9, \ + 9,9,10,10,10,10} + #elif DECDPUN==6 + #define DECDPUNMAX 999999 + #define D2UTABLE {0,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4, \ + 4,4,4,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,7,7,7,8, \ + 8,8,8,8,8,9} + #elif DECDPUN==7 + #define DECDPUNMAX 9999999 + #define D2UTABLE {0,1,1,1,1,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,3, \ + 4,4,4,4,4,4,4,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7, \ + 7,7,7,7,7,7} + #elif DECDPUN==8 + #define DECDPUNMAX 99999999 + #define D2UTABLE {0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3, \ + 3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6, \ + 6,6,6,6,6,7} + #elif DECDPUN==9 + #define DECDPUNMAX 999999999 + #define D2UTABLE {0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3, \ + 3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5, \ + 5,5,6,6,6,6} + #else + #error DECDPUN must be in the range 1-9 + #endif + + /* ----- Shared data (in decNumber.c) ----- */ + /* Public lookup table used by the D2U macro (see below) */ + #define DECMAXD2U 49 + /*extern const uByte d2utable[DECMAXD2U+1];*/ + + /* ----- Macros ----- */ + /* ISZERO -- return true if decNumber dn is a zero */ + /* [performance-critical in some situations] */ + #define ISZERO(dn) decNumberIsZero(dn) /* now just a local name */ + + /* D2U -- return the number of Units needed to hold d digits */ + /* (runtime version, with table lookaside for small d) */ + #if defined(DECDPUN) && DECDPUN==8 + #define D2U(d) ((unsigned)((d)<=DECMAXD2U?d2utable[d]:((d)+7)>>3)) + #elif defined(DECDPUN) && DECDPUN==4 + #define D2U(d) ((unsigned)((d)<=DECMAXD2U?d2utable[d]:((d)+3)>>2)) + #else + #define D2U(d) ((d)<=DECMAXD2U?d2utable[d]:((d)+DECDPUN-1)/DECDPUN) + #endif + /* SD2U -- static D2U macro (for compile-time calculation) */ + #define SD2U(d) (((d)+DECDPUN-1)/DECDPUN) + + /* MSUDIGITS -- returns digits in msu, from digits, calculated */ + /* using D2U */ + #define MSUDIGITS(d) ((d)-(D2U(d)-1)*DECDPUN) + + /* D2N -- return the number of decNumber structs that would be */ + /* needed to contain that number of digits (and the initial */ + /* decNumber struct) safely. Note that one Unit is included in the */ + /* initial structure. Used for allocating space that is aligned on */ + /* a decNumber struct boundary. */ + #define D2N(d) \ + ((((SD2U(d)-1)*sizeof(Unit))+sizeof(decNumber)*2-1)/sizeof(decNumber)) + + /* TODIGIT -- macro to remove the leading digit from the unsigned */ + /* integer u at column cut (counting from the right, LSD=0) and */ + /* place it as an ASCII character into the character pointed to by */ + /* c. Note that cut must be <= 9, and the maximum value for u is */ + /* 2,000,000,000 (as is needed for negative exponents of */ + /* subnormals). The unsigned integer pow is used as a temporary */ + /* variable. */ + #define TODIGIT(u, cut, c, pow) UPRV_BLOCK_MACRO_BEGIN { \ + *(c)='0'; \ + pow=DECPOWERS[cut]*2; \ + if ((u)>pow) { \ + pow*=4; \ + if ((u)>=pow) {(u)-=pow; *(c)+=8;} \ + pow/=2; \ + if ((u)>=pow) {(u)-=pow; *(c)+=4;} \ + pow/=2; \ + } \ + if ((u)>=pow) {(u)-=pow; *(c)+=2;} \ + pow/=2; \ + if ((u)>=pow) {(u)-=pow; *(c)+=1;} \ + } UPRV_BLOCK_MACRO_END + + /* ---------------------------------------------------------------- */ + /* Definitions for fixed-precision modules (only valid after */ + /* decSingle.h, decDouble.h, or decQuad.h has been included) */ + /* ---------------------------------------------------------------- */ + + /* bcdnum -- a structure describing a format-independent finite */ + /* number, whose coefficient is a string of bcd8 uBytes */ + typedef struct { + uByte *msd; /* -> most significant digit */ + uByte *lsd; /* -> least ditto */ + uInt sign; /* 0=positive, DECFLOAT_Sign=negative */ + Int exponent; /* Unadjusted signed exponent (q), or */ + /* DECFLOAT_NaN etc. for a special */ + } bcdnum; + + /* Test if exponent or bcdnum exponent must be a special, etc. */ + #define EXPISSPECIAL(exp) ((exp)>=DECFLOAT_MinSp) + #define EXPISINF(exp) (exp==DECFLOAT_Inf) + #define EXPISNAN(exp) (exp==DECFLOAT_qNaN || exp==DECFLOAT_sNaN) + #define NUMISSPECIAL(num) (EXPISSPECIAL((num)->exponent)) + + /* Refer to a 32-bit word or byte in a decFloat (df) by big-endian */ + /* (array) notation (the 0 word or byte contains the sign bit), */ + /* automatically adjusting for endianness; similarly address a word */ + /* in the next-wider format (decFloatWider, or dfw) */ + #define DECWORDS (DECBYTES/4) + #define DECWWORDS (DECWBYTES/4) + #if DECLITEND + #define DFBYTE(df, off) ((df)->bytes[DECBYTES-1-(off)]) + #define DFWORD(df, off) ((df)->words[DECWORDS-1-(off)]) + #define DFWWORD(dfw, off) ((dfw)->words[DECWWORDS-1-(off)]) + #else + #define DFBYTE(df, off) ((df)->bytes[off]) + #define DFWORD(df, off) ((df)->words[off]) + #define DFWWORD(dfw, off) ((dfw)->words[off]) + #endif + + /* Tests for sign or specials, directly on DECFLOATs */ + #define DFISSIGNED(df) (DFWORD(df, 0)&0x80000000) + #define DFISSPECIAL(df) ((DFWORD(df, 0)&0x78000000)==0x78000000) + #define DFISINF(df) ((DFWORD(df, 0)&0x7c000000)==0x78000000) + #define DFISNAN(df) ((DFWORD(df, 0)&0x7c000000)==0x7c000000) + #define DFISQNAN(df) ((DFWORD(df, 0)&0x7e000000)==0x7c000000) + #define DFISSNAN(df) ((DFWORD(df, 0)&0x7e000000)==0x7e000000) + + /* Shared lookup tables */ + extern const uInt DECCOMBMSD[64]; /* Combination field -> MSD */ + extern const uInt DECCOMBFROM[48]; /* exp+msd -> Combination */ + + /* Private generic (utility) routine */ + #if DECCHECK || DECTRACE + extern void decShowNum(const bcdnum *, const char *); + #endif + + /* Format-dependent macros and constants */ + #if defined(DECPMAX) + + /* Useful constants */ + #define DECPMAX9 (ROUNDUP(DECPMAX, 9)/9) /* 'Pmax' in 10**9s */ + /* Top words for a zero */ + #define SINGLEZERO 0x22500000 + #define DOUBLEZERO 0x22380000 + #define QUADZERO 0x22080000 + /* [ZEROWORD is defined to be one of these in the DFISZERO macro] */ + + /* Format-dependent common tests: */ + /* DFISZERO -- test for (any) zero */ + /* DFISCCZERO -- test for coefficient continuation being zero */ + /* DFISCC01 -- test for coefficient contains only 0s and 1s */ + /* DFISINT -- test for finite and exponent q=0 */ + /* DFISUINT01 -- test for sign=0, finite, exponent q=0, and */ + /* MSD=0 or 1 */ + /* ZEROWORD is also defined here. */ + /* In DFISZERO the first test checks the least-significant word */ + /* (most likely to be non-zero); the penultimate tests MSD and */ + /* DPDs in the signword, and the final test excludes specials and */ + /* MSD>7. DFISINT similarly has to allow for the two forms of */ + /* MSD codes. DFISUINT01 only has to allow for one form of MSD */ + /* code. */ + #if DECPMAX==7 + #define ZEROWORD SINGLEZERO + /* [test macros not needed except for Zero] */ + #define DFISZERO(df) ((DFWORD(df, 0)&0x1c0fffff)==0 \ + && (DFWORD(df, 0)&0x60000000)!=0x60000000) + #elif DECPMAX==16 + #define ZEROWORD DOUBLEZERO + #define DFISZERO(df) ((DFWORD(df, 1)==0 \ + && (DFWORD(df, 0)&0x1c03ffff)==0 \ + && (DFWORD(df, 0)&0x60000000)!=0x60000000)) + #define DFISINT(df) ((DFWORD(df, 0)&0x63fc0000)==0x22380000 \ + ||(DFWORD(df, 0)&0x7bfc0000)==0x6a380000) + #define DFISUINT01(df) ((DFWORD(df, 0)&0xfbfc0000)==0x22380000) + #define DFISCCZERO(df) (DFWORD(df, 1)==0 \ + && (DFWORD(df, 0)&0x0003ffff)==0) + #define DFISCC01(df) ((DFWORD(df, 0)&~0xfffc9124)==0 \ + && (DFWORD(df, 1)&~0x49124491)==0) + #elif DECPMAX==34 + #define ZEROWORD QUADZERO + #define DFISZERO(df) ((DFWORD(df, 3)==0 \ + && DFWORD(df, 2)==0 \ + && DFWORD(df, 1)==0 \ + && (DFWORD(df, 0)&0x1c003fff)==0 \ + && (DFWORD(df, 0)&0x60000000)!=0x60000000)) + #define DFISINT(df) ((DFWORD(df, 0)&0x63ffc000)==0x22080000 \ + ||(DFWORD(df, 0)&0x7bffc000)==0x6a080000) + #define DFISUINT01(df) ((DFWORD(df, 0)&0xfbffc000)==0x22080000) + #define DFISCCZERO(df) (DFWORD(df, 3)==0 \ + && DFWORD(df, 2)==0 \ + && DFWORD(df, 1)==0 \ + && (DFWORD(df, 0)&0x00003fff)==0) + + #define DFISCC01(df) ((DFWORD(df, 0)&~0xffffc912)==0 \ + && (DFWORD(df, 1)&~0x44912449)==0 \ + && (DFWORD(df, 2)&~0x12449124)==0 \ + && (DFWORD(df, 3)&~0x49124491)==0) + #endif + + /* Macros to test if a certain 10 bits of a uInt or pair of uInts */ + /* are a canonical declet [higher or lower bits are ignored]. */ + /* declet is at offset 0 (from the right) in a uInt: */ + #define CANONDPD(dpd) (((dpd)&0x300)==0 || ((dpd)&0x6e)!=0x6e) + /* declet is at offset k (a multiple of 2) in a uInt: */ + #define CANONDPDOFF(dpd, k) (((dpd)&(0x300<<(k)))==0 \ + || ((dpd)&(((uInt)0x6e)<<(k)))!=(((uInt)0x6e)<<(k))) + /* declet is at offset k (a multiple of 2) in a pair of uInts: */ + /* [the top 2 bits will always be in the more-significant uInt] */ + #define CANONDPDTWO(hi, lo, k) (((hi)&(0x300>>(32-(k))))==0 \ + || ((hi)&(0x6e>>(32-(k))))!=(0x6e>>(32-(k))) \ + || ((lo)&(((uInt)0x6e)<<(k)))!=(((uInt)0x6e)<<(k))) + + /* Macro to test whether a full-length (length DECPMAX) BCD8 */ + /* coefficient, starting at uByte u, is all zeros */ + /* Test just the LSWord first, then the remainder as a sequence */ + /* of tests in order to avoid same-level use of UBTOUI */ + #if DECPMAX==7 + #define ISCOEFFZERO(u) ( \ + UBTOUI((u)+DECPMAX-4)==0 \ + && UBTOUS((u)+DECPMAX-6)==0 \ + && *(u)==0) + #elif DECPMAX==16 + #define ISCOEFFZERO(u) ( \ + UBTOUI((u)+DECPMAX-4)==0 \ + && UBTOUI((u)+DECPMAX-8)==0 \ + && UBTOUI((u)+DECPMAX-12)==0 \ + && UBTOUI(u)==0) + #elif DECPMAX==34 + #define ISCOEFFZERO(u) ( \ + UBTOUI((u)+DECPMAX-4)==0 \ + && UBTOUI((u)+DECPMAX-8)==0 \ + && UBTOUI((u)+DECPMAX-12)==0 \ + && UBTOUI((u)+DECPMAX-16)==0 \ + && UBTOUI((u)+DECPMAX-20)==0 \ + && UBTOUI((u)+DECPMAX-24)==0 \ + && UBTOUI((u)+DECPMAX-28)==0 \ + && UBTOUI((u)+DECPMAX-32)==0 \ + && UBTOUS(u)==0) + #endif + + /* Macros and masks for the exponent continuation field and MSD */ + /* Get the exponent continuation from a decFloat *df as an Int */ + #define GETECON(df) ((Int)((DFWORD((df), 0)&0x03ffffff)>>(32-6-DECECONL))) + /* Ditto, from the next-wider format */ + #define GETWECON(df) ((Int)((DFWWORD((df), 0)&0x03ffffff)>>(32-6-DECWECONL))) + /* Get the biased exponent similarly */ + #define GETEXP(df) ((Int)(DECCOMBEXP[DFWORD((df), 0)>>26]+GETECON(df))) + /* Get the unbiased exponent similarly */ + #define GETEXPUN(df) ((Int)GETEXP(df)-DECBIAS) + /* Get the MSD similarly (as uInt) */ + #define GETMSD(df) (DECCOMBMSD[DFWORD((df), 0)>>26]) + + /* Compile-time computes of the exponent continuation field masks */ + /* full exponent continuation field: */ + #define ECONMASK ((0x03ffffff>>(32-6-DECECONL))<<(32-6-DECECONL)) + /* same, not including its first digit (the qNaN/sNaN selector): */ + #define ECONNANMASK ((0x01ffffff>>(32-6-DECECONL))<<(32-6-DECECONL)) + + /* Macros to decode the coefficient in a finite decFloat *df into */ + /* a BCD string (uByte *bcdin) of length DECPMAX uBytes. */ + + /* In-line sequence to convert least significant 10 bits of uInt */ + /* dpd to three BCD8 digits starting at uByte u. Note that an */ + /* extra byte is written to the right of the three digits because */ + /* four bytes are moved at a time for speed; the alternative */ + /* macro moves exactly three bytes (usually slower). */ + #define dpd2bcd8(u, dpd) memcpy(u, &DPD2BCD8[((dpd)&0x3ff)*4], 4) + #define dpd2bcd83(u, dpd) memcpy(u, &DPD2BCD8[((dpd)&0x3ff)*4], 3) + + /* Decode the declets. After extracting each one, it is decoded */ + /* to BCD8 using a table lookup (also used for variable-length */ + /* decode). Each DPD decode is 3 bytes BCD8 plus a one-byte */ + /* length which is not used, here). Fixed-length 4-byte moves */ + /* are fast, however, almost everywhere, and so are used except */ + /* for the final three bytes (to avoid overrun). The code below */ + /* is 36 instructions for Doubles and about 70 for Quads, even */ + /* on IA32. */ + + /* Two macros are defined for each format: */ + /* GETCOEFF extracts the coefficient of the current format */ + /* GETWCOEFF extracts the coefficient of the next-wider format. */ + /* The latter is a copy of the next-wider GETCOEFF using DFWWORD. */ + + #if DECPMAX==7 + #define GETCOEFF(df, bcd) { \ + uInt sourhi=DFWORD(df, 0); \ + *(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \ + dpd2bcd8(bcd+1, sourhi>>10); \ + dpd2bcd83(bcd+4, sourhi);} + #define GETWCOEFF(df, bcd) { \ + uInt sourhi=DFWWORD(df, 0); \ + uInt sourlo=DFWWORD(df, 1); \ + *(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \ + dpd2bcd8(bcd+1, sourhi>>8); \ + dpd2bcd8(bcd+4, (sourhi<<2) | (sourlo>>30)); \ + dpd2bcd8(bcd+7, sourlo>>20); \ + dpd2bcd8(bcd+10, sourlo>>10); \ + dpd2bcd83(bcd+13, sourlo);} + + #elif DECPMAX==16 + #define GETCOEFF(df, bcd) { \ + uInt sourhi=DFWORD(df, 0); \ + uInt sourlo=DFWORD(df, 1); \ + *(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \ + dpd2bcd8(bcd+1, sourhi>>8); \ + dpd2bcd8(bcd+4, (sourhi<<2) | (sourlo>>30)); \ + dpd2bcd8(bcd+7, sourlo>>20); \ + dpd2bcd8(bcd+10, sourlo>>10); \ + dpd2bcd83(bcd+13, sourlo);} + #define GETWCOEFF(df, bcd) { \ + uInt sourhi=DFWWORD(df, 0); \ + uInt sourmh=DFWWORD(df, 1); \ + uInt sourml=DFWWORD(df, 2); \ + uInt sourlo=DFWWORD(df, 3); \ + *(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \ + dpd2bcd8(bcd+1, sourhi>>4); \ + dpd2bcd8(bcd+4, ((sourhi)<<6) | (sourmh>>26)); \ + dpd2bcd8(bcd+7, sourmh>>16); \ + dpd2bcd8(bcd+10, sourmh>>6); \ + dpd2bcd8(bcd+13, ((sourmh)<<4) | (sourml>>28)); \ + dpd2bcd8(bcd+16, sourml>>18); \ + dpd2bcd8(bcd+19, sourml>>8); \ + dpd2bcd8(bcd+22, ((sourml)<<2) | (sourlo>>30)); \ + dpd2bcd8(bcd+25, sourlo>>20); \ + dpd2bcd8(bcd+28, sourlo>>10); \ + dpd2bcd83(bcd+31, sourlo);} + + #elif DECPMAX==34 + #define GETCOEFF(df, bcd) { \ + uInt sourhi=DFWORD(df, 0); \ + uInt sourmh=DFWORD(df, 1); \ + uInt sourml=DFWORD(df, 2); \ + uInt sourlo=DFWORD(df, 3); \ + *(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \ + dpd2bcd8(bcd+1, sourhi>>4); \ + dpd2bcd8(bcd+4, ((sourhi)<<6) | (sourmh>>26)); \ + dpd2bcd8(bcd+7, sourmh>>16); \ + dpd2bcd8(bcd+10, sourmh>>6); \ + dpd2bcd8(bcd+13, ((sourmh)<<4) | (sourml>>28)); \ + dpd2bcd8(bcd+16, sourml>>18); \ + dpd2bcd8(bcd+19, sourml>>8); \ + dpd2bcd8(bcd+22, ((sourml)<<2) | (sourlo>>30)); \ + dpd2bcd8(bcd+25, sourlo>>20); \ + dpd2bcd8(bcd+28, sourlo>>10); \ + dpd2bcd83(bcd+31, sourlo);} + + #define GETWCOEFF(df, bcd) {??} /* [should never be used] */ + #endif + + /* Macros to decode the coefficient in a finite decFloat *df into */ + /* a base-billion uInt array, with the least-significant */ + /* 0-999999999 'digit' at offset 0. */ + + /* Decode the declets. After extracting each one, it is decoded */ + /* to binary using a table lookup. Three tables are used; one */ + /* the usual DPD to binary, the other two pre-multiplied by 1000 */ + /* and 1000000 to avoid multiplication during decode. These */ + /* tables can also be used for multiplying up the MSD as the DPD */ + /* code for 0 through 9 is the identity. */ + #define DPD2BIN0 DPD2BIN /* for prettier code */ + + #if DECPMAX==7 + #define GETCOEFFBILL(df, buf) { \ + uInt sourhi=DFWORD(df, 0); \ + (buf)[0]=DPD2BIN0[sourhi&0x3ff] \ + +DPD2BINK[(sourhi>>10)&0x3ff] \ + +DPD2BINM[DECCOMBMSD[sourhi>>26]];} + + #elif DECPMAX==16 + #define GETCOEFFBILL(df, buf) { \ + uInt sourhi, sourlo; \ + sourlo=DFWORD(df, 1); \ + (buf)[0]=DPD2BIN0[sourlo&0x3ff] \ + +DPD2BINK[(sourlo>>10)&0x3ff] \ + +DPD2BINM[(sourlo>>20)&0x3ff]; \ + sourhi=DFWORD(df, 0); \ + (buf)[1]=DPD2BIN0[((sourhi<<2) | (sourlo>>30))&0x3ff] \ + +DPD2BINK[(sourhi>>8)&0x3ff] \ + +DPD2BINM[DECCOMBMSD[sourhi>>26]];} + + #elif DECPMAX==34 + #define GETCOEFFBILL(df, buf) { \ + uInt sourhi, sourmh, sourml, sourlo; \ + sourlo=DFWORD(df, 3); \ + (buf)[0]=DPD2BIN0[sourlo&0x3ff] \ + +DPD2BINK[(sourlo>>10)&0x3ff] \ + +DPD2BINM[(sourlo>>20)&0x3ff]; \ + sourml=DFWORD(df, 2); \ + (buf)[1]=DPD2BIN0[((sourml<<2) | (sourlo>>30))&0x3ff] \ + +DPD2BINK[(sourml>>8)&0x3ff] \ + +DPD2BINM[(sourml>>18)&0x3ff]; \ + sourmh=DFWORD(df, 1); \ + (buf)[2]=DPD2BIN0[((sourmh<<4) | (sourml>>28))&0x3ff] \ + +DPD2BINK[(sourmh>>6)&0x3ff] \ + +DPD2BINM[(sourmh>>16)&0x3ff]; \ + sourhi=DFWORD(df, 0); \ + (buf)[3]=DPD2BIN0[((sourhi<<6) | (sourmh>>26))&0x3ff] \ + +DPD2BINK[(sourhi>>4)&0x3ff] \ + +DPD2BINM[DECCOMBMSD[sourhi>>26]];} + + #endif + + /* Macros to decode the coefficient in a finite decFloat *df into */ + /* a base-thousand uInt array (of size DECLETS+1, to allow for */ + /* the MSD), with the least-significant 0-999 'digit' at offset 0.*/ + + /* Decode the declets. After extracting each one, it is decoded */ + /* to binary using a table lookup. */ + #if DECPMAX==7 + #define GETCOEFFTHOU(df, buf) { \ + uInt sourhi=DFWORD(df, 0); \ + (buf)[0]=DPD2BIN[sourhi&0x3ff]; \ + (buf)[1]=DPD2BIN[(sourhi>>10)&0x3ff]; \ + (buf)[2]=DECCOMBMSD[sourhi>>26];} + + #elif DECPMAX==16 + #define GETCOEFFTHOU(df, buf) { \ + uInt sourhi, sourlo; \ + sourlo=DFWORD(df, 1); \ + (buf)[0]=DPD2BIN[sourlo&0x3ff]; \ + (buf)[1]=DPD2BIN[(sourlo>>10)&0x3ff]; \ + (buf)[2]=DPD2BIN[(sourlo>>20)&0x3ff]; \ + sourhi=DFWORD(df, 0); \ + (buf)[3]=DPD2BIN[((sourhi<<2) | (sourlo>>30))&0x3ff]; \ + (buf)[4]=DPD2BIN[(sourhi>>8)&0x3ff]; \ + (buf)[5]=DECCOMBMSD[sourhi>>26];} + + #elif DECPMAX==34 + #define GETCOEFFTHOU(df, buf) { \ + uInt sourhi, sourmh, sourml, sourlo; \ + sourlo=DFWORD(df, 3); \ + (buf)[0]=DPD2BIN[sourlo&0x3ff]; \ + (buf)[1]=DPD2BIN[(sourlo>>10)&0x3ff]; \ + (buf)[2]=DPD2BIN[(sourlo>>20)&0x3ff]; \ + sourml=DFWORD(df, 2); \ + (buf)[3]=DPD2BIN[((sourml<<2) | (sourlo>>30))&0x3ff]; \ + (buf)[4]=DPD2BIN[(sourml>>8)&0x3ff]; \ + (buf)[5]=DPD2BIN[(sourml>>18)&0x3ff]; \ + sourmh=DFWORD(df, 1); \ + (buf)[6]=DPD2BIN[((sourmh<<4) | (sourml>>28))&0x3ff]; \ + (buf)[7]=DPD2BIN[(sourmh>>6)&0x3ff]; \ + (buf)[8]=DPD2BIN[(sourmh>>16)&0x3ff]; \ + sourhi=DFWORD(df, 0); \ + (buf)[9]=DPD2BIN[((sourhi<<6) | (sourmh>>26))&0x3ff]; \ + (buf)[10]=DPD2BIN[(sourhi>>4)&0x3ff]; \ + (buf)[11]=DECCOMBMSD[sourhi>>26];} + #endif + + + /* Macros to decode the coefficient in a finite decFloat *df and */ + /* add to a base-thousand uInt array (as for GETCOEFFTHOU). */ + /* After the addition then most significant 'digit' in the array */ + /* might have a value larger then 10 (with a maximum of 19). */ + #if DECPMAX==7 + #define ADDCOEFFTHOU(df, buf) { \ + uInt sourhi=DFWORD(df, 0); \ + (buf)[0]+=DPD2BIN[sourhi&0x3ff]; \ + if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \ + (buf)[1]+=DPD2BIN[(sourhi>>10)&0x3ff]; \ + if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \ + (buf)[2]+=DECCOMBMSD[sourhi>>26];} + + #elif DECPMAX==16 + #define ADDCOEFFTHOU(df, buf) { \ + uInt sourhi, sourlo; \ + sourlo=DFWORD(df, 1); \ + (buf)[0]+=DPD2BIN[sourlo&0x3ff]; \ + if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \ + (buf)[1]+=DPD2BIN[(sourlo>>10)&0x3ff]; \ + if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \ + (buf)[2]+=DPD2BIN[(sourlo>>20)&0x3ff]; \ + if (buf[2]>999) {buf[2]-=1000; buf[3]++;} \ + sourhi=DFWORD(df, 0); \ + (buf)[3]+=DPD2BIN[((sourhi<<2) | (sourlo>>30))&0x3ff]; \ + if (buf[3]>999) {buf[3]-=1000; buf[4]++;} \ + (buf)[4]+=DPD2BIN[(sourhi>>8)&0x3ff]; \ + if (buf[4]>999) {buf[4]-=1000; buf[5]++;} \ + (buf)[5]+=DECCOMBMSD[sourhi>>26];} + + #elif DECPMAX==34 + #define ADDCOEFFTHOU(df, buf) { \ + uInt sourhi, sourmh, sourml, sourlo; \ + sourlo=DFWORD(df, 3); \ + (buf)[0]+=DPD2BIN[sourlo&0x3ff]; \ + if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \ + (buf)[1]+=DPD2BIN[(sourlo>>10)&0x3ff]; \ + if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \ + (buf)[2]+=DPD2BIN[(sourlo>>20)&0x3ff]; \ + if (buf[2]>999) {buf[2]-=1000; buf[3]++;} \ + sourml=DFWORD(df, 2); \ + (buf)[3]+=DPD2BIN[((sourml<<2) | (sourlo>>30))&0x3ff]; \ + if (buf[3]>999) {buf[3]-=1000; buf[4]++;} \ + (buf)[4]+=DPD2BIN[(sourml>>8)&0x3ff]; \ + if (buf[4]>999) {buf[4]-=1000; buf[5]++;} \ + (buf)[5]+=DPD2BIN[(sourml>>18)&0x3ff]; \ + if (buf[5]>999) {buf[5]-=1000; buf[6]++;} \ + sourmh=DFWORD(df, 1); \ + (buf)[6]+=DPD2BIN[((sourmh<<4) | (sourml>>28))&0x3ff]; \ + if (buf[6]>999) {buf[6]-=1000; buf[7]++;} \ + (buf)[7]+=DPD2BIN[(sourmh>>6)&0x3ff]; \ + if (buf[7]>999) {buf[7]-=1000; buf[8]++;} \ + (buf)[8]+=DPD2BIN[(sourmh>>16)&0x3ff]; \ + if (buf[8]>999) {buf[8]-=1000; buf[9]++;} \ + sourhi=DFWORD(df, 0); \ + (buf)[9]+=DPD2BIN[((sourhi<<6) | (sourmh>>26))&0x3ff]; \ + if (buf[9]>999) {buf[9]-=1000; buf[10]++;} \ + (buf)[10]+=DPD2BIN[(sourhi>>4)&0x3ff]; \ + if (buf[10]>999) {buf[10]-=1000; buf[11]++;} \ + (buf)[11]+=DECCOMBMSD[sourhi>>26];} + #endif + + + /* Set a decFloat to the maximum positive finite number (Nmax) */ + #if DECPMAX==7 + #define DFSETNMAX(df) \ + {DFWORD(df, 0)=0x77f3fcff;} + #elif DECPMAX==16 + #define DFSETNMAX(df) \ + {DFWORD(df, 0)=0x77fcff3f; \ + DFWORD(df, 1)=0xcff3fcff;} + #elif DECPMAX==34 + #define DFSETNMAX(df) \ + {DFWORD(df, 0)=0x77ffcff3; \ + DFWORD(df, 1)=0xfcff3fcf; \ + DFWORD(df, 2)=0xf3fcff3f; \ + DFWORD(df, 3)=0xcff3fcff;} + #endif + + /* [end of format-dependent macros and constants] */ + #endif + +#else + #error decNumberLocal included more than once +#endif diff --git a/intl/icu/source/i18n/decimfmt.cpp b/intl/icu/source/i18n/decimfmt.cpp new file mode 100644 index 0000000000..d33fac7089 --- /dev/null +++ b/intl/icu/source/i18n/decimfmt.cpp @@ -0,0 +1,1920 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include +#include +#include +#include "unicode/errorcode.h" +#include "unicode/decimfmt.h" +#include "number_decimalquantity.h" +#include "number_types.h" +#include "numparse_impl.h" +#include "number_mapper.h" +#include "number_patternstring.h" +#include "putilimp.h" +#include "number_utils.h" +#include "number_utypes.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using namespace icu::numparse; +using namespace icu::numparse::impl; +using ERoundingMode = icu::DecimalFormat::ERoundingMode; +using EPadPosition = icu::DecimalFormat::EPadPosition; + +// MSVC VS2015 warns C4805 when comparing bool with UBool, VS2017 no longer emits this warning. +// TODO: Move this macro into a better place? +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#define UBOOL_TO_BOOL(b) static_cast(b) +#else +#define UBOOL_TO_BOOL(b) b +#endif + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DecimalFormat) + + +DecimalFormat::DecimalFormat(UErrorCode& status) + : DecimalFormat(nullptr, status) { + if (U_FAILURE(status)) { return; } + // Use the default locale and decimal pattern. + const char* localeName = Locale::getDefault().getName(); + LocalPointer ns(NumberingSystem::createInstance(status)); + UnicodeString patternString = utils::getPatternForStyle( + localeName, + ns->getName(), + CLDR_PATTERN_STYLE_DECIMAL, + status); + setPropertiesFromPattern(patternString, IGNORE_ROUNDING_IF_CURRENCY, status); + touch(status); +} + +DecimalFormat::DecimalFormat(const UnicodeString& pattern, UErrorCode& status) + : DecimalFormat(nullptr, status) { + if (U_FAILURE(status)) { return; } + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_IF_CURRENCY, status); + touch(status); +} + +DecimalFormat::DecimalFormat(const UnicodeString& pattern, DecimalFormatSymbols* symbolsToAdopt, + UErrorCode& status) + : DecimalFormat(symbolsToAdopt, status) { + if (U_FAILURE(status)) { return; } + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_IF_CURRENCY, status); + touch(status); +} + +DecimalFormat::DecimalFormat(const UnicodeString& pattern, DecimalFormatSymbols* symbolsToAdopt, + UNumberFormatStyle style, UErrorCode& status) + : DecimalFormat(symbolsToAdopt, status) { + if (U_FAILURE(status)) { return; } + // If choice is a currency type, ignore the rounding information. + if (style == UNumberFormatStyle::UNUM_CURRENCY || + style == UNumberFormatStyle::UNUM_CURRENCY_ISO || + style == UNumberFormatStyle::UNUM_CURRENCY_ACCOUNTING || + style == UNumberFormatStyle::UNUM_CASH_CURRENCY || + style == UNumberFormatStyle::UNUM_CURRENCY_STANDARD || + style == UNumberFormatStyle::UNUM_CURRENCY_PLURAL) { + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_ALWAYS, status); + } else { + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_IF_CURRENCY, status); + } + // Note: in Java, CurrencyPluralInfo is set in NumberFormat.java, but in C++, it is not set there, + // so we have to set it here. + if (style == UNumberFormatStyle::UNUM_CURRENCY_PLURAL) { + LocalPointer cpi( + new CurrencyPluralInfo(fields->symbols->getLocale(), status), + status); + if (U_FAILURE(status)) { return; } + fields->properties.currencyPluralInfo.fPtr.adoptInstead(cpi.orphan()); + } + touch(status); +} + +DecimalFormat::DecimalFormat(const DecimalFormatSymbols* symbolsToAdopt, UErrorCode& status) { + // we must take ownership of symbolsToAdopt, even in a failure case. + LocalPointer adoptedSymbols(symbolsToAdopt); + if (U_FAILURE(status)) { + return; + } + fields = new DecimalFormatFields(); + if (fields == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (adoptedSymbols.isNull()) { + fields->symbols.adoptInsteadAndCheckErrorCode(new DecimalFormatSymbols(status), status); + } else { + fields->symbols.adoptInsteadAndCheckErrorCode(adoptedSymbols.orphan(), status); + } + if (U_FAILURE(status)) { + delete fields; + fields = nullptr; + } +} + +#if UCONFIG_HAVE_PARSEALLINPUT + +void DecimalFormat::setParseAllInput(UNumberFormatAttributeValue value) { + if (fields == nullptr) { return; } + if (value == fields->properties.parseAllInput) { return; } + fields->properties.parseAllInput = value; +} + +#endif + +DecimalFormat& +DecimalFormat::setAttribute(UNumberFormatAttribute attr, int32_t newValue, UErrorCode& status) { + if (U_FAILURE(status)) { return *this; } + + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + + switch (attr) { + case UNUM_LENIENT_PARSE: + setLenient(newValue != 0); + break; + + case UNUM_PARSE_INT_ONLY: + setParseIntegerOnly(newValue != 0); + break; + + case UNUM_GROUPING_USED: + setGroupingUsed(newValue != 0); + break; + + case UNUM_DECIMAL_ALWAYS_SHOWN: + setDecimalSeparatorAlwaysShown(newValue != 0); + break; + + case UNUM_MAX_INTEGER_DIGITS: + setMaximumIntegerDigits(newValue); + break; + + case UNUM_MIN_INTEGER_DIGITS: + setMinimumIntegerDigits(newValue); + break; + + case UNUM_INTEGER_DIGITS: + setMinimumIntegerDigits(newValue); + setMaximumIntegerDigits(newValue); + break; + + case UNUM_MAX_FRACTION_DIGITS: + setMaximumFractionDigits(newValue); + break; + + case UNUM_MIN_FRACTION_DIGITS: + setMinimumFractionDigits(newValue); + break; + + case UNUM_FRACTION_DIGITS: + setMinimumFractionDigits(newValue); + setMaximumFractionDigits(newValue); + break; + + case UNUM_SIGNIFICANT_DIGITS_USED: + setSignificantDigitsUsed(newValue != 0); + break; + + case UNUM_MAX_SIGNIFICANT_DIGITS: + setMaximumSignificantDigits(newValue); + break; + + case UNUM_MIN_SIGNIFICANT_DIGITS: + setMinimumSignificantDigits(newValue); + break; + + case UNUM_MULTIPLIER: + setMultiplier(newValue); + break; + + case UNUM_SCALE: + setMultiplierScale(newValue); + break; + + case UNUM_GROUPING_SIZE: + setGroupingSize(newValue); + break; + + case UNUM_ROUNDING_MODE: + setRoundingMode((DecimalFormat::ERoundingMode) newValue); + break; + + case UNUM_FORMAT_WIDTH: + setFormatWidth(newValue); + break; + + case UNUM_PADDING_POSITION: + /** The position at which padding will take place. */ + setPadPosition((DecimalFormat::EPadPosition) newValue); + break; + + case UNUM_SECONDARY_GROUPING_SIZE: + setSecondaryGroupingSize(newValue); + break; + +#if UCONFIG_HAVE_PARSEALLINPUT + case UNUM_PARSE_ALL_INPUT: + setParseAllInput((UNumberFormatAttributeValue) newValue); + break; +#endif + + case UNUM_PARSE_NO_EXPONENT: + setParseNoExponent((UBool) newValue); + break; + + case UNUM_PARSE_DECIMAL_MARK_REQUIRED: + setDecimalPatternMatchRequired((UBool) newValue); + break; + + case UNUM_CURRENCY_USAGE: + setCurrencyUsage((UCurrencyUsage) newValue, &status); + break; + + case UNUM_MINIMUM_GROUPING_DIGITS: + setMinimumGroupingDigits(newValue); + break; + + case UNUM_PARSE_CASE_SENSITIVE: + setParseCaseSensitive(static_cast(newValue)); + break; + + case UNUM_SIGN_ALWAYS_SHOWN: + setSignAlwaysShown(static_cast(newValue)); + break; + + case UNUM_FORMAT_FAIL_IF_MORE_THAN_MAX_DIGITS: + setFormatFailIfMoreThanMaxDigits(static_cast(newValue)); + break; + + default: + status = U_UNSUPPORTED_ERROR; + break; + } + return *this; +} + +int32_t DecimalFormat::getAttribute(UNumberFormatAttribute attr, UErrorCode& status) const { + if (U_FAILURE(status)) { return -1; } + + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return -1; + } + + switch (attr) { + case UNUM_LENIENT_PARSE: + return isLenient(); + + case UNUM_PARSE_INT_ONLY: + return isParseIntegerOnly(); + + case UNUM_GROUPING_USED: + return isGroupingUsed(); + + case UNUM_DECIMAL_ALWAYS_SHOWN: + return isDecimalSeparatorAlwaysShown(); + + case UNUM_MAX_INTEGER_DIGITS: + return getMaximumIntegerDigits(); + + case UNUM_MIN_INTEGER_DIGITS: + return getMinimumIntegerDigits(); + + case UNUM_INTEGER_DIGITS: + // TBD: what should this return? + return getMinimumIntegerDigits(); + + case UNUM_MAX_FRACTION_DIGITS: + return getMaximumFractionDigits(); + + case UNUM_MIN_FRACTION_DIGITS: + return getMinimumFractionDigits(); + + case UNUM_FRACTION_DIGITS: + // TBD: what should this return? + return getMinimumFractionDigits(); + + case UNUM_SIGNIFICANT_DIGITS_USED: + return areSignificantDigitsUsed(); + + case UNUM_MAX_SIGNIFICANT_DIGITS: + return getMaximumSignificantDigits(); + + case UNUM_MIN_SIGNIFICANT_DIGITS: + return getMinimumSignificantDigits(); + + case UNUM_MULTIPLIER: + return getMultiplier(); + + case UNUM_SCALE: + return getMultiplierScale(); + + case UNUM_GROUPING_SIZE: + return getGroupingSize(); + + case UNUM_ROUNDING_MODE: + return getRoundingMode(); + + case UNUM_FORMAT_WIDTH: + return getFormatWidth(); + + case UNUM_PADDING_POSITION: + return getPadPosition(); + + case UNUM_SECONDARY_GROUPING_SIZE: + return getSecondaryGroupingSize(); + + case UNUM_PARSE_NO_EXPONENT: + return isParseNoExponent(); + + case UNUM_PARSE_DECIMAL_MARK_REQUIRED: + return isDecimalPatternMatchRequired(); + + case UNUM_CURRENCY_USAGE: + return getCurrencyUsage(); + + case UNUM_MINIMUM_GROUPING_DIGITS: + return getMinimumGroupingDigits(); + + case UNUM_PARSE_CASE_SENSITIVE: + return isParseCaseSensitive(); + + case UNUM_SIGN_ALWAYS_SHOWN: + return isSignAlwaysShown(); + + case UNUM_FORMAT_FAIL_IF_MORE_THAN_MAX_DIGITS: + return isFormatFailIfMoreThanMaxDigits(); + + default: + status = U_UNSUPPORTED_ERROR; + break; + } + + return -1; /* undefined */ +} + +void DecimalFormat::setGroupingUsed(UBool enabled) { + if (fields == nullptr) { + return; + } + if (UBOOL_TO_BOOL(enabled) == fields->properties.groupingUsed) { return; } + NumberFormat::setGroupingUsed(enabled); // to set field for compatibility + fields->properties.groupingUsed = enabled; + touchNoError(); +} + +void DecimalFormat::setParseIntegerOnly(UBool value) { + if (fields == nullptr) { + return; + } + if (UBOOL_TO_BOOL(value) == fields->properties.parseIntegerOnly) { return; } + NumberFormat::setParseIntegerOnly(value); // to set field for compatibility + fields->properties.parseIntegerOnly = value; + touchNoError(); +} + +void DecimalFormat::setLenient(UBool enable) { + if (fields == nullptr) { + return; + } + ParseMode mode = enable ? PARSE_MODE_LENIENT : PARSE_MODE_STRICT; + if (!fields->properties.parseMode.isNull() && mode == fields->properties.parseMode.getNoError()) { return; } + NumberFormat::setLenient(enable); // to set field for compatibility + fields->properties.parseMode = mode; + touchNoError(); +} + +DecimalFormat::DecimalFormat(const UnicodeString& pattern, DecimalFormatSymbols* symbolsToAdopt, + UParseError&, UErrorCode& status) + : DecimalFormat(symbolsToAdopt, status) { + if (U_FAILURE(status)) { return; } + // TODO: What is parseError for? + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_IF_CURRENCY, status); + touch(status); +} + +DecimalFormat::DecimalFormat(const UnicodeString& pattern, const DecimalFormatSymbols& symbols, + UErrorCode& status) + : DecimalFormat(nullptr, status) { + if (U_FAILURE(status)) { return; } + LocalPointer dfs(new DecimalFormatSymbols(symbols), status); + if (U_FAILURE(status)) { + // If we failed to allocate DecimalFormatSymbols, then release fields and its members. + // We must have a fully complete fields object, we cannot have partially populated members. + delete fields; + fields = nullptr; + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + fields->symbols.adoptInstead(dfs.orphan()); + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_IF_CURRENCY, status); + touch(status); +} + +DecimalFormat::DecimalFormat(const DecimalFormat& source) : NumberFormat(source) { + // If the object that we are copying from is invalid, no point in going further. + if (source.fields == nullptr) { + return; + } + // Note: it is not safe to copy fields->formatter or fWarehouse directly because fields->formatter might have + // dangling pointers to fields inside fWarehouse. The safe thing is to re-construct fields->formatter from + // the property bag, despite being somewhat slower. + fields = new DecimalFormatFields(source.fields->properties); + if (fields == nullptr) { + return; // no way to report an error. + } + UErrorCode status = U_ZERO_ERROR; + fields->symbols.adoptInsteadAndCheckErrorCode(new DecimalFormatSymbols(*source.getDecimalFormatSymbols()), status); + // In order to simplify error handling logic in the various getters/setters/etc, we do not allow + // any partially populated DecimalFormatFields object. We must have a fully complete fields object + // or else we set it to nullptr. + if (U_FAILURE(status)) { + delete fields; + fields = nullptr; + return; + } + touch(status); +} + +DecimalFormat& DecimalFormat::operator=(const DecimalFormat& rhs) { + // guard against self-assignment + if (this == &rhs) { + return *this; + } + // Make sure both objects are valid. + if (fields == nullptr || rhs.fields == nullptr) { + return *this; // unfortunately, no way to report an error. + } + fields->properties = rhs.fields->properties; + fields->exportedProperties.clear(); + UErrorCode status = U_ZERO_ERROR; + LocalPointer dfs(new DecimalFormatSymbols(*rhs.getDecimalFormatSymbols()), status); + if (U_FAILURE(status)) { + // We failed to allocate DecimalFormatSymbols, release fields and its members. + // We must have a fully complete fields object, we cannot have partially populated members. + delete fields; + fields = nullptr; + return *this; + } + fields->symbols.adoptInstead(dfs.orphan()); + touch(status); + + return *this; +} + +DecimalFormat::~DecimalFormat() { + if (fields == nullptr) { return; } + +#ifndef __wasi__ + delete fields->atomicParser.exchange(nullptr); + delete fields->atomicCurrencyParser.exchange(nullptr); +#else + delete fields->atomicParser; + delete fields->atomicCurrencyParser; +#endif + delete fields; +} + +DecimalFormat* DecimalFormat::clone() const { + // can only clone valid objects. + if (fields == nullptr) { + return nullptr; + } + LocalPointer df(new DecimalFormat(*this)); + if (df.isValid() && df->fields != nullptr) { + return df.orphan(); + } + return nullptr; +} + +bool DecimalFormat::operator==(const Format& other) const { + auto* otherDF = dynamic_cast(&other); + if (otherDF == nullptr) { + return false; + } + // If either object is in an invalid state, prevent dereferencing nullptr below. + // Additionally, invalid objects should not be considered equal to anything. + if (fields == nullptr || otherDF->fields == nullptr) { + return false; + } + return fields->properties == otherDF->fields->properties && *getDecimalFormatSymbols() == *otherDF->getDecimalFormatSymbols(); +} + +UnicodeString& DecimalFormat::format(double number, UnicodeString& appendTo, FieldPosition& pos) const { + if (fields == nullptr) { + appendTo.setToBogus(); + return appendTo; + } + if (pos.getField() == FieldPosition::DONT_CARE && fastFormatDouble(number, appendTo)) { + return appendTo; + } + UErrorCode localStatus = U_ZERO_ERROR; + UFormattedNumberData output; + output.quantity.setToDouble(number); + fields->formatter.formatImpl(&output, localStatus); + fieldPositionHelper(output, pos, appendTo.length(), localStatus); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, localStatus); + return appendTo; +} + +UnicodeString& DecimalFormat::format(double number, UnicodeString& appendTo, FieldPosition& pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + if (pos.getField() == FieldPosition::DONT_CARE && fastFormatDouble(number, appendTo)) { + return appendTo; + } + UFormattedNumberData output; + output.quantity.setToDouble(number); + fields->formatter.formatImpl(&output, status); + fieldPositionHelper(output, pos, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& +DecimalFormat::format(double number, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + if (posIter == nullptr && fastFormatDouble(number, appendTo)) { + return appendTo; + } + UFormattedNumberData output; + output.quantity.setToDouble(number); + fields->formatter.formatImpl(&output, status); + fieldPositionIteratorHelper(output, posIter, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& DecimalFormat::format(int32_t number, UnicodeString& appendTo, FieldPosition& pos) const { + return format(static_cast (number), appendTo, pos); +} + +UnicodeString& DecimalFormat::format(int32_t number, UnicodeString& appendTo, FieldPosition& pos, + UErrorCode& status) const { + return format(static_cast (number), appendTo, pos, status); +} + +UnicodeString& +DecimalFormat::format(int32_t number, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const { + return format(static_cast (number), appendTo, posIter, status); +} + +UnicodeString& DecimalFormat::format(int64_t number, UnicodeString& appendTo, FieldPosition& pos) const { + if (fields == nullptr) { + appendTo.setToBogus(); + return appendTo; + } + if (pos.getField() == FieldPosition::DONT_CARE && fastFormatInt64(number, appendTo)) { + return appendTo; + } + UErrorCode localStatus = U_ZERO_ERROR; + UFormattedNumberData output; + output.quantity.setToLong(number); + fields->formatter.formatImpl(&output, localStatus); + fieldPositionHelper(output, pos, appendTo.length(), localStatus); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, localStatus); + return appendTo; +} + +UnicodeString& DecimalFormat::format(int64_t number, UnicodeString& appendTo, FieldPosition& pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + if (pos.getField() == FieldPosition::DONT_CARE && fastFormatInt64(number, appendTo)) { + return appendTo; + } + UFormattedNumberData output; + output.quantity.setToLong(number); + fields->formatter.formatImpl(&output, status); + fieldPositionHelper(output, pos, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& +DecimalFormat::format(int64_t number, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + if (posIter == nullptr && fastFormatInt64(number, appendTo)) { + return appendTo; + } + UFormattedNumberData output; + output.quantity.setToLong(number); + fields->formatter.formatImpl(&output, status); + fieldPositionIteratorHelper(output, posIter, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& +DecimalFormat::format(StringPiece number, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + UFormattedNumberData output; + output.quantity.setToDecNumber(number, status); + fields->formatter.formatImpl(&output, status); + fieldPositionIteratorHelper(output, posIter, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& DecimalFormat::format(const DecimalQuantity& number, UnicodeString& appendTo, + FieldPositionIterator* posIter, UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + UFormattedNumberData output; + output.quantity = number; + fields->formatter.formatImpl(&output, status); + fieldPositionIteratorHelper(output, posIter, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +UnicodeString& +DecimalFormat::format(const DecimalQuantity& number, UnicodeString& appendTo, FieldPosition& pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; // don't overwrite status if it's already a failure. + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + appendTo.setToBogus(); + return appendTo; + } + UFormattedNumberData output; + output.quantity = number; + fields->formatter.formatImpl(&output, status); + fieldPositionHelper(output, pos, appendTo.length(), status); + auto appendable = UnicodeStringAppendable(appendTo); + output.appendTo(appendable, status); + return appendTo; +} + +void DecimalFormat::parse(const UnicodeString& text, Formattable& output, + ParsePosition& parsePosition) const { + if (fields == nullptr) { + return; + } + if (parsePosition.getIndex() < 0 || parsePosition.getIndex() >= text.length()) { + if (parsePosition.getIndex() == text.length()) { + // If there is nothing to parse, it is an error + parsePosition.setErrorIndex(parsePosition.getIndex()); + } + return; + } + + ErrorCode status; + ParsedNumber result; + // Note: if this is a currency instance, currencies will be matched despite the fact that we are not in the + // parseCurrency method (backwards compatibility) + int32_t startIndex = parsePosition.getIndex(); + const NumberParserImpl* parser = getParser(status); + if (U_FAILURE(status)) { + return; // unfortunately no way to report back the error. + } + parser->parse(text, startIndex, true, result, status); + if (U_FAILURE(status)) { + return; // unfortunately no way to report back the error. + } + // TODO: Do we need to check for fImpl->properties->parseAllInput (UCONFIG_HAVE_PARSEALLINPUT) here? + if (result.success()) { + parsePosition.setIndex(result.charEnd); + result.populateFormattable(output, parser->getParseFlags()); + } else { + parsePosition.setErrorIndex(startIndex + result.charEnd); + } +} + +CurrencyAmount* DecimalFormat::parseCurrency(const UnicodeString& text, ParsePosition& parsePosition) const { + if (fields == nullptr) { + return nullptr; + } + if (parsePosition.getIndex() < 0 || parsePosition.getIndex() >= text.length()) { + return nullptr; + } + + ErrorCode status; + ParsedNumber result; + // Note: if this is a currency instance, currencies will be matched despite the fact that we are not in the + // parseCurrency method (backwards compatibility) + int32_t startIndex = parsePosition.getIndex(); + const NumberParserImpl* parser = getCurrencyParser(status); + if (U_FAILURE(status)) { + return nullptr; + } + parser->parse(text, startIndex, true, result, status); + if (U_FAILURE(status)) { + return nullptr; + } + // TODO: Do we need to check for fImpl->properties->parseAllInput (UCONFIG_HAVE_PARSEALLINPUT) here? + if (result.success()) { + parsePosition.setIndex(result.charEnd); + Formattable formattable; + result.populateFormattable(formattable, parser->getParseFlags()); + LocalPointer currencyAmount( + new CurrencyAmount(formattable, result.currencyCode, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + return currencyAmount.orphan(); + } else { + parsePosition.setErrorIndex(startIndex + result.charEnd); + return nullptr; + } +} + +const DecimalFormatSymbols* DecimalFormat::getDecimalFormatSymbols() const { + if (fields == nullptr) { + return nullptr; + } + if (!fields->symbols.isNull()) { + return fields->symbols.getAlias(); + } else { + return fields->formatter.getDecimalFormatSymbols(); + } +} + +void DecimalFormat::adoptDecimalFormatSymbols(DecimalFormatSymbols* symbolsToAdopt) { + if (symbolsToAdopt == nullptr) { + return; // do not allow caller to set fields->symbols to nullptr + } + // we must take ownership of symbolsToAdopt, even in a failure case. + LocalPointer dfs(symbolsToAdopt); + if (fields == nullptr) { + return; + } + fields->symbols.adoptInstead(dfs.orphan()); + touchNoError(); +} + +void DecimalFormat::setDecimalFormatSymbols(const DecimalFormatSymbols& symbols) { + if (fields == nullptr) { + return; + } + UErrorCode status = U_ZERO_ERROR; + LocalPointer dfs(new DecimalFormatSymbols(symbols), status); + if (U_FAILURE(status)) { + // We failed to allocate DecimalFormatSymbols, release fields and its members. + // We must have a fully complete fields object, we cannot have partially populated members. + delete fields; + fields = nullptr; + return; + } + fields->symbols.adoptInstead(dfs.orphan()); + touchNoError(); +} + +const CurrencyPluralInfo* DecimalFormat::getCurrencyPluralInfo() const { + if (fields == nullptr) { + return nullptr; + } + return fields->properties.currencyPluralInfo.fPtr.getAlias(); +} + +void DecimalFormat::adoptCurrencyPluralInfo(CurrencyPluralInfo* toAdopt) { + // TODO: should we guard against nullptr input, like in adoptDecimalFormatSymbols? + // we must take ownership of toAdopt, even in a failure case. + LocalPointer cpi(toAdopt); + if (fields == nullptr) { + return; + } + fields->properties.currencyPluralInfo.fPtr.adoptInstead(cpi.orphan()); + touchNoError(); +} + +void DecimalFormat::setCurrencyPluralInfo(const CurrencyPluralInfo& info) { + if (fields == nullptr) { + return; + } + if (fields->properties.currencyPluralInfo.fPtr.isNull()) { + // Note: clone() can fail with OOM error, but we have no way to report it. :( + fields->properties.currencyPluralInfo.fPtr.adoptInstead(info.clone()); + } else { + *fields->properties.currencyPluralInfo.fPtr = info; // copy-assignment operator + } + touchNoError(); +} + +UnicodeString& DecimalFormat::getPositivePrefix(UnicodeString& result) const { + if (fields == nullptr) { + result.setToBogus(); + return result; + } + UErrorCode status = U_ZERO_ERROR; + fields->formatter.getAffixImpl(true, false, result, status); + if (U_FAILURE(status)) { result.setToBogus(); } + return result; +} + +void DecimalFormat::setPositivePrefix(const UnicodeString& newValue) { + if (fields == nullptr) { + return; + } + if (newValue == fields->properties.positivePrefix) { return; } + fields->properties.positivePrefix = newValue; + touchNoError(); +} + +UnicodeString& DecimalFormat::getNegativePrefix(UnicodeString& result) const { + if (fields == nullptr) { + result.setToBogus(); + return result; + } + UErrorCode status = U_ZERO_ERROR; + fields->formatter.getAffixImpl(true, true, result, status); + if (U_FAILURE(status)) { result.setToBogus(); } + return result; +} + +void DecimalFormat::setNegativePrefix(const UnicodeString& newValue) { + if (fields == nullptr) { + return; + } + if (newValue == fields->properties.negativePrefix) { return; } + fields->properties.negativePrefix = newValue; + touchNoError(); +} + +UnicodeString& DecimalFormat::getPositiveSuffix(UnicodeString& result) const { + if (fields == nullptr) { + result.setToBogus(); + return result; + } + UErrorCode status = U_ZERO_ERROR; + fields->formatter.getAffixImpl(false, false, result, status); + if (U_FAILURE(status)) { result.setToBogus(); } + return result; +} + +void DecimalFormat::setPositiveSuffix(const UnicodeString& newValue) { + if (fields == nullptr) { + return; + } + if (newValue == fields->properties.positiveSuffix) { return; } + fields->properties.positiveSuffix = newValue; + touchNoError(); +} + +UnicodeString& DecimalFormat::getNegativeSuffix(UnicodeString& result) const { + if (fields == nullptr) { + result.setToBogus(); + return result; + } + UErrorCode status = U_ZERO_ERROR; + fields->formatter.getAffixImpl(false, true, result, status); + if (U_FAILURE(status)) { result.setToBogus(); } + return result; +} + +void DecimalFormat::setNegativeSuffix(const UnicodeString& newValue) { + if (fields == nullptr) { + return; + } + if (newValue == fields->properties.negativeSuffix) { return; } + fields->properties.negativeSuffix = newValue; + touchNoError(); +} + +UBool DecimalFormat::isSignAlwaysShown() const { + // Not much we can do to report an error. + if (fields == nullptr) { + return DecimalFormatProperties::getDefault().signAlwaysShown; + } + return fields->properties.signAlwaysShown; +} + +void DecimalFormat::setSignAlwaysShown(UBool value) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(value) == fields->properties.signAlwaysShown) { return; } + fields->properties.signAlwaysShown = value; + touchNoError(); +} + +int32_t DecimalFormat::getMultiplier() const { + const DecimalFormatProperties *dfp; + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + dfp = &(DecimalFormatProperties::getDefault()); + } else { + dfp = &fields->properties; + } + if (dfp->multiplier != 1) { + return dfp->multiplier; + } else if (dfp->magnitudeMultiplier != 0) { + return static_cast(uprv_pow10(dfp->magnitudeMultiplier)); + } else { + return 1; + } +} + +void DecimalFormat::setMultiplier(int32_t multiplier) { + if (fields == nullptr) { + return; + } + if (multiplier == 0) { + multiplier = 1; // one being the benign default value for a multiplier. + } + + // Try to convert to a magnitude multiplier first + int delta = 0; + int value = multiplier; + while (value != 1) { + delta++; + int temp = value / 10; + if (temp * 10 != value) { + delta = -1; + break; + } + value = temp; + } + if (delta != -1) { + fields->properties.magnitudeMultiplier = delta; + fields->properties.multiplier = 1; + } else { + fields->properties.magnitudeMultiplier = 0; + fields->properties.multiplier = multiplier; + } + touchNoError(); +} + +int32_t DecimalFormat::getMultiplierScale() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().multiplierScale; + } + return fields->properties.multiplierScale; +} + +void DecimalFormat::setMultiplierScale(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.multiplierScale) { return; } + fields->properties.multiplierScale = newValue; + touchNoError(); +} + +double DecimalFormat::getRoundingIncrement() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().roundingIncrement; + } + return fields->exportedProperties.roundingIncrement; +} + +void DecimalFormat::setRoundingIncrement(double newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.roundingIncrement) { return; } + fields->properties.roundingIncrement = newValue; + touchNoError(); +} + +ERoundingMode DecimalFormat::getRoundingMode() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return static_cast(DecimalFormatProperties::getDefault().roundingMode.getNoError()); + } + // UNumberFormatRoundingMode and ERoundingMode have the same values. + return static_cast(fields->exportedProperties.roundingMode.getNoError()); +} + +void DecimalFormat::setRoundingMode(ERoundingMode roundingMode) UPRV_NO_SANITIZE_UNDEFINED { + if (fields == nullptr) { return; } + auto uRoundingMode = static_cast(roundingMode); + if (!fields->properties.roundingMode.isNull() && uRoundingMode == fields->properties.roundingMode.getNoError()) { + return; + } + NumberFormat::setMaximumIntegerDigits(roundingMode); // to set field for compatibility + fields->properties.roundingMode = uRoundingMode; + touchNoError(); +} + +int32_t DecimalFormat::getFormatWidth() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().formatWidth; + } + return fields->properties.formatWidth; +} + +void DecimalFormat::setFormatWidth(int32_t width) { + if (fields == nullptr) { return; } + if (width == fields->properties.formatWidth) { return; } + fields->properties.formatWidth = width; + touchNoError(); +} + +UnicodeString DecimalFormat::getPadCharacterString() const { + if (fields == nullptr || fields->properties.padString.isBogus()) { + // Readonly-alias the static string kFallbackPaddingString + return {true, kFallbackPaddingString, -1}; + } else { + return fields->properties.padString; + } +} + +void DecimalFormat::setPadCharacter(const UnicodeString& padChar) { + if (fields == nullptr) { return; } + if (padChar == fields->properties.padString) { return; } + if (padChar.length() > 0) { + fields->properties.padString = UnicodeString(padChar.char32At(0)); + } else { + fields->properties.padString.setToBogus(); + } + touchNoError(); +} + +EPadPosition DecimalFormat::getPadPosition() const { + if (fields == nullptr || fields->properties.padPosition.isNull()) { + return EPadPosition::kPadBeforePrefix; + } else { + // UNumberFormatPadPosition and EPadPosition have the same values. + return static_cast(fields->properties.padPosition.getNoError()); + } +} + +void DecimalFormat::setPadPosition(EPadPosition padPos) { + if (fields == nullptr) { return; } + auto uPadPos = static_cast(padPos); + if (!fields->properties.padPosition.isNull() && uPadPos == fields->properties.padPosition.getNoError()) { + return; + } + fields->properties.padPosition = uPadPos; + touchNoError(); +} + +UBool DecimalFormat::isScientificNotation() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return (DecimalFormatProperties::getDefault().minimumExponentDigits != -1); + } + return (fields->properties.minimumExponentDigits != -1); +} + +void DecimalFormat::setScientificNotation(UBool useScientific) { + if (fields == nullptr) { return; } + int32_t minExp = useScientific ? 1 : -1; + if (fields->properties.minimumExponentDigits == minExp) { return; } + if (useScientific) { + fields->properties.minimumExponentDigits = 1; + } else { + fields->properties.minimumExponentDigits = -1; + } + touchNoError(); +} + +int8_t DecimalFormat::getMinimumExponentDigits() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return static_cast(DecimalFormatProperties::getDefault().minimumExponentDigits); + } + return static_cast(fields->properties.minimumExponentDigits); +} + +void DecimalFormat::setMinimumExponentDigits(int8_t minExpDig) { + if (fields == nullptr) { return; } + if (minExpDig == fields->properties.minimumExponentDigits) { return; } + fields->properties.minimumExponentDigits = minExpDig; + touchNoError(); +} + +UBool DecimalFormat::isExponentSignAlwaysShown() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().exponentSignAlwaysShown; + } + return fields->properties.exponentSignAlwaysShown; +} + +void DecimalFormat::setExponentSignAlwaysShown(UBool expSignAlways) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(expSignAlways) == fields->properties.exponentSignAlwaysShown) { return; } + fields->properties.exponentSignAlwaysShown = expSignAlways; + touchNoError(); +} + +int32_t DecimalFormat::getGroupingSize() const { + int32_t groupingSize; + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + groupingSize = DecimalFormatProperties::getDefault().groupingSize; + } else { + groupingSize = fields->properties.groupingSize; + } + if (groupingSize < 0) { + return 0; + } + return groupingSize; +} + +void DecimalFormat::setGroupingSize(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.groupingSize) { return; } + fields->properties.groupingSize = newValue; + touchNoError(); +} + +int32_t DecimalFormat::getSecondaryGroupingSize() const { + int32_t grouping2; + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + grouping2 = DecimalFormatProperties::getDefault().secondaryGroupingSize; + } else { + grouping2 = fields->properties.secondaryGroupingSize; + } + if (grouping2 < 0) { + return 0; + } + return grouping2; +} + +void DecimalFormat::setSecondaryGroupingSize(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.secondaryGroupingSize) { return; } + fields->properties.secondaryGroupingSize = newValue; + touchNoError(); +} + +int32_t DecimalFormat::getMinimumGroupingDigits() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().minimumGroupingDigits; + } + return fields->properties.minimumGroupingDigits; +} + +void DecimalFormat::setMinimumGroupingDigits(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.minimumGroupingDigits) { return; } + fields->properties.minimumGroupingDigits = newValue; + touchNoError(); +} + +UBool DecimalFormat::isDecimalSeparatorAlwaysShown() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().decimalSeparatorAlwaysShown; + } + return fields->properties.decimalSeparatorAlwaysShown; +} + +void DecimalFormat::setDecimalSeparatorAlwaysShown(UBool newValue) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(newValue) == fields->properties.decimalSeparatorAlwaysShown) { return; } + fields->properties.decimalSeparatorAlwaysShown = newValue; + touchNoError(); +} + +UBool DecimalFormat::isDecimalPatternMatchRequired() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().decimalPatternMatchRequired; + } + return fields->properties.decimalPatternMatchRequired; +} + +void DecimalFormat::setDecimalPatternMatchRequired(UBool newValue) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(newValue) == fields->properties.decimalPatternMatchRequired) { return; } + fields->properties.decimalPatternMatchRequired = newValue; + touchNoError(); +} + +UBool DecimalFormat::isParseNoExponent() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().parseNoExponent; + } + return fields->properties.parseNoExponent; +} + +void DecimalFormat::setParseNoExponent(UBool value) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(value) == fields->properties.parseNoExponent) { return; } + fields->properties.parseNoExponent = value; + touchNoError(); +} + +UBool DecimalFormat::isParseCaseSensitive() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().parseCaseSensitive; + } + return fields->properties.parseCaseSensitive; +} + +void DecimalFormat::setParseCaseSensitive(UBool value) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(value) == fields->properties.parseCaseSensitive) { return; } + fields->properties.parseCaseSensitive = value; + touchNoError(); +} + +UBool DecimalFormat::isFormatFailIfMoreThanMaxDigits() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().formatFailIfMoreThanMaxDigits; + } + return fields->properties.formatFailIfMoreThanMaxDigits; +} + +void DecimalFormat::setFormatFailIfMoreThanMaxDigits(UBool value) { + if (fields == nullptr) { return; } + if (UBOOL_TO_BOOL(value) == fields->properties.formatFailIfMoreThanMaxDigits) { return; } + fields->properties.formatFailIfMoreThanMaxDigits = value; + touchNoError(); +} + +UnicodeString& DecimalFormat::toPattern(UnicodeString& result) const { + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + result.setToBogus(); + return result; + } + // Pull some properties from exportedProperties and others from properties + // to keep affix patterns intact. In particular, pull rounding properties + // so that CurrencyUsage is reflected properly. + // TODO: Consider putting this logic in number_patternstring.cpp instead. + ErrorCode localStatus; + DecimalFormatProperties tprops(fields->properties); + bool useCurrency = ( + !tprops.currency.isNull() || + !tprops.currencyPluralInfo.fPtr.isNull() || + !tprops.currencyUsage.isNull() || + tprops.currencyAsDecimal || + AffixUtils::hasCurrencySymbols(tprops.positivePrefixPattern, localStatus) || + AffixUtils::hasCurrencySymbols(tprops.positiveSuffixPattern, localStatus) || + AffixUtils::hasCurrencySymbols(tprops.negativePrefixPattern, localStatus) || + AffixUtils::hasCurrencySymbols(tprops.negativeSuffixPattern, localStatus)); + if (useCurrency) { + tprops.minimumFractionDigits = fields->exportedProperties.minimumFractionDigits; + tprops.maximumFractionDigits = fields->exportedProperties.maximumFractionDigits; + tprops.roundingIncrement = fields->exportedProperties.roundingIncrement; + } + result = PatternStringUtils::propertiesToPatternString(tprops, localStatus); + return result; +} + +UnicodeString& DecimalFormat::toLocalizedPattern(UnicodeString& result) const { + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + result.setToBogus(); + return result; + } + ErrorCode localStatus; + result = toPattern(result); + result = PatternStringUtils::convertLocalized(result, *getDecimalFormatSymbols(), true, localStatus); + return result; +} + +void DecimalFormat::applyPattern(const UnicodeString& pattern, UParseError&, UErrorCode& status) { + // TODO: What is parseError for? + applyPattern(pattern, status); +} + +void DecimalFormat::applyPattern(const UnicodeString& pattern, UErrorCode& status) { + // don't overwrite status if it's already a failure. + if (U_FAILURE(status)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + setPropertiesFromPattern(pattern, IGNORE_ROUNDING_NEVER, status); + touch(status); +} + +void DecimalFormat::applyLocalizedPattern(const UnicodeString& localizedPattern, UParseError&, + UErrorCode& status) { + // TODO: What is parseError for? + applyLocalizedPattern(localizedPattern, status); +} + +void DecimalFormat::applyLocalizedPattern(const UnicodeString& localizedPattern, UErrorCode& status) { + // don't overwrite status if it's already a failure. + if (U_FAILURE(status)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + UnicodeString pattern = PatternStringUtils::convertLocalized( + localizedPattern, *getDecimalFormatSymbols(), false, status); + applyPattern(pattern, status); +} + +void DecimalFormat::setMaximumIntegerDigits(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.maximumIntegerDigits) { return; } + // For backwards compatibility, conflicting min/max need to keep the most recent setting. + int32_t min = fields->properties.minimumIntegerDigits; + if (min >= 0 && min > newValue) { + fields->properties.minimumIntegerDigits = newValue; + } + fields->properties.maximumIntegerDigits = newValue; + touchNoError(); +} + +void DecimalFormat::setMinimumIntegerDigits(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.minimumIntegerDigits) { return; } + // For backwards compatibility, conflicting min/max need to keep the most recent setting. + int32_t max = fields->properties.maximumIntegerDigits; + if (max >= 0 && max < newValue) { + fields->properties.maximumIntegerDigits = newValue; + } + fields->properties.minimumIntegerDigits = newValue; + touchNoError(); +} + +void DecimalFormat::setMaximumFractionDigits(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.maximumFractionDigits) { return; } + // cap for backward compatibility, formerly 340, now 999 + if (newValue > kMaxIntFracSig) { + newValue = kMaxIntFracSig; + } + // For backwards compatibility, conflicting min/max need to keep the most recent setting. + int32_t min = fields->properties.minimumFractionDigits; + if (min >= 0 && min > newValue) { + fields->properties.minimumFractionDigits = newValue; + } + fields->properties.maximumFractionDigits = newValue; + touchNoError(); +} + +void DecimalFormat::setMinimumFractionDigits(int32_t newValue) { + if (fields == nullptr) { return; } + if (newValue == fields->properties.minimumFractionDigits) { return; } + // For backwards compatibility, conflicting min/max need to keep the most recent setting. + int32_t max = fields->properties.maximumFractionDigits; + if (max >= 0 && max < newValue) { + fields->properties.maximumFractionDigits = newValue; + } + fields->properties.minimumFractionDigits = newValue; + touchNoError(); +} + +int32_t DecimalFormat::getMinimumSignificantDigits() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().minimumSignificantDigits; + } + return fields->exportedProperties.minimumSignificantDigits; +} + +int32_t DecimalFormat::getMaximumSignificantDigits() const { + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + return DecimalFormatProperties::getDefault().maximumSignificantDigits; + } + return fields->exportedProperties.maximumSignificantDigits; +} + +void DecimalFormat::setMinimumSignificantDigits(int32_t value) { + if (fields == nullptr) { return; } + if (value == fields->properties.minimumSignificantDigits) { return; } + int32_t max = fields->properties.maximumSignificantDigits; + if (max >= 0 && max < value) { + fields->properties.maximumSignificantDigits = value; + } + fields->properties.minimumSignificantDigits = value; + touchNoError(); +} + +void DecimalFormat::setMaximumSignificantDigits(int32_t value) { + if (fields == nullptr) { return; } + if (value == fields->properties.maximumSignificantDigits) { return; } + int32_t min = fields->properties.minimumSignificantDigits; + if (min >= 0 && min > value) { + fields->properties.minimumSignificantDigits = value; + } + fields->properties.maximumSignificantDigits = value; + touchNoError(); +} + +UBool DecimalFormat::areSignificantDigitsUsed() const { + const DecimalFormatProperties* dfp; + // Not much we can do to report an error. + if (fields == nullptr) { + // Fallback to using the default instance of DecimalFormatProperties. + dfp = &(DecimalFormatProperties::getDefault()); + } else { + dfp = &fields->properties; + } + return dfp->minimumSignificantDigits != -1 || dfp->maximumSignificantDigits != -1; +} + +void DecimalFormat::setSignificantDigitsUsed(UBool useSignificantDigits) { + if (fields == nullptr) { return; } + + // These are the default values from the old implementation. + if (useSignificantDigits) { + if (fields->properties.minimumSignificantDigits != -1 || + fields->properties.maximumSignificantDigits != -1) { + return; + } + } else { + if (fields->properties.minimumSignificantDigits == -1 && + fields->properties.maximumSignificantDigits == -1) { + return; + } + } + int32_t minSig = useSignificantDigits ? 1 : -1; + int32_t maxSig = useSignificantDigits ? 6 : -1; + fields->properties.minimumSignificantDigits = minSig; + fields->properties.maximumSignificantDigits = maxSig; + touchNoError(); +} + +void DecimalFormat::setCurrency(const char16_t* theCurrency, UErrorCode& ec) { + // don't overwrite ec if it's already a failure. + if (U_FAILURE(ec)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + ec = U_MEMORY_ALLOCATION_ERROR; + return; + } + CurrencyUnit currencyUnit(theCurrency, ec); + if (U_FAILURE(ec)) { return; } + if (!fields->properties.currency.isNull() && fields->properties.currency.getNoError() == currencyUnit) { + return; + } + NumberFormat::setCurrency(theCurrency, ec); // to set field for compatibility + fields->properties.currency = currencyUnit; + // In Java, the DecimalFormatSymbols is mutable. Why not in C++? + LocalPointer newSymbols(new DecimalFormatSymbols(*getDecimalFormatSymbols()), ec); + newSymbols->setCurrency(currencyUnit.getISOCurrency(), ec); + fields->symbols.adoptInsteadAndCheckErrorCode(newSymbols.orphan(), ec); + touch(ec); +} + +void DecimalFormat::setCurrency(const char16_t* theCurrency) { + ErrorCode localStatus; + setCurrency(theCurrency, localStatus); +} + +void DecimalFormat::setCurrencyUsage(UCurrencyUsage newUsage, UErrorCode* ec) { + // don't overwrite ec if it's already a failure. + if (U_FAILURE(*ec)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + *ec = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (!fields->properties.currencyUsage.isNull() && newUsage == fields->properties.currencyUsage.getNoError()) { + return; + } + fields->properties.currencyUsage = newUsage; + touch(*ec); +} + +UCurrencyUsage DecimalFormat::getCurrencyUsage() const { + // CurrencyUsage is not exported, so we have to get it from the input property bag. + // TODO: Should we export CurrencyUsage instead? + if (fields == nullptr || fields->properties.currencyUsage.isNull()) { + return UCURR_USAGE_STANDARD; + } + return fields->properties.currencyUsage.getNoError(); +} + +void +DecimalFormat::formatToDecimalQuantity(double number, DecimalQuantity& output, UErrorCode& status) const { + // don't overwrite status if it's already a failure. + if (U_FAILURE(status)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + fields->formatter.formatDouble(number, status).getDecimalQuantity(output, status); +} + +void DecimalFormat::formatToDecimalQuantity(const Formattable& number, DecimalQuantity& output, + UErrorCode& status) const { + // don't overwrite status if it's already a failure. + if (U_FAILURE(status)) { return; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + UFormattedNumberData obj; + number.populateDecimalQuantity(obj.quantity, status); + fields->formatter.formatImpl(&obj, status); + output = std::move(obj.quantity); +} + +const number::LocalizedNumberFormatter* DecimalFormat::toNumberFormatter(UErrorCode& status) const { + // We sometimes need to return nullptr here (see ICU-20380) + if (U_FAILURE(status)) { return nullptr; } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return &fields->formatter; +} + +/** Rebuilds the formatter object from the property bag. */ +void DecimalFormat::touch(UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fields == nullptr) { + // We only get here if an OOM error happened during construction, copy construction, assignment, or modification. + // For regular construction, the caller should have checked the status variable for errors. + // For copy construction, there is unfortunately nothing to report the error, so we need to guard against + // this possible bad state here and set the status to an error. + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + // In C++, fields->symbols (or, if it's null, the DecimalFormatSymbols owned by the underlying LocalizedNumberFormatter) + // is the source of truth for the locale. + const DecimalFormatSymbols* symbols = getDecimalFormatSymbols(); + Locale locale = symbols->getLocale(); + + // Note: The formatter is relatively cheap to create, and we need it to populate fields->exportedProperties, + // so automatically recompute it here. The parser is a bit more expensive and is not needed until the + // parse method is called, so defer that until needed. + // TODO: Only update the pieces that changed instead of re-computing the whole formatter? + + // Since memory has already been allocated for the formatter, we can move assign a stack-allocated object + // and don't need to call new. (Which is slower and could possibly fail). + // [Note that "symbols" above might point to the DecimalFormatSymbols object owned by fields->formatter. + // That's okay, because NumberPropertyMapper::create() will clone it before fields->formatter's assignment + // operator deletes it. But it does mean that "symbols" can't be counted on to be good after this line.] + fields->formatter = NumberPropertyMapper::create( + fields->properties, *symbols, fields->warehouse, fields->exportedProperties, status + ).locale(locale); + fields->symbols.adoptInstead(nullptr); // the fields->symbols property is only temporary, until we can copy it into a new LocalizedNumberFormatter + + // Do this after fields->exportedProperties are set up + setupFastFormat(); + + // Delete the parsers if they were made previously +#ifndef __wasi__ + delete fields->atomicParser.exchange(nullptr); + delete fields->atomicCurrencyParser.exchange(nullptr); +#else + delete fields->atomicParser; + delete fields->atomicCurrencyParser; +#endif + + // In order for the getters to work, we need to populate some fields in NumberFormat. + NumberFormat::setCurrency(fields->exportedProperties.currency.get(status).getISOCurrency(), status); + NumberFormat::setMaximumIntegerDigits(fields->exportedProperties.maximumIntegerDigits); + NumberFormat::setMinimumIntegerDigits(fields->exportedProperties.minimumIntegerDigits); + NumberFormat::setMaximumFractionDigits(fields->exportedProperties.maximumFractionDigits); + NumberFormat::setMinimumFractionDigits(fields->exportedProperties.minimumFractionDigits); + // fImpl->properties, not fields->exportedProperties, since this information comes from the pattern: + NumberFormat::setGroupingUsed(fields->properties.groupingUsed); +} + +void DecimalFormat::touchNoError() { + UErrorCode localStatus = U_ZERO_ERROR; + touch(localStatus); +} + +void DecimalFormat::setPropertiesFromPattern(const UnicodeString& pattern, int32_t ignoreRounding, + UErrorCode& status) { + if (U_SUCCESS(status)) { + // Cast workaround to get around putting the enum in the public header file + auto actualIgnoreRounding = static_cast(ignoreRounding); + PatternParser::parseToExistingProperties(pattern, fields->properties, actualIgnoreRounding, status); + } +} + +const numparse::impl::NumberParserImpl* DecimalFormat::getParser(UErrorCode& status) const { + // TODO: Move this into umutex.h? (similar logic also in numrange_fluent.cpp) + // See ICU-20146 + + if (U_FAILURE(status)) { + return nullptr; + } + + // First try to get the pre-computed parser +#ifndef __wasi__ + auto* ptr = fields->atomicParser.load(); +#else + auto* ptr = fields->atomicParser; +#endif + if (ptr != nullptr) { + return ptr; + } + + // Try computing the parser on our own + auto* temp = NumberParserImpl::createParserFromProperties(fields->properties, *getDecimalFormatSymbols(), false, status); + if (U_FAILURE(status)) { + return nullptr; + } + if (temp == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + + // Note: ptr starts as nullptr; during compare_exchange, + // it is set to what is actually stored in the atomic + // if another thread beat us to computing the parser object. + auto* nonConstThis = const_cast(this); +#ifndef __wasi__ + if (!nonConstThis->fields->atomicParser.compare_exchange_strong(ptr, temp)) { + // Another thread beat us to computing the parser + delete temp; + return ptr; + } else { + // Our copy of the parser got stored in the atomic + return temp; + } +#else + nonConstThis->fields->atomicParser = temp; + return temp; +#endif +} + +const numparse::impl::NumberParserImpl* DecimalFormat::getCurrencyParser(UErrorCode& status) const { + if (U_FAILURE(status)) { return nullptr; } + + // First try to get the pre-computed parser +#ifndef __wasi__ + auto* ptr = fields->atomicCurrencyParser.load(); +#else + auto* ptr = fields->atomicCurrencyParser; +#endif + if (ptr != nullptr) { + return ptr; + } + + // Try computing the parser on our own + auto* temp = NumberParserImpl::createParserFromProperties(fields->properties, *getDecimalFormatSymbols(), true, status); + if (temp == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + // although we may still dereference, call sites should be guarded + } + + // Note: ptr starts as nullptr; during compare_exchange, it is set to what is actually stored in the + // atomic if another thread beat us to computing the parser object. + auto* nonConstThis = const_cast(this); +#ifndef __wasi__ + if (!nonConstThis->fields->atomicCurrencyParser.compare_exchange_strong(ptr, temp)) { + // Another thread beat us to computing the parser + delete temp; + return ptr; + } else { + // Our copy of the parser got stored in the atomic + return temp; + } +#else + nonConstThis->fields->atomicCurrencyParser = temp; + return temp; +#endif +} + +void +DecimalFormat::fieldPositionHelper( + const UFormattedNumberData& formatted, + FieldPosition& fieldPosition, + int32_t offset, + UErrorCode& status) { + if (U_FAILURE(status)) { return; } + // always return first occurrence: + fieldPosition.setBeginIndex(0); + fieldPosition.setEndIndex(0); + bool found = formatted.nextFieldPosition(fieldPosition, status); + if (found && offset != 0) { + FieldPositionOnlyHandler fpoh(fieldPosition); + fpoh.shiftLast(offset); + } +} + +void +DecimalFormat::fieldPositionIteratorHelper( + const UFormattedNumberData& formatted, + FieldPositionIterator* fpi, + int32_t offset, + UErrorCode& status) { + if (U_SUCCESS(status) && (fpi != nullptr)) { + FieldPositionIteratorHandler fpih(fpi, status); + fpih.setShift(offset); + formatted.getAllFieldPositions(fpih, status); + } +} + +// To debug fast-format, change void(x) to printf(x) +#define trace(x) void(x) + +void DecimalFormat::setupFastFormat() { + // Check the majority of properties: + if (!fields->properties.equalsDefaultExceptFastFormat()) { + trace("no fast format: equality\n"); + fields->canUseFastFormat = false; + return; + } + + // Now check the remaining properties. + // Nontrivial affixes: + UBool trivialPP = fields->properties.positivePrefixPattern.isEmpty(); + UBool trivialPS = fields->properties.positiveSuffixPattern.isEmpty(); + UBool trivialNP = fields->properties.negativePrefixPattern.isBogus() || ( + fields->properties.negativePrefixPattern.length() == 1 && + fields->properties.negativePrefixPattern.charAt(0) == u'-'); + UBool trivialNS = fields->properties.negativeSuffixPattern.isEmpty(); + if (!trivialPP || !trivialPS || !trivialNP || !trivialNS) { + trace("no fast format: affixes\n"); + fields->canUseFastFormat = false; + return; + } + + const DecimalFormatSymbols* symbols = getDecimalFormatSymbols(); + + // Grouping (secondary grouping is forbidden in equalsDefaultExceptFastFormat): + bool groupingUsed = fields->properties.groupingUsed; + int32_t groupingSize = fields->properties.groupingSize; + bool unusualGroupingSize = groupingSize > 0 && groupingSize != 3; + const UnicodeString& groupingString = symbols->getConstSymbol(DecimalFormatSymbols::kGroupingSeparatorSymbol); + if (groupingUsed && (unusualGroupingSize || groupingString.length() != 1)) { + trace("no fast format: grouping\n"); + fields->canUseFastFormat = false; + return; + } + + // Integer length: + int32_t minInt = fields->exportedProperties.minimumIntegerDigits; + int32_t maxInt = fields->exportedProperties.maximumIntegerDigits; + // Fastpath supports up to only 10 digits (length of INT32_MIN) + if (minInt > 10) { + trace("no fast format: integer\n"); + fields->canUseFastFormat = false; + return; + } + + // Fraction length (no fraction part allowed in fast path): + int32_t minFrac = fields->exportedProperties.minimumFractionDigits; + if (minFrac > 0) { + trace("no fast format: fraction\n"); + fields->canUseFastFormat = false; + return; + } + + // Other symbols: + const UnicodeString& minusSignString = symbols->getConstSymbol(DecimalFormatSymbols::kMinusSignSymbol); + UChar32 codePointZero = symbols->getCodePointZero(); + if (minusSignString.length() != 1 || U16_LENGTH(codePointZero) != 1) { + trace("no fast format: symbols\n"); + fields->canUseFastFormat = false; + return; + } + + // Good to go! + trace("can use fast format!\n"); + fields->canUseFastFormat = true; + fields->fastData.cpZero = static_cast(codePointZero); + fields->fastData.cpGroupingSeparator = groupingUsed && groupingSize == 3 ? groupingString.charAt(0) : 0; + fields->fastData.cpMinusSign = minusSignString.charAt(0); + fields->fastData.minInt = (minInt < 0 || minInt > 127) ? 0 : static_cast(minInt); + fields->fastData.maxInt = (maxInt < 0 || maxInt > 127) ? 127 : static_cast(maxInt); +} + +bool DecimalFormat::fastFormatDouble(double input, UnicodeString& output) const { + if (!fields->canUseFastFormat) { + return false; + } + if (std::isnan(input) + || uprv_trunc(input) != input + || input <= INT32_MIN + || input > INT32_MAX) { + return false; + } + doFastFormatInt32(static_cast(input), std::signbit(input), output); + return true; +} + +bool DecimalFormat::fastFormatInt64(int64_t input, UnicodeString& output) const { + if (!fields->canUseFastFormat) { + return false; + } + if (input <= INT32_MIN || input > INT32_MAX) { + return false; + } + doFastFormatInt32(static_cast(input), input < 0, output); + return true; +} + +void DecimalFormat::doFastFormatInt32(int32_t input, bool isNegative, UnicodeString& output) const { + U_ASSERT(fields->canUseFastFormat); + if (isNegative) { + output.append(fields->fastData.cpMinusSign); + U_ASSERT(input != INT32_MIN); // handled by callers + input = -input; + } + // Cap at int32_t to make the buffer small and operations fast. + // Longest string: "2,147,483,648" (13 chars in length) + static constexpr int32_t localCapacity = 13; + char16_t localBuffer[localCapacity]; + char16_t* ptr = localBuffer + localCapacity; + int8_t group = 0; + int8_t minInt = (fields->fastData.minInt < 1)? 1: fields->fastData.minInt; + for (int8_t i = 0; i < fields->fastData.maxInt && (input != 0 || i < minInt); i++) { + if (group++ == 3 && fields->fastData.cpGroupingSeparator != 0) { + *(--ptr) = fields->fastData.cpGroupingSeparator; + group = 1; + } + std::div_t res = std::div(input, 10); + *(--ptr) = static_cast(fields->fastData.cpZero + res.rem); + input = res.quot; + } + int32_t len = localCapacity - static_cast(ptr - localBuffer); + output.append(ptr, len); +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/displayoptions.cpp b/intl/icu/source/i18n/displayoptions.cpp new file mode 100644 index 0000000000..bb49e6033f --- /dev/null +++ b/intl/icu/source/i18n/displayoptions.cpp @@ -0,0 +1,167 @@ +// © 2022 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/displayoptions.h" +#include "unicode/udisplayoptions.h" +#include "cstring.h" + +U_NAMESPACE_BEGIN + +DisplayOptions::Builder DisplayOptions::builder() { return DisplayOptions::Builder(); } + +DisplayOptions::Builder DisplayOptions::copyToBuilder() const { return Builder(*this); } + +DisplayOptions::DisplayOptions(const Builder &builder) { + grammaticalCase = builder.grammaticalCase; + nounClass = builder.nounClass; + pluralCategory = builder.pluralCategory; + capitalization = builder.capitalization; + nameStyle = builder.nameStyle; + displayLength = builder.displayLength; + substituteHandling = builder.substituteHandling; +} + +DisplayOptions::Builder::Builder() { + // Sets default values. + grammaticalCase = UDISPOPT_GRAMMATICAL_CASE_UNDEFINED; + nounClass = UDISPOPT_NOUN_CLASS_UNDEFINED; + pluralCategory = UDISPOPT_PLURAL_CATEGORY_UNDEFINED; + capitalization = UDISPOPT_CAPITALIZATION_UNDEFINED; + nameStyle = UDISPOPT_NAME_STYLE_UNDEFINED; + displayLength = UDISPOPT_DISPLAY_LENGTH_UNDEFINED; + substituteHandling = UDISPOPT_SUBSTITUTE_HANDLING_UNDEFINED; +} + +DisplayOptions::Builder::Builder(const DisplayOptions &displayOptions) { + grammaticalCase = displayOptions.grammaticalCase; + nounClass = displayOptions.nounClass; + pluralCategory = displayOptions.pluralCategory; + capitalization = displayOptions.capitalization; + nameStyle = displayOptions.nameStyle; + displayLength = displayOptions.displayLength; + substituteHandling = displayOptions.substituteHandling; +} + +U_NAMESPACE_END + +// C API ------------------------------------------------------------------- *** + +U_NAMESPACE_USE + +namespace { + +const char *grammaticalCaseIds[] = { + "undefined", // 0 + "ablative", // 1 + "accusative", // 2 + "comitative", // 3 + "dative", // 4 + "ergative", // 5 + "genitive", // 6 + "instrumental", // 7 + "locative", // 8 + "locative_copulative", // 9 + "nominative", // 10 + "oblique", // 11 + "prepositional", // 12 + "sociative", // 13 + "vocative", // 14 +}; + +} // namespace + +U_CAPI const char * U_EXPORT2 +udispopt_getGrammaticalCaseIdentifier(UDisplayOptionsGrammaticalCase grammaticalCase) { + if (grammaticalCase >= 0 && grammaticalCase < UPRV_LENGTHOF(grammaticalCaseIds)) { + return grammaticalCaseIds[grammaticalCase]; + } + + return grammaticalCaseIds[0]; +} + +U_CAPI UDisplayOptionsGrammaticalCase U_EXPORT2 +udispopt_fromGrammaticalCaseIdentifier(const char *identifier) { + for (int32_t i = 0; i < UPRV_LENGTHOF(grammaticalCaseIds); i++) { + if (uprv_strcmp(identifier, grammaticalCaseIds[i]) == 0) { + return static_cast(i); + } + } + + return UDISPOPT_GRAMMATICAL_CASE_UNDEFINED; +} + +namespace { + +const char *pluralCategoryIds[] = { + "undefined", // 0 + "zero", // 1 + "one", // 2 + "two", // 3 + "few", // 4 + "many", // 5 + "other", // 6 +}; + +} // namespace + +U_CAPI const char * U_EXPORT2 +udispopt_getPluralCategoryIdentifier(UDisplayOptionsPluralCategory pluralCategory) { + if (pluralCategory >= 0 && pluralCategory < UPRV_LENGTHOF(pluralCategoryIds)) { + return pluralCategoryIds[pluralCategory]; + } + + return pluralCategoryIds[0]; +} + +U_CAPI UDisplayOptionsPluralCategory U_EXPORT2 +udispopt_fromPluralCategoryIdentifier(const char *identifier) { + for (int32_t i = 0; i < UPRV_LENGTHOF(pluralCategoryIds); i++) { + if (uprv_strcmp(identifier, pluralCategoryIds[i]) == 0) { + return static_cast(i); + } + } + + return UDISPOPT_PLURAL_CATEGORY_UNDEFINED; +} + +namespace { + +const char *nounClassIds[] = { + "undefined", // 0 + "other", // 1 + "neuter", // 2 + "feminine", // 3 + "masculine", // 4 + "animate", // 5 + "inanimate", // 6 + "personal", // 7 + "common", // 8 +}; + +} // namespace + +U_CAPI const char * U_EXPORT2 +udispopt_getNounClassIdentifier(UDisplayOptionsNounClass nounClass) { + if (nounClass >= 0 && nounClass < UPRV_LENGTHOF(nounClassIds)) { + return nounClassIds[nounClass]; + } + + return nounClassIds[0]; +} + +U_CAPI UDisplayOptionsNounClass U_EXPORT2 +udispopt_fromNounClassIdentifier(const char *identifier) { + for (int32_t i = 0; i < UPRV_LENGTHOF(nounClassIds); i++) { + if (uprv_strcmp(identifier, nounClassIds[i]) == 0) { + return static_cast(i); + } + } + + return UDISPOPT_NOUN_CLASS_UNDEFINED; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp b/intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp new file mode 100644 index 0000000000..638e9cb046 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp @@ -0,0 +1,659 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-bignum-dtoa.h" + +#include "double-conversion-bignum.h" +#include "double-conversion-ieee.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +static int NormalizedExponent(uint64_t significand, int exponent) { + DOUBLE_CONVERSION_ASSERT(significand != 0); + while ((significand & Double::kHiddenBit) == 0) { + significand = significand << 1; + exponent = exponent - 1; + } + return exponent; +} + + +// Forward declarations: +// Returns an estimation of k such that 10^(k-1) <= v < 10^k. +static int EstimatePower(int exponent); +// Computes v / 10^estimated_power exactly, as a ratio of two bignums, numerator +// and denominator. +static void InitialScaledStartValues(uint64_t significand, + int exponent, + bool lower_boundary_is_closer, + int estimated_power, + bool need_boundary_deltas, + Bignum* numerator, + Bignum* denominator, + Bignum* delta_minus, + Bignum* delta_plus); +// Multiplies numerator/denominator so that its values lies in the range 1-10. +// Returns decimal_point s.t. +// v = numerator'/denominator' * 10^(decimal_point-1) +// where numerator' and denominator' are the values of numerator and +// denominator after the call to this function. +static void FixupMultiply10(int estimated_power, bool is_even, + int* decimal_point, + Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus); +// Generates digits from the left to the right and stops when the generated +// digits yield the shortest decimal representation of v. +static void GenerateShortestDigits(Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus, + bool is_even, + Vector buffer, int* length); +// Generates 'requested_digits' after the decimal point. +static void BignumToFixed(int requested_digits, int* decimal_point, + Bignum* numerator, Bignum* denominator, + Vector buffer, int* length); +// Generates 'count' digits of numerator/denominator. +// Once 'count' digits have been produced rounds the result depending on the +// remainder (remainders of exactly .5 round upwards). Might update the +// decimal_point when rounding up (for example for 0.9999). +static void GenerateCountedDigits(int count, int* decimal_point, + Bignum* numerator, Bignum* denominator, + Vector buffer, int* length); + + +void BignumDtoa(double v, BignumDtoaMode mode, int requested_digits, + Vector buffer, int* length, int* decimal_point) { + DOUBLE_CONVERSION_ASSERT(v > 0); + DOUBLE_CONVERSION_ASSERT(!Double(v).IsSpecial()); + uint64_t significand; + int exponent; + bool lower_boundary_is_closer; + if (mode == BIGNUM_DTOA_SHORTEST_SINGLE) { + float f = static_cast(v); + DOUBLE_CONVERSION_ASSERT(f == v); + significand = Single(f).Significand(); + exponent = Single(f).Exponent(); + lower_boundary_is_closer = Single(f).LowerBoundaryIsCloser(); + } else { + significand = Double(v).Significand(); + exponent = Double(v).Exponent(); + lower_boundary_is_closer = Double(v).LowerBoundaryIsCloser(); + } + bool need_boundary_deltas = + (mode == BIGNUM_DTOA_SHORTEST || mode == BIGNUM_DTOA_SHORTEST_SINGLE); + + bool is_even = (significand & 1) == 0; + int normalized_exponent = NormalizedExponent(significand, exponent); + // estimated_power might be too low by 1. + int estimated_power = EstimatePower(normalized_exponent); + + // Shortcut for Fixed. + // The requested digits correspond to the digits after the point. If the + // number is much too small, then there is no need in trying to get any + // digits. + if (mode == BIGNUM_DTOA_FIXED && -estimated_power - 1 > requested_digits) { + buffer[0] = '\0'; + *length = 0; + // Set decimal-point to -requested_digits. This is what Gay does. + // Note that it should not have any effect anyways since the string is + // empty. + *decimal_point = -requested_digits; + return; + } + + Bignum numerator; + Bignum denominator; + Bignum delta_minus; + Bignum delta_plus; + // Make sure the bignum can grow large enough. The smallest double equals + // 4e-324. In this case the denominator needs fewer than 324*4 binary digits. + // The maximum double is 1.7976931348623157e308 which needs fewer than + // 308*4 binary digits. + DOUBLE_CONVERSION_ASSERT(Bignum::kMaxSignificantBits >= 324*4); + InitialScaledStartValues(significand, exponent, lower_boundary_is_closer, + estimated_power, need_boundary_deltas, + &numerator, &denominator, + &delta_minus, &delta_plus); + // We now have v = (numerator / denominator) * 10^estimated_power. + FixupMultiply10(estimated_power, is_even, decimal_point, + &numerator, &denominator, + &delta_minus, &delta_plus); + // We now have v = (numerator / denominator) * 10^(decimal_point-1), and + // 1 <= (numerator + delta_plus) / denominator < 10 + switch (mode) { + case BIGNUM_DTOA_SHORTEST: + case BIGNUM_DTOA_SHORTEST_SINGLE: + GenerateShortestDigits(&numerator, &denominator, + &delta_minus, &delta_plus, + is_even, buffer, length); + break; + case BIGNUM_DTOA_FIXED: + BignumToFixed(requested_digits, decimal_point, + &numerator, &denominator, + buffer, length); + break; + case BIGNUM_DTOA_PRECISION: + GenerateCountedDigits(requested_digits, decimal_point, + &numerator, &denominator, + buffer, length); + break; + default: + DOUBLE_CONVERSION_UNREACHABLE(); + } + buffer[*length] = '\0'; +} + + +// The procedure starts generating digits from the left to the right and stops +// when the generated digits yield the shortest decimal representation of v. A +// decimal representation of v is a number lying closer to v than to any other +// double, so it converts to v when read. +// +// This is true if d, the decimal representation, is between m- and m+, the +// upper and lower boundaries. d must be strictly between them if !is_even. +// m- := (numerator - delta_minus) / denominator +// m+ := (numerator + delta_plus) / denominator +// +// Precondition: 0 <= (numerator+delta_plus) / denominator < 10. +// If 1 <= (numerator+delta_plus) / denominator < 10 then no leading 0 digit +// will be produced. This should be the standard precondition. +static void GenerateShortestDigits(Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus, + bool is_even, + Vector buffer, int* length) { + // Small optimization: if delta_minus and delta_plus are the same just reuse + // one of the two bignums. + if (Bignum::Equal(*delta_minus, *delta_plus)) { + delta_plus = delta_minus; + } + *length = 0; + for (;;) { + uint16_t digit; + digit = numerator->DivideModuloIntBignum(*denominator); + DOUBLE_CONVERSION_ASSERT(digit <= 9); // digit is a uint16_t and therefore always positive. + // digit = numerator / denominator (integer division). + // numerator = numerator % denominator. + buffer[(*length)++] = static_cast(digit + '0'); + + // Can we stop already? + // If the remainder of the division is less than the distance to the lower + // boundary we can stop. In this case we simply round down (discarding the + // remainder). + // Similarly we test if we can round up (using the upper boundary). + bool in_delta_room_minus; + bool in_delta_room_plus; + if (is_even) { + in_delta_room_minus = Bignum::LessEqual(*numerator, *delta_minus); + } else { + in_delta_room_minus = Bignum::Less(*numerator, *delta_minus); + } + if (is_even) { + in_delta_room_plus = + Bignum::PlusCompare(*numerator, *delta_plus, *denominator) >= 0; + } else { + in_delta_room_plus = + Bignum::PlusCompare(*numerator, *delta_plus, *denominator) > 0; + } + if (!in_delta_room_minus && !in_delta_room_plus) { + // Prepare for next iteration. + numerator->Times10(); + delta_minus->Times10(); + // We optimized delta_plus to be equal to delta_minus (if they share the + // same value). So don't multiply delta_plus if they point to the same + // object. + if (delta_minus != delta_plus) { + delta_plus->Times10(); + } + } else if (in_delta_room_minus && in_delta_room_plus) { + // Let's see if 2*numerator < denominator. + // If yes, then the next digit would be < 5 and we can round down. + int compare = Bignum::PlusCompare(*numerator, *numerator, *denominator); + if (compare < 0) { + // Remaining digits are less than .5. -> Round down (== do nothing). + } else if (compare > 0) { + // Remaining digits are more than .5 of denominator. -> Round up. + // Note that the last digit could not be a '9' as otherwise the whole + // loop would have stopped earlier. + // We still have an assert here in case the preconditions were not + // satisfied. + DOUBLE_CONVERSION_ASSERT(buffer[(*length) - 1] != '9'); + buffer[(*length) - 1]++; + } else { + // Halfway case. + // TODO(floitsch): need a way to solve half-way cases. + // For now let's round towards even (since this is what Gay seems to + // do). + + if ((buffer[(*length) - 1] - '0') % 2 == 0) { + // Round down => Do nothing. + } else { + DOUBLE_CONVERSION_ASSERT(buffer[(*length) - 1] != '9'); + buffer[(*length) - 1]++; + } + } + return; + } else if (in_delta_room_minus) { + // Round down (== do nothing). + return; + } else { // in_delta_room_plus + // Round up. + // Note again that the last digit could not be '9' since this would have + // stopped the loop earlier. + // We still have an DOUBLE_CONVERSION_ASSERT here, in case the preconditions were not + // satisfied. + DOUBLE_CONVERSION_ASSERT(buffer[(*length) -1] != '9'); + buffer[(*length) - 1]++; + return; + } + } +} + + +// Let v = numerator / denominator < 10. +// Then we generate 'count' digits of d = x.xxxxx... (without the decimal point) +// from left to right. Once 'count' digits have been produced we decide whether +// to round up or down. Remainders of exactly .5 round upwards. Numbers such +// as 9.999999 propagate a carry all the way, and change the +// exponent (decimal_point), when rounding upwards. +static void GenerateCountedDigits(int count, int* decimal_point, + Bignum* numerator, Bignum* denominator, + Vector buffer, int* length) { + DOUBLE_CONVERSION_ASSERT(count >= 0); + for (int i = 0; i < count - 1; ++i) { + uint16_t digit; + digit = numerator->DivideModuloIntBignum(*denominator); + DOUBLE_CONVERSION_ASSERT(digit <= 9); // digit is a uint16_t and therefore always positive. + // digit = numerator / denominator (integer division). + // numerator = numerator % denominator. + buffer[i] = static_cast(digit + '0'); + // Prepare for next iteration. + numerator->Times10(); + } + // Generate the last digit. + uint16_t digit; + digit = numerator->DivideModuloIntBignum(*denominator); + if (Bignum::PlusCompare(*numerator, *numerator, *denominator) >= 0) { + digit++; + } + DOUBLE_CONVERSION_ASSERT(digit <= 10); + buffer[count - 1] = static_cast(digit + '0'); + // Correct bad digits (in case we had a sequence of '9's). Propagate the + // carry until we hat a non-'9' or til we reach the first digit. + for (int i = count - 1; i > 0; --i) { + if (buffer[i] != '0' + 10) break; + buffer[i] = '0'; + buffer[i - 1]++; + } + if (buffer[0] == '0' + 10) { + // Propagate a carry past the top place. + buffer[0] = '1'; + (*decimal_point)++; + } + *length = count; +} + + +// Generates 'requested_digits' after the decimal point. It might omit +// trailing '0's. If the input number is too small then no digits at all are +// generated (ex.: 2 fixed digits for 0.00001). +// +// Input verifies: 1 <= (numerator + delta) / denominator < 10. +static void BignumToFixed(int requested_digits, int* decimal_point, + Bignum* numerator, Bignum* denominator, + Vector buffer, int* length) { + // Note that we have to look at more than just the requested_digits, since + // a number could be rounded up. Example: v=0.5 with requested_digits=0. + // Even though the power of v equals 0 we can't just stop here. + if (-(*decimal_point) > requested_digits) { + // The number is definitively too small. + // Ex: 0.001 with requested_digits == 1. + // Set decimal-point to -requested_digits. This is what Gay does. + // Note that it should not have any effect anyways since the string is + // empty. + *decimal_point = -requested_digits; + *length = 0; + return; + } else if (-(*decimal_point) == requested_digits) { + // We only need to verify if the number rounds down or up. + // Ex: 0.04 and 0.06 with requested_digits == 1. + DOUBLE_CONVERSION_ASSERT(*decimal_point == -requested_digits); + // Initially the fraction lies in range (1, 10]. Multiply the denominator + // by 10 so that we can compare more easily. + denominator->Times10(); + if (Bignum::PlusCompare(*numerator, *numerator, *denominator) >= 0) { + // If the fraction is >= 0.5 then we have to include the rounded + // digit. + buffer[0] = '1'; + *length = 1; + (*decimal_point)++; + } else { + // Note that we caught most of similar cases earlier. + *length = 0; + } + return; + } else { + // The requested digits correspond to the digits after the point. + // The variable 'needed_digits' includes the digits before the point. + int needed_digits = (*decimal_point) + requested_digits; + GenerateCountedDigits(needed_digits, decimal_point, + numerator, denominator, + buffer, length); + } +} + + +// Returns an estimation of k such that 10^(k-1) <= v < 10^k where +// v = f * 2^exponent and 2^52 <= f < 2^53. +// v is hence a normalized double with the given exponent. The output is an +// approximation for the exponent of the decimal approximation .digits * 10^k. +// +// The result might undershoot by 1 in which case 10^k <= v < 10^k+1. +// Note: this property holds for v's upper boundary m+ too. +// 10^k <= m+ < 10^k+1. +// (see explanation below). +// +// Examples: +// EstimatePower(0) => 16 +// EstimatePower(-52) => 0 +// +// Note: e >= 0 => EstimatedPower(e) > 0. No similar claim can be made for e<0. +static int EstimatePower(int exponent) { + // This function estimates log10 of v where v = f*2^e (with e == exponent). + // Note that 10^floor(log10(v)) <= v, but v <= 10^ceil(log10(v)). + // Note that f is bounded by its container size. Let p = 53 (the double's + // significand size). Then 2^(p-1) <= f < 2^p. + // + // Given that log10(v) == log2(v)/log2(10) and e+(len(f)-1) is quite close + // to log2(v) the function is simplified to (e+(len(f)-1)/log2(10)). + // The computed number undershoots by less than 0.631 (when we compute log3 + // and not log10). + // + // Optimization: since we only need an approximated result this computation + // can be performed on 64 bit integers. On x86/x64 architecture the speedup is + // not really measurable, though. + // + // Since we want to avoid overshooting we decrement by 1e10 so that + // floating-point imprecisions don't affect us. + // + // Explanation for v's boundary m+: the computation takes advantage of + // the fact that 2^(p-1) <= f < 2^p. Boundaries still satisfy this requirement + // (even for denormals where the delta can be much more important). + + const double k1Log10 = 0.30102999566398114; // 1/lg(10) + + // For doubles len(f) == 53 (don't forget the hidden bit). + const int kSignificandSize = Double::kSignificandSize; + double estimate = ceil((exponent + kSignificandSize - 1) * k1Log10 - 1e-10); + return static_cast(estimate); +} + + +// See comments for InitialScaledStartValues. +static void InitialScaledStartValuesPositiveExponent( + uint64_t significand, int exponent, + int estimated_power, bool need_boundary_deltas, + Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus) { + // A positive exponent implies a positive power. + DOUBLE_CONVERSION_ASSERT(estimated_power >= 0); + // Since the estimated_power is positive we simply multiply the denominator + // by 10^estimated_power. + + // numerator = v. + numerator->AssignUInt64(significand); + numerator->ShiftLeft(exponent); + // denominator = 10^estimated_power. + denominator->AssignPowerUInt16(10, estimated_power); + + if (need_boundary_deltas) { + // Introduce a common denominator so that the deltas to the boundaries are + // integers. + denominator->ShiftLeft(1); + numerator->ShiftLeft(1); + // Let v = f * 2^e, then m+ - v = 1/2 * 2^e; With the common + // denominator (of 2) delta_plus equals 2^e. + delta_plus->AssignUInt16(1); + delta_plus->ShiftLeft(exponent); + // Same for delta_minus. The adjustments if f == 2^p-1 are done later. + delta_minus->AssignUInt16(1); + delta_minus->ShiftLeft(exponent); + } +} + + +// See comments for InitialScaledStartValues +static void InitialScaledStartValuesNegativeExponentPositivePower( + uint64_t significand, int exponent, + int estimated_power, bool need_boundary_deltas, + Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus) { + // v = f * 2^e with e < 0, and with estimated_power >= 0. + // This means that e is close to 0 (have a look at how estimated_power is + // computed). + + // numerator = significand + // since v = significand * 2^exponent this is equivalent to + // numerator = v * / 2^-exponent + numerator->AssignUInt64(significand); + // denominator = 10^estimated_power * 2^-exponent (with exponent < 0) + denominator->AssignPowerUInt16(10, estimated_power); + denominator->ShiftLeft(-exponent); + + if (need_boundary_deltas) { + // Introduce a common denominator so that the deltas to the boundaries are + // integers. + denominator->ShiftLeft(1); + numerator->ShiftLeft(1); + // Let v = f * 2^e, then m+ - v = 1/2 * 2^e; With the common + // denominator (of 2) delta_plus equals 2^e. + // Given that the denominator already includes v's exponent the distance + // to the boundaries is simply 1. + delta_plus->AssignUInt16(1); + // Same for delta_minus. The adjustments if f == 2^p-1 are done later. + delta_minus->AssignUInt16(1); + } +} + + +// See comments for InitialScaledStartValues +static void InitialScaledStartValuesNegativeExponentNegativePower( + uint64_t significand, int exponent, + int estimated_power, bool need_boundary_deltas, + Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus) { + // Instead of multiplying the denominator with 10^estimated_power we + // multiply all values (numerator and deltas) by 10^-estimated_power. + + // Use numerator as temporary container for power_ten. + Bignum* power_ten = numerator; + power_ten->AssignPowerUInt16(10, -estimated_power); + + if (need_boundary_deltas) { + // Since power_ten == numerator we must make a copy of 10^estimated_power + // before we complete the computation of the numerator. + // delta_plus = delta_minus = 10^estimated_power + delta_plus->AssignBignum(*power_ten); + delta_minus->AssignBignum(*power_ten); + } + + // numerator = significand * 2 * 10^-estimated_power + // since v = significand * 2^exponent this is equivalent to + // numerator = v * 10^-estimated_power * 2 * 2^-exponent. + // Remember: numerator has been abused as power_ten. So no need to assign it + // to itself. + DOUBLE_CONVERSION_ASSERT(numerator == power_ten); + numerator->MultiplyByUInt64(significand); + + // denominator = 2 * 2^-exponent with exponent < 0. + denominator->AssignUInt16(1); + denominator->ShiftLeft(-exponent); + + if (need_boundary_deltas) { + // Introduce a common denominator so that the deltas to the boundaries are + // integers. + numerator->ShiftLeft(1); + denominator->ShiftLeft(1); + // With this shift the boundaries have their correct value, since + // delta_plus = 10^-estimated_power, and + // delta_minus = 10^-estimated_power. + // These assignments have been done earlier. + // The adjustments if f == 2^p-1 (lower boundary is closer) are done later. + } +} + + +// Let v = significand * 2^exponent. +// Computes v / 10^estimated_power exactly, as a ratio of two bignums, numerator +// and denominator. The functions GenerateShortestDigits and +// GenerateCountedDigits will then convert this ratio to its decimal +// representation d, with the required accuracy. +// Then d * 10^estimated_power is the representation of v. +// (Note: the fraction and the estimated_power might get adjusted before +// generating the decimal representation.) +// +// The initial start values consist of: +// - a scaled numerator: s.t. numerator/denominator == v / 10^estimated_power. +// - a scaled (common) denominator. +// optionally (used by GenerateShortestDigits to decide if it has the shortest +// decimal converting back to v): +// - v - m-: the distance to the lower boundary. +// - m+ - v: the distance to the upper boundary. +// +// v, m+, m-, and therefore v - m- and m+ - v all share the same denominator. +// +// Let ep == estimated_power, then the returned values will satisfy: +// v / 10^ep = numerator / denominator. +// v's boundaries m- and m+: +// m- / 10^ep == v / 10^ep - delta_minus / denominator +// m+ / 10^ep == v / 10^ep + delta_plus / denominator +// Or in other words: +// m- == v - delta_minus * 10^ep / denominator; +// m+ == v + delta_plus * 10^ep / denominator; +// +// Since 10^(k-1) <= v < 10^k (with k == estimated_power) +// or 10^k <= v < 10^(k+1) +// we then have 0.1 <= numerator/denominator < 1 +// or 1 <= numerator/denominator < 10 +// +// It is then easy to kickstart the digit-generation routine. +// +// The boundary-deltas are only filled if the mode equals BIGNUM_DTOA_SHORTEST +// or BIGNUM_DTOA_SHORTEST_SINGLE. + +static void InitialScaledStartValues(uint64_t significand, + int exponent, + bool lower_boundary_is_closer, + int estimated_power, + bool need_boundary_deltas, + Bignum* numerator, + Bignum* denominator, + Bignum* delta_minus, + Bignum* delta_plus) { + if (exponent >= 0) { + InitialScaledStartValuesPositiveExponent( + significand, exponent, estimated_power, need_boundary_deltas, + numerator, denominator, delta_minus, delta_plus); + } else if (estimated_power >= 0) { + InitialScaledStartValuesNegativeExponentPositivePower( + significand, exponent, estimated_power, need_boundary_deltas, + numerator, denominator, delta_minus, delta_plus); + } else { + InitialScaledStartValuesNegativeExponentNegativePower( + significand, exponent, estimated_power, need_boundary_deltas, + numerator, denominator, delta_minus, delta_plus); + } + + if (need_boundary_deltas && lower_boundary_is_closer) { + // The lower boundary is closer at half the distance of "normal" numbers. + // Increase the common denominator and adapt all but the delta_minus. + denominator->ShiftLeft(1); // *2 + numerator->ShiftLeft(1); // *2 + delta_plus->ShiftLeft(1); // *2 + } +} + + +// This routine multiplies numerator/denominator so that its values lies in the +// range 1-10. That is after a call to this function we have: +// 1 <= (numerator + delta_plus) /denominator < 10. +// Let numerator the input before modification and numerator' the argument +// after modification, then the output-parameter decimal_point is such that +// numerator / denominator * 10^estimated_power == +// numerator' / denominator' * 10^(decimal_point - 1) +// In some cases estimated_power was too low, and this is already the case. We +// then simply adjust the power so that 10^(k-1) <= v < 10^k (with k == +// estimated_power) but do not touch the numerator or denominator. +// Otherwise the routine multiplies the numerator and the deltas by 10. +static void FixupMultiply10(int estimated_power, bool is_even, + int* decimal_point, + Bignum* numerator, Bignum* denominator, + Bignum* delta_minus, Bignum* delta_plus) { + bool in_range; + if (is_even) { + // For IEEE doubles half-way cases (in decimal system numbers ending with 5) + // are rounded to the closest floating-point number with even significand. + in_range = Bignum::PlusCompare(*numerator, *delta_plus, *denominator) >= 0; + } else { + in_range = Bignum::PlusCompare(*numerator, *delta_plus, *denominator) > 0; + } + if (in_range) { + // Since numerator + delta_plus >= denominator we already have + // 1 <= numerator/denominator < 10. Simply update the estimated_power. + *decimal_point = estimated_power + 1; + } else { + *decimal_point = estimated_power; + numerator->Times10(); + if (Bignum::Equal(*delta_minus, *delta_plus)) { + delta_minus->Times10(); + delta_plus->AssignBignum(*delta_minus); + } else { + delta_minus->Times10(); + delta_plus->Times10(); + } + } +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-bignum-dtoa.h b/intl/icu/source/i18n/double-conversion-bignum-dtoa.h new file mode 100644 index 0000000000..edc21b0f2e --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-bignum-dtoa.h @@ -0,0 +1,102 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_BIGNUM_DTOA_H_ +#define DOUBLE_CONVERSION_BIGNUM_DTOA_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +enum BignumDtoaMode { + // Return the shortest correct representation. + // For example the output of 0.299999999999999988897 is (the less accurate but + // correct) 0.3. + BIGNUM_DTOA_SHORTEST, + // Same as BIGNUM_DTOA_SHORTEST but for single-precision floats. + BIGNUM_DTOA_SHORTEST_SINGLE, + // Return a fixed number of digits after the decimal point. + // For instance fixed(0.1, 4) becomes 0.1000 + // If the input number is big, the output will be big. + BIGNUM_DTOA_FIXED, + // Return a fixed number of digits, no matter what the exponent is. + BIGNUM_DTOA_PRECISION +}; + +// Converts the given double 'v' to ascii. +// The result should be interpreted as buffer * 10^(point-length). +// The buffer will be null-terminated. +// +// The input v must be > 0 and different from NaN, and Infinity. +// +// The output depends on the given mode: +// - SHORTEST: produce the least amount of digits for which the internal +// identity requirement is still satisfied. If the digits are printed +// (together with the correct exponent) then reading this number will give +// 'v' again. The buffer will choose the representation that is closest to +// 'v'. If there are two at the same distance, than the number is round up. +// In this mode the 'requested_digits' parameter is ignored. +// - FIXED: produces digits necessary to print a given number with +// 'requested_digits' digits after the decimal point. The produced digits +// might be too short in which case the caller has to fill the gaps with '0's. +// Example: toFixed(0.001, 5) is allowed to return buffer="1", point=-2. +// Halfway cases are rounded up. The call toFixed(0.15, 2) thus returns +// buffer="2", point=0. +// Note: the length of the returned buffer has no meaning wrt the significance +// of its digits. That is, just because it contains '0's does not mean that +// any other digit would not satisfy the internal identity requirement. +// - PRECISION: produces 'requested_digits' where the first digit is not '0'. +// Even though the length of produced digits usually equals +// 'requested_digits', the function is allowed to return fewer digits, in +// which case the caller has to fill the missing digits with '0's. +// Halfway cases are again rounded up. +// 'BignumDtoa' expects the given buffer to be big enough to hold all digits +// and a terminating null-character. +void BignumDtoa(double v, BignumDtoaMode mode, int requested_digits, + Vector buffer, int* length, int* point); + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_BIGNUM_DTOA_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-bignum.cpp b/intl/icu/source/i18n/double-conversion-bignum.cpp new file mode 100644 index 0000000000..d2b701a21d --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-bignum.cpp @@ -0,0 +1,815 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include +#include + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-bignum.h" +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +Bignum::Chunk& Bignum::RawBigit(const int index) { + DOUBLE_CONVERSION_ASSERT(static_cast(index) < kBigitCapacity); + return bigits_buffer_[index]; +} + + +const Bignum::Chunk& Bignum::RawBigit(const int index) const { + DOUBLE_CONVERSION_ASSERT(static_cast(index) < kBigitCapacity); + return bigits_buffer_[index]; +} + + +template +static int BitSize(const S value) { + (void) value; // Mark variable as used. + return 8 * sizeof(value); +} + +// Guaranteed to lie in one Bigit. +void Bignum::AssignUInt16(const uint16_t value) { + DOUBLE_CONVERSION_ASSERT(kBigitSize >= BitSize(value)); + Zero(); + if (value > 0) { + RawBigit(0) = value; + used_bigits_ = 1; + } +} + + +void Bignum::AssignUInt64(uint64_t value) { + Zero(); + for(int i = 0; value > 0; ++i) { + RawBigit(i) = value & kBigitMask; + value >>= kBigitSize; + ++used_bigits_; + } +} + + +void Bignum::AssignBignum(const Bignum& other) { + exponent_ = other.exponent_; + for (int i = 0; i < other.used_bigits_; ++i) { + RawBigit(i) = other.RawBigit(i); + } + used_bigits_ = other.used_bigits_; +} + + +static uint64_t ReadUInt64(const Vector buffer, + const int from, + const int digits_to_read) { + uint64_t result = 0; + for (int i = from; i < from + digits_to_read; ++i) { + const int digit = buffer[i] - '0'; + DOUBLE_CONVERSION_ASSERT(0 <= digit && digit <= 9); + result = result * 10 + digit; + } + return result; +} + + +void Bignum::AssignDecimalString(const Vector value) { + // 2^64 = 18446744073709551616 > 10^19 + static const int kMaxUint64DecimalDigits = 19; + Zero(); + int length = value.length(); + unsigned pos = 0; + // Let's just say that each digit needs 4 bits. + while (length >= kMaxUint64DecimalDigits) { + const uint64_t digits = ReadUInt64(value, pos, kMaxUint64DecimalDigits); + pos += kMaxUint64DecimalDigits; + length -= kMaxUint64DecimalDigits; + MultiplyByPowerOfTen(kMaxUint64DecimalDigits); + AddUInt64(digits); + } + const uint64_t digits = ReadUInt64(value, pos, length); + MultiplyByPowerOfTen(length); + AddUInt64(digits); + Clamp(); +} + + +static uint64_t HexCharValue(const int c) { + if ('0' <= c && c <= '9') { + return c - '0'; + } + if ('a' <= c && c <= 'f') { + return 10 + c - 'a'; + } + DOUBLE_CONVERSION_ASSERT('A' <= c && c <= 'F'); + return 10 + c - 'A'; +} + + +// Unlike AssignDecimalString(), this function is "only" used +// for unit-tests and therefore not performance critical. +void Bignum::AssignHexString(Vector value) { + Zero(); + // Required capacity could be reduced by ignoring leading zeros. + EnsureCapacity(((value.length() * 4) + kBigitSize - 1) / kBigitSize); + DOUBLE_CONVERSION_ASSERT(sizeof(uint64_t) * 8 >= kBigitSize + 4); // TODO: static_assert + // Accumulates converted hex digits until at least kBigitSize bits. + // Works with non-factor-of-four kBigitSizes. + uint64_t tmp = 0; + for (int cnt = 0; !value.is_empty(); value.pop_back()) { + tmp |= (HexCharValue(value.last()) << cnt); + if ((cnt += 4) >= kBigitSize) { + RawBigit(used_bigits_++) = (tmp & kBigitMask); + cnt -= kBigitSize; + tmp >>= kBigitSize; + } + } + if (tmp > 0) { + DOUBLE_CONVERSION_ASSERT(tmp <= kBigitMask); + RawBigit(used_bigits_++) = static_cast(tmp & kBigitMask); + } + Clamp(); +} + + +void Bignum::AddUInt64(const uint64_t operand) { + if (operand == 0) { + return; + } + Bignum other; + other.AssignUInt64(operand); + AddBignum(other); +} + + +void Bignum::AddBignum(const Bignum& other) { + DOUBLE_CONVERSION_ASSERT(IsClamped()); + DOUBLE_CONVERSION_ASSERT(other.IsClamped()); + + // If this has a greater exponent than other append zero-bigits to this. + // After this call exponent_ <= other.exponent_. + Align(other); + + // There are two possibilities: + // aaaaaaaaaaa 0000 (where the 0s represent a's exponent) + // bbbbb 00000000 + // ---------------- + // ccccccccccc 0000 + // or + // aaaaaaaaaa 0000 + // bbbbbbbbb 0000000 + // ----------------- + // cccccccccccc 0000 + // In both cases we might need a carry bigit. + + EnsureCapacity(1 + (std::max)(BigitLength(), other.BigitLength()) - exponent_); + Chunk carry = 0; + int bigit_pos = other.exponent_ - exponent_; + DOUBLE_CONVERSION_ASSERT(bigit_pos >= 0); + for (int i = used_bigits_; i < bigit_pos; ++i) { + RawBigit(i) = 0; + } + for (int i = 0; i < other.used_bigits_; ++i) { + const Chunk my = (bigit_pos < used_bigits_) ? RawBigit(bigit_pos) : 0; + const Chunk sum = my + other.RawBigit(i) + carry; + RawBigit(bigit_pos) = sum & kBigitMask; + carry = sum >> kBigitSize; + ++bigit_pos; + } + while (carry != 0) { + const Chunk my = (bigit_pos < used_bigits_) ? RawBigit(bigit_pos) : 0; + const Chunk sum = my + carry; + RawBigit(bigit_pos) = sum & kBigitMask; + carry = sum >> kBigitSize; + ++bigit_pos; + } + used_bigits_ = static_cast(std::max(bigit_pos, static_cast(used_bigits_))); + DOUBLE_CONVERSION_ASSERT(IsClamped()); +} + + +void Bignum::SubtractBignum(const Bignum& other) { + DOUBLE_CONVERSION_ASSERT(IsClamped()); + DOUBLE_CONVERSION_ASSERT(other.IsClamped()); + // We require this to be bigger than other. + DOUBLE_CONVERSION_ASSERT(LessEqual(other, *this)); + + Align(other); + + const int offset = other.exponent_ - exponent_; + Chunk borrow = 0; + int i; + for (i = 0; i < other.used_bigits_; ++i) { + DOUBLE_CONVERSION_ASSERT((borrow == 0) || (borrow == 1)); + const Chunk difference = RawBigit(i + offset) - other.RawBigit(i) - borrow; + RawBigit(i + offset) = difference & kBigitMask; + borrow = difference >> (kChunkSize - 1); + } + while (borrow != 0) { + const Chunk difference = RawBigit(i + offset) - borrow; + RawBigit(i + offset) = difference & kBigitMask; + borrow = difference >> (kChunkSize - 1); + ++i; + } + Clamp(); +} + + +void Bignum::ShiftLeft(const int shift_amount) { + if (used_bigits_ == 0) { + return; + } + exponent_ += static_cast(shift_amount / kBigitSize); + const int local_shift = shift_amount % kBigitSize; + EnsureCapacity(used_bigits_ + 1); + BigitsShiftLeft(local_shift); +} + + +void Bignum::MultiplyByUInt32(const uint32_t factor) { + if (factor == 1) { + return; + } + if (factor == 0) { + Zero(); + return; + } + if (used_bigits_ == 0) { + return; + } + // The product of a bigit with the factor is of size kBigitSize + 32. + // Assert that this number + 1 (for the carry) fits into double chunk. + DOUBLE_CONVERSION_ASSERT(kDoubleChunkSize >= kBigitSize + 32 + 1); + DoubleChunk carry = 0; + for (int i = 0; i < used_bigits_; ++i) { + const DoubleChunk product = static_cast(factor) * RawBigit(i) + carry; + RawBigit(i) = static_cast(product & kBigitMask); + carry = (product >> kBigitSize); + } + while (carry != 0) { + EnsureCapacity(used_bigits_ + 1); + RawBigit(used_bigits_) = carry & kBigitMask; + used_bigits_++; + carry >>= kBigitSize; + } +} + + +void Bignum::MultiplyByUInt64(const uint64_t factor) { + if (factor == 1) { + return; + } + if (factor == 0) { + Zero(); + return; + } + if (used_bigits_ == 0) { + return; + } + DOUBLE_CONVERSION_ASSERT(kBigitSize < 32); + uint64_t carry = 0; + const uint64_t low = factor & 0xFFFFFFFF; + const uint64_t high = factor >> 32; + for (int i = 0; i < used_bigits_; ++i) { + const uint64_t product_low = low * RawBigit(i); + const uint64_t product_high = high * RawBigit(i); + const uint64_t tmp = (carry & kBigitMask) + product_low; + RawBigit(i) = tmp & kBigitMask; + carry = (carry >> kBigitSize) + (tmp >> kBigitSize) + + (product_high << (32 - kBigitSize)); + } + while (carry != 0) { + EnsureCapacity(used_bigits_ + 1); + RawBigit(used_bigits_) = carry & kBigitMask; + used_bigits_++; + carry >>= kBigitSize; + } +} + + +void Bignum::MultiplyByPowerOfTen(const int exponent) { + static const uint64_t kFive27 = DOUBLE_CONVERSION_UINT64_2PART_C(0x6765c793, fa10079d); + static const uint16_t kFive1 = 5; + static const uint16_t kFive2 = kFive1 * 5; + static const uint16_t kFive3 = kFive2 * 5; + static const uint16_t kFive4 = kFive3 * 5; + static const uint16_t kFive5 = kFive4 * 5; + static const uint16_t kFive6 = kFive5 * 5; + static const uint32_t kFive7 = kFive6 * 5; + static const uint32_t kFive8 = kFive7 * 5; + static const uint32_t kFive9 = kFive8 * 5; + static const uint32_t kFive10 = kFive9 * 5; + static const uint32_t kFive11 = kFive10 * 5; + static const uint32_t kFive12 = kFive11 * 5; + static const uint32_t kFive13 = kFive12 * 5; + static const uint32_t kFive1_to_12[] = + { kFive1, kFive2, kFive3, kFive4, kFive5, kFive6, + kFive7, kFive8, kFive9, kFive10, kFive11, kFive12 }; + + DOUBLE_CONVERSION_ASSERT(exponent >= 0); + + if (exponent == 0) { + return; + } + if (used_bigits_ == 0) { + return; + } + // We shift by exponent at the end just before returning. + int remaining_exponent = exponent; + while (remaining_exponent >= 27) { + MultiplyByUInt64(kFive27); + remaining_exponent -= 27; + } + while (remaining_exponent >= 13) { + MultiplyByUInt32(kFive13); + remaining_exponent -= 13; + } + if (remaining_exponent > 0) { + MultiplyByUInt32(kFive1_to_12[remaining_exponent - 1]); + } + ShiftLeft(exponent); +} + + +void Bignum::Square() { + DOUBLE_CONVERSION_ASSERT(IsClamped()); + const int product_length = 2 * used_bigits_; + EnsureCapacity(product_length); + + // Comba multiplication: compute each column separately. + // Example: r = a2a1a0 * b2b1b0. + // r = 1 * a0b0 + + // 10 * (a1b0 + a0b1) + + // 100 * (a2b0 + a1b1 + a0b2) + + // 1000 * (a2b1 + a1b2) + + // 10000 * a2b2 + // + // In the worst case we have to accumulate nb-digits products of digit*digit. + // + // Assert that the additional number of bits in a DoubleChunk are enough to + // sum up used_digits of Bigit*Bigit. + if ((1 << (2 * (kChunkSize - kBigitSize))) <= used_bigits_) { + DOUBLE_CONVERSION_UNIMPLEMENTED(); + } + DoubleChunk accumulator = 0; + // First shift the digits so we don't overwrite them. + const int copy_offset = used_bigits_; + for (int i = 0; i < used_bigits_; ++i) { + RawBigit(copy_offset + i) = RawBigit(i); + } + // We have two loops to avoid some 'if's in the loop. + for (int i = 0; i < used_bigits_; ++i) { + // Process temporary digit i with power i. + // The sum of the two indices must be equal to i. + int bigit_index1 = i; + int bigit_index2 = 0; + // Sum all of the sub-products. + while (bigit_index1 >= 0) { + const Chunk chunk1 = RawBigit(copy_offset + bigit_index1); + const Chunk chunk2 = RawBigit(copy_offset + bigit_index2); + accumulator += static_cast(chunk1) * chunk2; + bigit_index1--; + bigit_index2++; + } + RawBigit(i) = static_cast(accumulator) & kBigitMask; + accumulator >>= kBigitSize; + } + for (int i = used_bigits_; i < product_length; ++i) { + int bigit_index1 = used_bigits_ - 1; + int bigit_index2 = i - bigit_index1; + // Invariant: sum of both indices is again equal to i. + // Inner loop runs 0 times on last iteration, emptying accumulator. + while (bigit_index2 < used_bigits_) { + const Chunk chunk1 = RawBigit(copy_offset + bigit_index1); + const Chunk chunk2 = RawBigit(copy_offset + bigit_index2); + accumulator += static_cast(chunk1) * chunk2; + bigit_index1--; + bigit_index2++; + } + // The overwritten RawBigit(i) will never be read in further loop iterations, + // because bigit_index1 and bigit_index2 are always greater + // than i - used_bigits_. + RawBigit(i) = static_cast(accumulator) & kBigitMask; + accumulator >>= kBigitSize; + } + // Since the result was guaranteed to lie inside the number the + // accumulator must be 0 now. + DOUBLE_CONVERSION_ASSERT(accumulator == 0); + + // Don't forget to update the used_digits and the exponent. + used_bigits_ = static_cast(product_length); + exponent_ *= 2; + Clamp(); +} + + +void Bignum::AssignPowerUInt16(uint16_t base, const int power_exponent) { + DOUBLE_CONVERSION_ASSERT(base != 0); + DOUBLE_CONVERSION_ASSERT(power_exponent >= 0); + if (power_exponent == 0) { + AssignUInt16(1); + return; + } + Zero(); + int shifts = 0; + // We expect base to be in range 2-32, and most often to be 10. + // It does not make much sense to implement different algorithms for counting + // the bits. + while ((base & 1) == 0) { + base >>= 1; + shifts++; + } + int bit_size = 0; + int tmp_base = base; + while (tmp_base != 0) { + tmp_base >>= 1; + bit_size++; + } + const int final_size = bit_size * power_exponent; + // 1 extra bigit for the shifting, and one for rounded final_size. + EnsureCapacity(final_size / kBigitSize + 2); + + // Left to Right exponentiation. + int mask = 1; + while (power_exponent >= mask) mask <<= 1; + + // The mask is now pointing to the bit above the most significant 1-bit of + // power_exponent. + // Get rid of first 1-bit; + mask >>= 2; + uint64_t this_value = base; + + bool delayed_multiplication = false; + const uint64_t max_32bits = 0xFFFFFFFF; + while (mask != 0 && this_value <= max_32bits) { + this_value = this_value * this_value; + // Verify that there is enough space in this_value to perform the + // multiplication. The first bit_size bits must be 0. + if ((power_exponent & mask) != 0) { + DOUBLE_CONVERSION_ASSERT(bit_size > 0); + const uint64_t base_bits_mask = + ~((static_cast(1) << (64 - bit_size)) - 1); + const bool high_bits_zero = (this_value & base_bits_mask) == 0; + if (high_bits_zero) { + this_value *= base; + } else { + delayed_multiplication = true; + } + } + mask >>= 1; + } + AssignUInt64(this_value); + if (delayed_multiplication) { + MultiplyByUInt32(base); + } + + // Now do the same thing as a bignum. + while (mask != 0) { + Square(); + if ((power_exponent & mask) != 0) { + MultiplyByUInt32(base); + } + mask >>= 1; + } + + // And finally add the saved shifts. + ShiftLeft(shifts * power_exponent); +} + + +// Precondition: this/other < 16bit. +uint16_t Bignum::DivideModuloIntBignum(const Bignum& other) { + DOUBLE_CONVERSION_ASSERT(IsClamped()); + DOUBLE_CONVERSION_ASSERT(other.IsClamped()); + DOUBLE_CONVERSION_ASSERT(other.used_bigits_ > 0); + + // Easy case: if we have less digits than the divisor than the result is 0. + // Note: this handles the case where this == 0, too. + if (BigitLength() < other.BigitLength()) { + return 0; + } + + Align(other); + + uint16_t result = 0; + + // Start by removing multiples of 'other' until both numbers have the same + // number of digits. + while (BigitLength() > other.BigitLength()) { + // This naive approach is extremely inefficient if `this` divided by other + // is big. This function is implemented for doubleToString where + // the result should be small (less than 10). + DOUBLE_CONVERSION_ASSERT(other.RawBigit(other.used_bigits_ - 1) >= ((1 << kBigitSize) / 16)); + DOUBLE_CONVERSION_ASSERT(RawBigit(used_bigits_ - 1) < 0x10000); + // Remove the multiples of the first digit. + // Example this = 23 and other equals 9. -> Remove 2 multiples. + result += static_cast(RawBigit(used_bigits_ - 1)); + SubtractTimes(other, RawBigit(used_bigits_ - 1)); + } + + DOUBLE_CONVERSION_ASSERT(BigitLength() == other.BigitLength()); + + // Both bignums are at the same length now. + // Since other has more than 0 digits we know that the access to + // RawBigit(used_bigits_ - 1) is safe. + const Chunk this_bigit = RawBigit(used_bigits_ - 1); + const Chunk other_bigit = other.RawBigit(other.used_bigits_ - 1); + + if (other.used_bigits_ == 1) { + // Shortcut for easy (and common) case. + int quotient = this_bigit / other_bigit; + RawBigit(used_bigits_ - 1) = this_bigit - other_bigit * quotient; + DOUBLE_CONVERSION_ASSERT(quotient < 0x10000); + result += static_cast(quotient); + Clamp(); + return result; + } + + const int division_estimate = this_bigit / (other_bigit + 1); + DOUBLE_CONVERSION_ASSERT(division_estimate < 0x10000); + result += static_cast(division_estimate); + SubtractTimes(other, division_estimate); + + if (other_bigit * (division_estimate + 1) > this_bigit) { + // No need to even try to subtract. Even if other's remaining digits were 0 + // another subtraction would be too much. + return result; + } + + while (LessEqual(other, *this)) { + SubtractBignum(other); + result++; + } + return result; +} + + +template +static int SizeInHexChars(S number) { + DOUBLE_CONVERSION_ASSERT(number > 0); + int result = 0; + while (number != 0) { + number >>= 4; + result++; + } + return result; +} + + +static char HexCharOfValue(const int value) { + DOUBLE_CONVERSION_ASSERT(0 <= value && value <= 16); + if (value < 10) { + return static_cast(value + '0'); + } + return static_cast(value - 10 + 'A'); +} + + +bool Bignum::ToHexString(char* buffer, const int buffer_size) const { + DOUBLE_CONVERSION_ASSERT(IsClamped()); + // Each bigit must be printable as separate hex-character. + DOUBLE_CONVERSION_ASSERT(kBigitSize % 4 == 0); + static const int kHexCharsPerBigit = kBigitSize / 4; + + if (used_bigits_ == 0) { + if (buffer_size < 2) { + return false; + } + buffer[0] = '0'; + buffer[1] = '\0'; + return true; + } + // We add 1 for the terminating '\0' character. + const int needed_chars = (BigitLength() - 1) * kHexCharsPerBigit + + SizeInHexChars(RawBigit(used_bigits_ - 1)) + 1; + if (needed_chars > buffer_size) { + return false; + } + int string_index = needed_chars - 1; + buffer[string_index--] = '\0'; + for (int i = 0; i < exponent_; ++i) { + for (int j = 0; j < kHexCharsPerBigit; ++j) { + buffer[string_index--] = '0'; + } + } + for (int i = 0; i < used_bigits_ - 1; ++i) { + Chunk current_bigit = RawBigit(i); + for (int j = 0; j < kHexCharsPerBigit; ++j) { + buffer[string_index--] = HexCharOfValue(current_bigit & 0xF); + current_bigit >>= 4; + } + } + // And finally the last bigit. + Chunk most_significant_bigit = RawBigit(used_bigits_ - 1); + while (most_significant_bigit != 0) { + buffer[string_index--] = HexCharOfValue(most_significant_bigit & 0xF); + most_significant_bigit >>= 4; + } + return true; +} + + +Bignum::Chunk Bignum::BigitOrZero(const int index) const { + if (index >= BigitLength()) { + return 0; + } + if (index < exponent_) { + return 0; + } + return RawBigit(index - exponent_); +} + + +int Bignum::Compare(const Bignum& a, const Bignum& b) { + DOUBLE_CONVERSION_ASSERT(a.IsClamped()); + DOUBLE_CONVERSION_ASSERT(b.IsClamped()); + const int bigit_length_a = a.BigitLength(); + const int bigit_length_b = b.BigitLength(); + if (bigit_length_a < bigit_length_b) { + return -1; + } + if (bigit_length_a > bigit_length_b) { + return +1; + } + for (int i = bigit_length_a - 1; i >= (std::min)(a.exponent_, b.exponent_); --i) { + const Chunk bigit_a = a.BigitOrZero(i); + const Chunk bigit_b = b.BigitOrZero(i); + if (bigit_a < bigit_b) { + return -1; + } + if (bigit_a > bigit_b) { + return +1; + } + // Otherwise they are equal up to this digit. Try the next digit. + } + return 0; +} + + +int Bignum::PlusCompare(const Bignum& a, const Bignum& b, const Bignum& c) { + DOUBLE_CONVERSION_ASSERT(a.IsClamped()); + DOUBLE_CONVERSION_ASSERT(b.IsClamped()); + DOUBLE_CONVERSION_ASSERT(c.IsClamped()); + if (a.BigitLength() < b.BigitLength()) { + return PlusCompare(b, a, c); + } + if (a.BigitLength() + 1 < c.BigitLength()) { + return -1; + } + if (a.BigitLength() > c.BigitLength()) { + return +1; + } + // The exponent encodes 0-bigits. So if there are more 0-digits in 'a' than + // 'b' has digits, then the bigit-length of 'a'+'b' must be equal to the one + // of 'a'. + if (a.exponent_ >= b.BigitLength() && a.BigitLength() < c.BigitLength()) { + return -1; + } + + Chunk borrow = 0; + // Starting at min_exponent all digits are == 0. So no need to compare them. + const int min_exponent = (std::min)((std::min)(a.exponent_, b.exponent_), c.exponent_); + for (int i = c.BigitLength() - 1; i >= min_exponent; --i) { + const Chunk chunk_a = a.BigitOrZero(i); + const Chunk chunk_b = b.BigitOrZero(i); + const Chunk chunk_c = c.BigitOrZero(i); + const Chunk sum = chunk_a + chunk_b; + if (sum > chunk_c + borrow) { + return +1; + } else { + borrow = chunk_c + borrow - sum; + if (borrow > 1) { + return -1; + } + borrow <<= kBigitSize; + } + } + if (borrow == 0) { + return 0; + } + return -1; +} + + +void Bignum::Clamp() { + while (used_bigits_ > 0 && RawBigit(used_bigits_ - 1) == 0) { + used_bigits_--; + } + if (used_bigits_ == 0) { + // Zero. + exponent_ = 0; + } +} + + +void Bignum::Align(const Bignum& other) { + if (exponent_ > other.exponent_) { + // If "X" represents a "hidden" bigit (by the exponent) then we are in the + // following case (a == this, b == other): + // a: aaaaaaXXXX or a: aaaaaXXX + // b: bbbbbbX b: bbbbbbbbXX + // We replace some of the hidden digits (X) of a with 0 digits. + // a: aaaaaa000X or a: aaaaa0XX + const int zero_bigits = exponent_ - other.exponent_; + EnsureCapacity(used_bigits_ + zero_bigits); + for (int i = used_bigits_ - 1; i >= 0; --i) { + RawBigit(i + zero_bigits) = RawBigit(i); + } + for (int i = 0; i < zero_bigits; ++i) { + RawBigit(i) = 0; + } + used_bigits_ += static_cast(zero_bigits); + exponent_ -= static_cast(zero_bigits); + + DOUBLE_CONVERSION_ASSERT(used_bigits_ >= 0); + DOUBLE_CONVERSION_ASSERT(exponent_ >= 0); + } +} + + +void Bignum::BigitsShiftLeft(const int shift_amount) { + DOUBLE_CONVERSION_ASSERT(shift_amount < kBigitSize); + DOUBLE_CONVERSION_ASSERT(shift_amount >= 0); + Chunk carry = 0; + for (int i = 0; i < used_bigits_; ++i) { + const Chunk new_carry = RawBigit(i) >> (kBigitSize - shift_amount); + RawBigit(i) = ((RawBigit(i) << shift_amount) + carry) & kBigitMask; + carry = new_carry; + } + if (carry != 0) { + RawBigit(used_bigits_) = carry; + used_bigits_++; + } +} + + +void Bignum::SubtractTimes(const Bignum& other, const int factor) { + DOUBLE_CONVERSION_ASSERT(exponent_ <= other.exponent_); + if (factor < 3) { + for (int i = 0; i < factor; ++i) { + SubtractBignum(other); + } + return; + } + Chunk borrow = 0; + const int exponent_diff = other.exponent_ - exponent_; + for (int i = 0; i < other.used_bigits_; ++i) { + const DoubleChunk product = static_cast(factor) * other.RawBigit(i); + const DoubleChunk remove = borrow + product; + const Chunk difference = RawBigit(i + exponent_diff) - (remove & kBigitMask); + RawBigit(i + exponent_diff) = difference & kBigitMask; + borrow = static_cast((difference >> (kChunkSize - 1)) + + (remove >> kBigitSize)); + } + for (int i = other.used_bigits_ + exponent_diff; i < used_bigits_; ++i) { + if (borrow == 0) { + return; + } + const Chunk difference = RawBigit(i) - borrow; + RawBigit(i) = difference & kBigitMask; + borrow = difference >> (kChunkSize - 1); + } + Clamp(); +} + + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-bignum.h b/intl/icu/source/i18n/double-conversion-bignum.h new file mode 100644 index 0000000000..bae900a15a --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-bignum.h @@ -0,0 +1,170 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_BIGNUM_H_ +#define DOUBLE_CONVERSION_BIGNUM_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +class Bignum { + public: + // 3584 = 128 * 28. We can represent 2^3584 > 10^1000 accurately. + // This bignum can encode much bigger numbers, since it contains an + // exponent. + static const int kMaxSignificantBits = 3584; + + Bignum() : used_bigits_(0), exponent_(0) {} + + void AssignUInt16(const uint16_t value); + void AssignUInt64(uint64_t value); + void AssignBignum(const Bignum& other); + + void AssignDecimalString(const Vector value); + void AssignHexString(const Vector value); + + void AssignPowerUInt16(uint16_t base, const int exponent); + + void AddUInt64(const uint64_t operand); + void AddBignum(const Bignum& other); + // Precondition: this >= other. + void SubtractBignum(const Bignum& other); + + void Square(); + void ShiftLeft(const int shift_amount); + void MultiplyByUInt32(const uint32_t factor); + void MultiplyByUInt64(const uint64_t factor); + void MultiplyByPowerOfTen(const int exponent); + void Times10() { return MultiplyByUInt32(10); } + // Pseudocode: + // int result = this / other; + // this = this % other; + // In the worst case this function is in O(this/other). + uint16_t DivideModuloIntBignum(const Bignum& other); + + bool ToHexString(char* buffer, const int buffer_size) const; + + // Returns + // -1 if a < b, + // 0 if a == b, and + // +1 if a > b. + static int Compare(const Bignum& a, const Bignum& b); + static bool Equal(const Bignum& a, const Bignum& b) { + return Compare(a, b) == 0; + } + static bool LessEqual(const Bignum& a, const Bignum& b) { + return Compare(a, b) <= 0; + } + static bool Less(const Bignum& a, const Bignum& b) { + return Compare(a, b) < 0; + } + // Returns Compare(a + b, c); + static int PlusCompare(const Bignum& a, const Bignum& b, const Bignum& c); + // Returns a + b == c + static bool PlusEqual(const Bignum& a, const Bignum& b, const Bignum& c) { + return PlusCompare(a, b, c) == 0; + } + // Returns a + b <= c + static bool PlusLessEqual(const Bignum& a, const Bignum& b, const Bignum& c) { + return PlusCompare(a, b, c) <= 0; + } + // Returns a + b < c + static bool PlusLess(const Bignum& a, const Bignum& b, const Bignum& c) { + return PlusCompare(a, b, c) < 0; + } + private: + typedef uint32_t Chunk; + typedef uint64_t DoubleChunk; + + static const int kChunkSize = sizeof(Chunk) * 8; + static const int kDoubleChunkSize = sizeof(DoubleChunk) * 8; + // With bigit size of 28 we loose some bits, but a double still fits easily + // into two chunks, and more importantly we can use the Comba multiplication. + static const int kBigitSize = 28; + static const Chunk kBigitMask = (1 << kBigitSize) - 1; + // Every instance allocates kBigitLength chunks on the stack. Bignums cannot + // grow. There are no checks if the stack-allocated space is sufficient. + static const int kBigitCapacity = kMaxSignificantBits / kBigitSize; + + static void EnsureCapacity(const int size) { + if (size > kBigitCapacity) { + DOUBLE_CONVERSION_UNREACHABLE(); + } + } + void Align(const Bignum& other); + void Clamp(); + bool IsClamped() const { + return used_bigits_ == 0 || RawBigit(used_bigits_ - 1) != 0; + } + void Zero() { + used_bigits_ = 0; + exponent_ = 0; + } + // Requires this to have enough capacity (no tests done). + // Updates used_bigits_ if necessary. + // shift_amount must be < kBigitSize. + void BigitsShiftLeft(const int shift_amount); + // BigitLength includes the "hidden" bigits encoded in the exponent. + int BigitLength() const { return used_bigits_ + exponent_; } + Chunk& RawBigit(const int index); + const Chunk& RawBigit(const int index) const; + Chunk BigitOrZero(const int index) const; + void SubtractTimes(const Bignum& other, const int factor); + + // The Bignum's value is value(bigits_buffer_) * 2^(exponent_ * kBigitSize), + // where the value of the buffer consists of the lower kBigitSize bits of + // the first used_bigits_ Chunks in bigits_buffer_, first chunk has lowest + // significant bits. + int16_t used_bigits_; + int16_t exponent_; + Chunk bigits_buffer_[kBigitCapacity]; + + DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN(Bignum); +}; + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_BIGNUM_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-cached-powers.cpp b/intl/icu/source/i18n/double-conversion-cached-powers.cpp new file mode 100644 index 0000000000..3bc35c8aaf --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-cached-powers.cpp @@ -0,0 +1,193 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2006-2008 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include +#include +#include + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +#include "double-conversion-cached-powers.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +namespace PowersOfTenCache { + +struct CachedPower { + uint64_t significand; + int16_t binary_exponent; + int16_t decimal_exponent; +}; + +static const CachedPower kCachedPowers[] = { + {DOUBLE_CONVERSION_UINT64_2PART_C(0xfa8fd5a0, 081c0288), -1220, -348}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xbaaee17f, a23ebf76), -1193, -340}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8b16fb20, 3055ac76), -1166, -332}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xcf42894a, 5dce35ea), -1140, -324}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9a6bb0aa, 55653b2d), -1113, -316}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xe61acf03, 3d1a45df), -1087, -308}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xab70fe17, c79ac6ca), -1060, -300}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xff77b1fc, bebcdc4f), -1034, -292}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xbe5691ef, 416bd60c), -1007, -284}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8dd01fad, 907ffc3c), -980, -276}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd3515c28, 31559a83), -954, -268}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9d71ac8f, ada6c9b5), -927, -260}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xea9c2277, 23ee8bcb), -901, -252}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xaecc4991, 4078536d), -874, -244}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x823c1279, 5db6ce57), -847, -236}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc2109436, 4dfb5637), -821, -228}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9096ea6f, 3848984f), -794, -220}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd77485cb, 25823ac7), -768, -212}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa086cfcd, 97bf97f4), -741, -204}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xef340a98, 172aace5), -715, -196}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb23867fb, 2a35b28e), -688, -188}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x84c8d4df, d2c63f3b), -661, -180}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc5dd4427, 1ad3cdba), -635, -172}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x936b9fce, bb25c996), -608, -164}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xdbac6c24, 7d62a584), -582, -156}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa3ab6658, 0d5fdaf6), -555, -148}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xf3e2f893, dec3f126), -529, -140}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb5b5ada8, aaff80b8), -502, -132}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x87625f05, 6c7c4a8b), -475, -124}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc9bcff60, 34c13053), -449, -116}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x964e858c, 91ba2655), -422, -108}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xdff97724, 70297ebd), -396, -100}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa6dfbd9f, b8e5b88f), -369, -92}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xf8a95fcf, 88747d94), -343, -84}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb9447093, 8fa89bcf), -316, -76}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8a08f0f8, bf0f156b), -289, -68}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xcdb02555, 653131b6), -263, -60}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x993fe2c6, d07b7fac), -236, -52}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xe45c10c4, 2a2b3b06), -210, -44}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xaa242499, 697392d3), -183, -36}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xfd87b5f2, 8300ca0e), -157, -28}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xbce50864, 92111aeb), -130, -20}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8cbccc09, 6f5088cc), -103, -12}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd1b71758, e219652c), -77, -4}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9c400000, 00000000), -50, 4}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xe8d4a510, 00000000), -24, 12}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xad78ebc5, ac620000), 3, 20}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x813f3978, f8940984), 30, 28}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc097ce7b, c90715b3), 56, 36}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8f7e32ce, 7bea5c70), 83, 44}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd5d238a4, abe98068), 109, 52}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9f4f2726, 179a2245), 136, 60}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xed63a231, d4c4fb27), 162, 68}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb0de6538, 8cc8ada8), 189, 76}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x83c7088e, 1aab65db), 216, 84}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc45d1df9, 42711d9a), 242, 92}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x924d692c, a61be758), 269, 100}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xda01ee64, 1a708dea), 295, 108}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa26da399, 9aef774a), 322, 116}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xf209787b, b47d6b85), 348, 124}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb454e4a1, 79dd1877), 375, 132}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x865b8692, 5b9bc5c2), 402, 140}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xc83553c5, c8965d3d), 428, 148}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x952ab45c, fa97a0b3), 455, 156}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xde469fbd, 99a05fe3), 481, 164}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa59bc234, db398c25), 508, 172}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xf6c69a72, a3989f5c), 534, 180}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xb7dcbf53, 54e9bece), 561, 188}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x88fcf317, f22241e2), 588, 196}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xcc20ce9b, d35c78a5), 614, 204}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x98165af3, 7b2153df), 641, 212}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xe2a0b5dc, 971f303a), 667, 220}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xa8d9d153, 5ce3b396), 694, 228}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xfb9b7cd9, a4a7443c), 720, 236}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xbb764c4c, a7a44410), 747, 244}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8bab8eef, b6409c1a), 774, 252}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd01fef10, a657842c), 800, 260}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9b10a4e5, e9913129), 827, 268}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xe7109bfb, a19c0c9d), 853, 276}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xac2820d9, 623bf429), 880, 284}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x80444b5e, 7aa7cf85), 907, 292}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xbf21e440, 03acdd2d), 933, 300}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x8e679c2f, 5e44ff8f), 960, 308}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xd433179d, 9c8cb841), 986, 316}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0x9e19db92, b4e31ba9), 1013, 324}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xeb96bf6e, badf77d9), 1039, 332}, + {DOUBLE_CONVERSION_UINT64_2PART_C(0xaf87023b, 9bf0ee6b), 1066, 340}, +}; + +static const int kCachedPowersOffset = 348; // -1 * the first decimal_exponent. +static const double kD_1_LOG2_10 = 0.30102999566398114; // 1 / lg(10) + +void GetCachedPowerForBinaryExponentRange( + int min_exponent, + int max_exponent, + DiyFp* power, + int* decimal_exponent) { + int kQ = DiyFp::kSignificandSize; + double k = ceil((min_exponent + kQ - 1) * kD_1_LOG2_10); + int foo = kCachedPowersOffset; + int index = + (foo + static_cast(k) - 1) / kDecimalExponentDistance + 1; + DOUBLE_CONVERSION_ASSERT(0 <= index && index < static_cast(DOUBLE_CONVERSION_ARRAY_SIZE(kCachedPowers))); + CachedPower cached_power = kCachedPowers[index]; + DOUBLE_CONVERSION_ASSERT(min_exponent <= cached_power.binary_exponent); + (void) max_exponent; // Mark variable as used. + DOUBLE_CONVERSION_ASSERT(cached_power.binary_exponent <= max_exponent); + *decimal_exponent = cached_power.decimal_exponent; + *power = DiyFp(cached_power.significand, cached_power.binary_exponent); +} + + +void GetCachedPowerForDecimalExponent(int requested_exponent, + DiyFp* power, + int* found_exponent) { + DOUBLE_CONVERSION_ASSERT(kMinDecimalExponent <= requested_exponent); + DOUBLE_CONVERSION_ASSERT(requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance); + int index = + (requested_exponent + kCachedPowersOffset) / kDecimalExponentDistance; + CachedPower cached_power = kCachedPowers[index]; + *power = DiyFp(cached_power.significand, cached_power.binary_exponent); + *found_exponent = cached_power.decimal_exponent; + DOUBLE_CONVERSION_ASSERT(*found_exponent <= requested_exponent); + DOUBLE_CONVERSION_ASSERT(requested_exponent < *found_exponent + kDecimalExponentDistance); +} + +} // namespace PowersOfTenCache + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-cached-powers.h b/intl/icu/source/i18n/double-conversion-cached-powers.h new file mode 100644 index 0000000000..ade27baef8 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-cached-powers.h @@ -0,0 +1,82 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_CACHED_POWERS_H_ +#define DOUBLE_CONVERSION_CACHED_POWERS_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-diy-fp.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +namespace PowersOfTenCache { + + // Not all powers of ten are cached. The decimal exponent of two neighboring + // cached numbers will differ by kDecimalExponentDistance. + static const int kDecimalExponentDistance = 8; + + static const int kMinDecimalExponent = -348; + static const int kMaxDecimalExponent = 340; + + // Returns a cached power-of-ten with a binary exponent in the range + // [min_exponent; max_exponent] (boundaries included). + void GetCachedPowerForBinaryExponentRange(int min_exponent, + int max_exponent, + DiyFp* power, + int* decimal_exponent); + + // Returns a cached power of ten x ~= 10^k such that + // k <= decimal_exponent < k + kCachedPowersDecimalDistance. + // The given decimal_exponent must satisfy + // kMinDecimalExponent <= requested_exponent, and + // requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance. + void GetCachedPowerForDecimalExponent(int requested_exponent, + DiyFp* power, + int* found_exponent); + +} // namespace PowersOfTenCache + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_CACHED_POWERS_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-diy-fp.h b/intl/icu/source/i18n/double-conversion-diy-fp.h new file mode 100644 index 0000000000..5820abedf3 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-diy-fp.h @@ -0,0 +1,155 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_DIY_FP_H_ +#define DOUBLE_CONVERSION_DIY_FP_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +// This "Do It Yourself Floating Point" class implements a floating-point number +// with a uint64 significand and an int exponent. Normalized DiyFp numbers will +// have the most significant bit of the significand set. +// Multiplication and Subtraction do not normalize their results. +// DiyFp store only non-negative numbers and are not designed to contain special +// doubles (NaN and Infinity). +class DiyFp { + public: + static const int kSignificandSize = 64; + + DiyFp() : f_(0), e_(0) {} + DiyFp(const uint64_t significand, const int32_t exponent) : f_(significand), e_(exponent) {} + + // this -= other. + // The exponents of both numbers must be the same and the significand of this + // must be greater or equal than the significand of other. + // The result will not be normalized. + void Subtract(const DiyFp& other) { + DOUBLE_CONVERSION_ASSERT(e_ == other.e_); + DOUBLE_CONVERSION_ASSERT(f_ >= other.f_); + f_ -= other.f_; + } + + // Returns a - b. + // The exponents of both numbers must be the same and a must be greater + // or equal than b. The result will not be normalized. + static DiyFp Minus(const DiyFp& a, const DiyFp& b) { + DiyFp result = a; + result.Subtract(b); + return result; + } + + // this *= other. + void Multiply(const DiyFp& other) { + // Simply "emulates" a 128 bit multiplication. + // However: the resulting number only contains 64 bits. The least + // significant 64 bits are only used for rounding the most significant 64 + // bits. + const uint64_t kM32 = 0xFFFFFFFFU; + const uint64_t a = f_ >> 32; + const uint64_t b = f_ & kM32; + const uint64_t c = other.f_ >> 32; + const uint64_t d = other.f_ & kM32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + // By adding 1U << 31 to tmp we round the final result. + // Halfway cases will be rounded up. + const uint64_t tmp = (bd >> 32) + (ad & kM32) + (bc & kM32) + (1U << 31); + e_ += other.e_ + 64; + f_ = ac + (ad >> 32) + (bc >> 32) + (tmp >> 32); + } + + // returns a * b; + static DiyFp Times(const DiyFp& a, const DiyFp& b) { + DiyFp result = a; + result.Multiply(b); + return result; + } + + void Normalize() { + DOUBLE_CONVERSION_ASSERT(f_ != 0); + uint64_t significand = f_; + int32_t exponent = e_; + + // This method is mainly called for normalizing boundaries. In general, + // boundaries need to be shifted by 10 bits, and we optimize for this case. + const uint64_t k10MSBits = DOUBLE_CONVERSION_UINT64_2PART_C(0xFFC00000, 00000000); + while ((significand & k10MSBits) == 0) { + significand <<= 10; + exponent -= 10; + } + while ((significand & kUint64MSB) == 0) { + significand <<= 1; + exponent--; + } + f_ = significand; + e_ = exponent; + } + + static DiyFp Normalize(const DiyFp& a) { + DiyFp result = a; + result.Normalize(); + return result; + } + + uint64_t f() const { return f_; } + int32_t e() const { return e_; } + + void set_f(uint64_t new_value) { f_ = new_value; } + void set_e(int32_t new_value) { e_ = new_value; } + + private: + static const uint64_t kUint64MSB = DOUBLE_CONVERSION_UINT64_2PART_C(0x80000000, 00000000); + + uint64_t f_; + int32_t e_; +}; + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_DIY_FP_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-double-to-string.cpp b/intl/icu/source/i18n/double-conversion-double-to-string.cpp new file mode 100644 index 0000000000..5ee6d2b8e8 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-double-to-string.cpp @@ -0,0 +1,462 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include +#include +#include + +// ICU PATCH: Customize header file paths for ICU. +// The file fixed-dtoa.h is not needed. + +#include "double-conversion-double-to-string.h" + +#include "double-conversion-bignum-dtoa.h" +#include "double-conversion-fast-dtoa.h" +#include "double-conversion-ieee.h" +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +#if 0 // not needed for ICU +const DoubleToStringConverter& DoubleToStringConverter::EcmaScriptConverter() { + int flags = UNIQUE_ZERO | EMIT_POSITIVE_EXPONENT_SIGN; + static DoubleToStringConverter converter(flags, + "Infinity", + "NaN", + 'e', + -6, 21, + 6, 0); + return converter; +} + + +bool DoubleToStringConverter::HandleSpecialValues( + double value, + StringBuilder* result_builder) const { + Double double_inspect(value); + if (double_inspect.IsInfinite()) { + if (infinity_symbol_ == DOUBLE_CONVERSION_NULLPTR) return false; + if (value < 0) { + result_builder->AddCharacter('-'); + } + result_builder->AddString(infinity_symbol_); + return true; + } + if (double_inspect.IsNan()) { + if (nan_symbol_ == DOUBLE_CONVERSION_NULLPTR) return false; + result_builder->AddString(nan_symbol_); + return true; + } + return false; +} + + +void DoubleToStringConverter::CreateExponentialRepresentation( + const char* decimal_digits, + int length, + int exponent, + StringBuilder* result_builder) const { + DOUBLE_CONVERSION_ASSERT(length != 0); + result_builder->AddCharacter(decimal_digits[0]); + if (length != 1) { + result_builder->AddCharacter('.'); + result_builder->AddSubstring(&decimal_digits[1], length-1); + } + result_builder->AddCharacter(exponent_character_); + if (exponent < 0) { + result_builder->AddCharacter('-'); + exponent = -exponent; + } else { + if ((flags_ & EMIT_POSITIVE_EXPONENT_SIGN) != 0) { + result_builder->AddCharacter('+'); + } + } + DOUBLE_CONVERSION_ASSERT(exponent < 1e4); + // Changing this constant requires updating the comment of DoubleToStringConverter constructor + const int kMaxExponentLength = 5; + char buffer[kMaxExponentLength + 1]; + buffer[kMaxExponentLength] = '\0'; + int first_char_pos = kMaxExponentLength; + if (exponent == 0) { + buffer[--first_char_pos] = '0'; + } else { + while (exponent > 0) { + buffer[--first_char_pos] = '0' + (exponent % 10); + exponent /= 10; + } + } + // Add prefix '0' to make exponent width >= min(min_exponent_with_, kMaxExponentLength) + // For example: convert 1e+9 -> 1e+09, if min_exponent_with_ is set to 2 + while(kMaxExponentLength - first_char_pos < std::min(min_exponent_width_, kMaxExponentLength)) { + buffer[--first_char_pos] = '0'; + } + result_builder->AddSubstring(&buffer[first_char_pos], + kMaxExponentLength - first_char_pos); +} + + +void DoubleToStringConverter::CreateDecimalRepresentation( + const char* decimal_digits, + int length, + int decimal_point, + int digits_after_point, + StringBuilder* result_builder) const { + // Create a representation that is padded with zeros if needed. + if (decimal_point <= 0) { + // "0.00000decimal_rep" or "0.000decimal_rep00". + result_builder->AddCharacter('0'); + if (digits_after_point > 0) { + result_builder->AddCharacter('.'); + result_builder->AddPadding('0', -decimal_point); + DOUBLE_CONVERSION_ASSERT(length <= digits_after_point - (-decimal_point)); + result_builder->AddSubstring(decimal_digits, length); + int remaining_digits = digits_after_point - (-decimal_point) - length; + result_builder->AddPadding('0', remaining_digits); + } + } else if (decimal_point >= length) { + // "decimal_rep0000.00000" or "decimal_rep.0000". + result_builder->AddSubstring(decimal_digits, length); + result_builder->AddPadding('0', decimal_point - length); + if (digits_after_point > 0) { + result_builder->AddCharacter('.'); + result_builder->AddPadding('0', digits_after_point); + } + } else { + // "decima.l_rep000". + DOUBLE_CONVERSION_ASSERT(digits_after_point > 0); + result_builder->AddSubstring(decimal_digits, decimal_point); + result_builder->AddCharacter('.'); + DOUBLE_CONVERSION_ASSERT(length - decimal_point <= digits_after_point); + result_builder->AddSubstring(&decimal_digits[decimal_point], + length - decimal_point); + int remaining_digits = digits_after_point - (length - decimal_point); + result_builder->AddPadding('0', remaining_digits); + } + if (digits_after_point == 0) { + if ((flags_ & EMIT_TRAILING_DECIMAL_POINT) != 0) { + result_builder->AddCharacter('.'); + } + if ((flags_ & EMIT_TRAILING_ZERO_AFTER_POINT) != 0) { + result_builder->AddCharacter('0'); + } + } +} + + +bool DoubleToStringConverter::ToShortestIeeeNumber( + double value, + StringBuilder* result_builder, + DoubleToStringConverter::DtoaMode mode) const { + DOUBLE_CONVERSION_ASSERT(mode == SHORTEST || mode == SHORTEST_SINGLE); + if (Double(value).IsSpecial()) { + return HandleSpecialValues(value, result_builder); + } + + int decimal_point; + bool sign; + const int kDecimalRepCapacity = kBase10MaximalLength + 1; + char decimal_rep[kDecimalRepCapacity]; + int decimal_rep_length; + + DoubleToAscii(value, mode, 0, decimal_rep, kDecimalRepCapacity, + &sign, &decimal_rep_length, &decimal_point); + + bool unique_zero = (flags_ & UNIQUE_ZERO) != 0; + if (sign && (value != 0.0 || !unique_zero)) { + result_builder->AddCharacter('-'); + } + + int exponent = decimal_point - 1; + if ((decimal_in_shortest_low_ <= exponent) && + (exponent < decimal_in_shortest_high_)) { + CreateDecimalRepresentation(decimal_rep, decimal_rep_length, + decimal_point, + (std::max)(0, decimal_rep_length - decimal_point), + result_builder); + } else { + CreateExponentialRepresentation(decimal_rep, decimal_rep_length, exponent, + result_builder); + } + return true; +} + + +bool DoubleToStringConverter::ToFixed(double value, + int requested_digits, + StringBuilder* result_builder) const { + DOUBLE_CONVERSION_ASSERT(kMaxFixedDigitsBeforePoint == 60); + const double kFirstNonFixed = 1e60; + + if (Double(value).IsSpecial()) { + return HandleSpecialValues(value, result_builder); + } + + if (requested_digits > kMaxFixedDigitsAfterPoint) return false; + if (value >= kFirstNonFixed || value <= -kFirstNonFixed) return false; + + // Find a sufficiently precise decimal representation of n. + int decimal_point; + bool sign; + // Add space for the '\0' byte. + const int kDecimalRepCapacity = + kMaxFixedDigitsBeforePoint + kMaxFixedDigitsAfterPoint + 1; + char decimal_rep[kDecimalRepCapacity]; + int decimal_rep_length; + DoubleToAscii(value, FIXED, requested_digits, + decimal_rep, kDecimalRepCapacity, + &sign, &decimal_rep_length, &decimal_point); + + bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0); + if (sign && (value != 0.0 || !unique_zero)) { + result_builder->AddCharacter('-'); + } + + CreateDecimalRepresentation(decimal_rep, decimal_rep_length, decimal_point, + requested_digits, result_builder); + return true; +} + + +bool DoubleToStringConverter::ToExponential( + double value, + int requested_digits, + StringBuilder* result_builder) const { + if (Double(value).IsSpecial()) { + return HandleSpecialValues(value, result_builder); + } + + if (requested_digits < -1) return false; + if (requested_digits > kMaxExponentialDigits) return false; + + int decimal_point; + bool sign; + // Add space for digit before the decimal point and the '\0' character. + const int kDecimalRepCapacity = kMaxExponentialDigits + 2; + DOUBLE_CONVERSION_ASSERT(kDecimalRepCapacity > kBase10MaximalLength); + char decimal_rep[kDecimalRepCapacity]; +#ifndef NDEBUG + // Problem: there is an assert in StringBuilder::AddSubstring() that + // will pass this buffer to strlen(), and this buffer is not generally + // null-terminated. + memset(decimal_rep, 0, sizeof(decimal_rep)); +#endif + int decimal_rep_length; + + if (requested_digits == -1) { + DoubleToAscii(value, SHORTEST, 0, + decimal_rep, kDecimalRepCapacity, + &sign, &decimal_rep_length, &decimal_point); + } else { + DoubleToAscii(value, PRECISION, requested_digits + 1, + decimal_rep, kDecimalRepCapacity, + &sign, &decimal_rep_length, &decimal_point); + DOUBLE_CONVERSION_ASSERT(decimal_rep_length <= requested_digits + 1); + + for (int i = decimal_rep_length; i < requested_digits + 1; ++i) { + decimal_rep[i] = '0'; + } + decimal_rep_length = requested_digits + 1; + } + + bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0); + if (sign && (value != 0.0 || !unique_zero)) { + result_builder->AddCharacter('-'); + } + + int exponent = decimal_point - 1; + CreateExponentialRepresentation(decimal_rep, + decimal_rep_length, + exponent, + result_builder); + return true; +} + + +bool DoubleToStringConverter::ToPrecision(double value, + int precision, + StringBuilder* result_builder) const { + if (Double(value).IsSpecial()) { + return HandleSpecialValues(value, result_builder); + } + + if (precision < kMinPrecisionDigits || precision > kMaxPrecisionDigits) { + return false; + } + + // Find a sufficiently precise decimal representation of n. + int decimal_point; + bool sign; + // Add one for the terminating null character. + const int kDecimalRepCapacity = kMaxPrecisionDigits + 1; + char decimal_rep[kDecimalRepCapacity]; + int decimal_rep_length; + + DoubleToAscii(value, PRECISION, precision, + decimal_rep, kDecimalRepCapacity, + &sign, &decimal_rep_length, &decimal_point); + DOUBLE_CONVERSION_ASSERT(decimal_rep_length <= precision); + + bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0); + if (sign && (value != 0.0 || !unique_zero)) { + result_builder->AddCharacter('-'); + } + + // The exponent if we print the number as x.xxeyyy. That is with the + // decimal point after the first digit. + int exponent = decimal_point - 1; + + int extra_zero = ((flags_ & EMIT_TRAILING_ZERO_AFTER_POINT) != 0) ? 1 : 0; + bool as_exponential = + (-decimal_point + 1 > max_leading_padding_zeroes_in_precision_mode_) || + (decimal_point - precision + extra_zero > + max_trailing_padding_zeroes_in_precision_mode_); + if ((flags_ & NO_TRAILING_ZERO) != 0) { + // Truncate trailing zeros that occur after the decimal point (if exponential, + // that is everything after the first digit). + int stop = as_exponential ? 1 : std::max(1, decimal_point); + while (decimal_rep_length > stop && decimal_rep[decimal_rep_length - 1] == '0') { + --decimal_rep_length; + } + // Clamp precision to avoid the code below re-adding the zeros. + precision = std::min(precision, decimal_rep_length); + } + if (as_exponential) { + // Fill buffer to contain 'precision' digits. + // Usually the buffer is already at the correct length, but 'DoubleToAscii' + // is allowed to return less characters. + for (int i = decimal_rep_length; i < precision; ++i) { + decimal_rep[i] = '0'; + } + + CreateExponentialRepresentation(decimal_rep, + precision, + exponent, + result_builder); + } else { + CreateDecimalRepresentation(decimal_rep, decimal_rep_length, decimal_point, + (std::max)(0, precision - decimal_point), + result_builder); + } + return true; +} +#endif // not needed for ICU + + +static BignumDtoaMode DtoaToBignumDtoaMode( + DoubleToStringConverter::DtoaMode dtoa_mode) { + switch (dtoa_mode) { + case DoubleToStringConverter::SHORTEST: return BIGNUM_DTOA_SHORTEST; + case DoubleToStringConverter::SHORTEST_SINGLE: + return BIGNUM_DTOA_SHORTEST_SINGLE; + case DoubleToStringConverter::FIXED: return BIGNUM_DTOA_FIXED; + case DoubleToStringConverter::PRECISION: return BIGNUM_DTOA_PRECISION; + default: + DOUBLE_CONVERSION_UNREACHABLE(); + } +} + + +void DoubleToStringConverter::DoubleToAscii(double v, + DtoaMode mode, + int requested_digits, + char* buffer, + int buffer_length, + bool* sign, + int* length, + int* point) { + Vector vector(buffer, buffer_length); + DOUBLE_CONVERSION_ASSERT(!Double(v).IsSpecial()); + DOUBLE_CONVERSION_ASSERT(mode == SHORTEST || mode == SHORTEST_SINGLE || requested_digits >= 0); + + if (Double(v).Sign() < 0) { + *sign = true; + v = -v; + } else { + *sign = false; + } + + if (mode == PRECISION && requested_digits == 0) { + vector[0] = '\0'; + *length = 0; + return; + } + + if (v == 0) { + vector[0] = '0'; + vector[1] = '\0'; + *length = 1; + *point = 1; + return; + } + + bool fast_worked; + switch (mode) { + case SHORTEST: + fast_worked = FastDtoa(v, FAST_DTOA_SHORTEST, 0, vector, length, point); + break; +#if 0 // not needed for ICU + case SHORTEST_SINGLE: + fast_worked = FastDtoa(v, FAST_DTOA_SHORTEST_SINGLE, 0, + vector, length, point); + break; + case FIXED: + fast_worked = FastFixedDtoa(v, requested_digits, vector, length, point); + break; + case PRECISION: + fast_worked = FastDtoa(v, FAST_DTOA_PRECISION, requested_digits, + vector, length, point); + break; +#endif // not needed for ICU + default: + fast_worked = false; + DOUBLE_CONVERSION_UNREACHABLE(); + } + if (fast_worked) return; + + // If the fast dtoa didn't succeed use the slower bignum version. + BignumDtoaMode bignum_mode = DtoaToBignumDtoaMode(mode); + BignumDtoa(v, bignum_mode, requested_digits, vector, length, point); + vector[*length] = '\0'; +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-double-to-string.h b/intl/icu/source/i18n/double-conversion-double-to-string.h new file mode 100644 index 0000000000..9940052c64 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-double-to-string.h @@ -0,0 +1,468 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2012 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_DOUBLE_TO_STRING_H_ +#define DOUBLE_CONVERSION_DOUBLE_TO_STRING_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +class DoubleToStringConverter { + public: + // When calling ToFixed with a double > 10^kMaxFixedDigitsBeforePoint + // or a requested_digits parameter > kMaxFixedDigitsAfterPoint then the + // function returns false. + static const int kMaxFixedDigitsBeforePoint = 60; + static const int kMaxFixedDigitsAfterPoint = 100; + + // When calling ToExponential with a requested_digits + // parameter > kMaxExponentialDigits then the function returns false. + static const int kMaxExponentialDigits = 120; + + // When calling ToPrecision with a requested_digits + // parameter < kMinPrecisionDigits or requested_digits > kMaxPrecisionDigits + // then the function returns false. + static const int kMinPrecisionDigits = 1; + static const int kMaxPrecisionDigits = 120; + + // The maximal number of digits that are needed to emit a double in base 10. + // A higher precision can be achieved by using more digits, but the shortest + // accurate representation of any double will never use more digits than + // kBase10MaximalLength. + // Note that DoubleToAscii null-terminates its input. So the given buffer + // should be at least kBase10MaximalLength + 1 characters long. + static const int kBase10MaximalLength = 17; + + // The maximal number of digits that are needed to emit a single in base 10. + // A higher precision can be achieved by using more digits, but the shortest + // accurate representation of any single will never use more digits than + // kBase10MaximalLengthSingle. + static const int kBase10MaximalLengthSingle = 9; + + // The length of the longest string that 'ToShortest' can produce when the + // converter is instantiated with EcmaScript defaults (see + // 'EcmaScriptConverter') + // This value does not include the trailing '\0' character. + // This amount of characters is needed for negative values that hit the + // 'decimal_in_shortest_low' limit. For example: "-0.0000033333333333333333" + static const int kMaxCharsEcmaScriptShortest = 25; + +#if 0 // not needed for ICU + enum Flags { + NO_FLAGS = 0, + EMIT_POSITIVE_EXPONENT_SIGN = 1, + EMIT_TRAILING_DECIMAL_POINT = 2, + EMIT_TRAILING_ZERO_AFTER_POINT = 4, + UNIQUE_ZERO = 8, + NO_TRAILING_ZERO = 16 + }; + + // Flags should be a bit-or combination of the possible Flags-enum. + // - NO_FLAGS: no special flags. + // - EMIT_POSITIVE_EXPONENT_SIGN: when the number is converted into exponent + // form, emits a '+' for positive exponents. Example: 1.2e+2. + // - EMIT_TRAILING_DECIMAL_POINT: when the input number is an integer and is + // converted into decimal format then a trailing decimal point is appended. + // Example: 2345.0 is converted to "2345.". + // - EMIT_TRAILING_ZERO_AFTER_POINT: in addition to a trailing decimal point + // emits a trailing '0'-character. This flag requires the + // EMIT_TRAILING_DECIMAL_POINT flag. + // Example: 2345.0 is converted to "2345.0". + // - UNIQUE_ZERO: "-0.0" is converted to "0.0". + // - NO_TRAILING_ZERO: Trailing zeros are removed from the fractional portion + // of the result in precision mode. Matches printf's %g. + // When EMIT_TRAILING_ZERO_AFTER_POINT is also given, one trailing zero is + // preserved. + // + // Infinity symbol and nan_symbol provide the string representation for these + // special values. If the string is nullptr and the special value is encountered + // then the conversion functions return false. + // + // The exponent_character is used in exponential representations. It is + // usually 'e' or 'E'. + // + // When converting to the shortest representation the converter will + // represent input numbers in decimal format if they are in the interval + // [10^decimal_in_shortest_low; 10^decimal_in_shortest_high[ + // (lower boundary included, greater boundary excluded). + // Example: with decimal_in_shortest_low = -6 and + // decimal_in_shortest_high = 21: + // ToShortest(0.000001) -> "0.000001" + // ToShortest(0.0000001) -> "1e-7" + // ToShortest(111111111111111111111.0) -> "111111111111111110000" + // ToShortest(100000000000000000000.0) -> "100000000000000000000" + // ToShortest(1111111111111111111111.0) -> "1.1111111111111111e+21" + // + // When converting to precision mode the converter may add + // max_leading_padding_zeroes before returning the number in exponential + // format. + // Example with max_leading_padding_zeroes_in_precision_mode = 6. + // ToPrecision(0.0000012345, 2) -> "0.0000012" + // ToPrecision(0.00000012345, 2) -> "1.2e-7" + // Similarly the converter may add up to + // max_trailing_padding_zeroes_in_precision_mode in precision mode to avoid + // returning an exponential representation. A zero added by the + // EMIT_TRAILING_ZERO_AFTER_POINT flag is counted for this limit. + // Examples for max_trailing_padding_zeroes_in_precision_mode = 1: + // ToPrecision(230.0, 2) -> "230" + // ToPrecision(230.0, 2) -> "230." with EMIT_TRAILING_DECIMAL_POINT. + // ToPrecision(230.0, 2) -> "2.3e2" with EMIT_TRAILING_ZERO_AFTER_POINT. + // + // The min_exponent_width is used for exponential representations. + // The converter adds leading '0's to the exponent until the exponent + // is at least min_exponent_width digits long. + // The min_exponent_width is clamped to 5. + // As such, the exponent may never have more than 5 digits in total. + DoubleToStringConverter(int flags, + const char* infinity_symbol, + const char* nan_symbol, + char exponent_character, + int decimal_in_shortest_low, + int decimal_in_shortest_high, + int max_leading_padding_zeroes_in_precision_mode, + int max_trailing_padding_zeroes_in_precision_mode, + int min_exponent_width = 0) + : flags_(flags), + infinity_symbol_(infinity_symbol), + nan_symbol_(nan_symbol), + exponent_character_(exponent_character), + decimal_in_shortest_low_(decimal_in_shortest_low), + decimal_in_shortest_high_(decimal_in_shortest_high), + max_leading_padding_zeroes_in_precision_mode_( + max_leading_padding_zeroes_in_precision_mode), + max_trailing_padding_zeroes_in_precision_mode_( + max_trailing_padding_zeroes_in_precision_mode), + min_exponent_width_(min_exponent_width) { + // When 'trailing zero after the point' is set, then 'trailing point' + // must be set too. + DOUBLE_CONVERSION_ASSERT(((flags & EMIT_TRAILING_DECIMAL_POINT) != 0) || + !((flags & EMIT_TRAILING_ZERO_AFTER_POINT) != 0)); + } + + // Returns a converter following the EcmaScript specification. + // + // Flags: UNIQUE_ZERO and EMIT_POSITIVE_EXPONENT_SIGN. + // Special values: "Infinity" and "NaN". + // Lower case 'e' for exponential values. + // decimal_in_shortest_low: -6 + // decimal_in_shortest_high: 21 + // max_leading_padding_zeroes_in_precision_mode: 6 + // max_trailing_padding_zeroes_in_precision_mode: 0 + static const DoubleToStringConverter& EcmaScriptConverter(); + + // Computes the shortest string of digits that correctly represent the input + // number. Depending on decimal_in_shortest_low and decimal_in_shortest_high + // (see constructor) it then either returns a decimal representation, or an + // exponential representation. + // Example with decimal_in_shortest_low = -6, + // decimal_in_shortest_high = 21, + // EMIT_POSITIVE_EXPONENT_SIGN activated, and + // EMIT_TRAILING_DECIMAL_POINT deactivated: + // ToShortest(0.000001) -> "0.000001" + // ToShortest(0.0000001) -> "1e-7" + // ToShortest(111111111111111111111.0) -> "111111111111111110000" + // ToShortest(100000000000000000000.0) -> "100000000000000000000" + // ToShortest(1111111111111111111111.0) -> "1.1111111111111111e+21" + // + // Note: the conversion may round the output if the returned string + // is accurate enough to uniquely identify the input-number. + // For example the most precise representation of the double 9e59 equals + // "899999999999999918767229449717619953810131273674690656206848", but + // the converter will return the shorter (but still correct) "9e59". + // + // Returns true if the conversion succeeds. The conversion always succeeds + // except when the input value is special and no infinity_symbol or + // nan_symbol has been given to the constructor. + // + // The length of the longest result is the maximum of the length of the + // following string representations (each with possible examples): + // - NaN and negative infinity: "NaN", "-Infinity", "-inf". + // - -10^(decimal_in_shortest_high - 1): + // "-100000000000000000000", "-1000000000000000.0" + // - the longest string in range [0; -10^decimal_in_shortest_low]. Generally, + // this string is 3 + kBase10MaximalLength - decimal_in_shortest_low. + // (Sign, '0', decimal point, padding zeroes for decimal_in_shortest_low, + // and the significant digits). + // "-0.0000033333333333333333", "-0.0012345678901234567" + // - the longest exponential representation. (A negative number with + // kBase10MaximalLength significant digits). + // "-1.7976931348623157e+308", "-1.7976931348623157E308" + // In addition, the buffer must be able to hold the trailing '\0' character. + bool ToShortest(double value, StringBuilder* result_builder) const { + return ToShortestIeeeNumber(value, result_builder, SHORTEST); + } + + // Same as ToShortest, but for single-precision floats. + bool ToShortestSingle(float value, StringBuilder* result_builder) const { + return ToShortestIeeeNumber(value, result_builder, SHORTEST_SINGLE); + } + + + // Computes a decimal representation with a fixed number of digits after the + // decimal point. The last emitted digit is rounded. + // + // Examples: + // ToFixed(3.12, 1) -> "3.1" + // ToFixed(3.1415, 3) -> "3.142" + // ToFixed(1234.56789, 4) -> "1234.5679" + // ToFixed(1.23, 5) -> "1.23000" + // ToFixed(0.1, 4) -> "0.1000" + // ToFixed(1e30, 2) -> "1000000000000000019884624838656.00" + // ToFixed(0.1, 30) -> "0.100000000000000005551115123126" + // ToFixed(0.1, 17) -> "0.10000000000000001" + // + // If requested_digits equals 0, then the tail of the result depends on + // the EMIT_TRAILING_DECIMAL_POINT and EMIT_TRAILING_ZERO_AFTER_POINT. + // Examples, for requested_digits == 0, + // let EMIT_TRAILING_DECIMAL_POINT and EMIT_TRAILING_ZERO_AFTER_POINT be + // - false and false: then 123.45 -> 123 + // 0.678 -> 1 + // - true and false: then 123.45 -> 123. + // 0.678 -> 1. + // - true and true: then 123.45 -> 123.0 + // 0.678 -> 1.0 + // + // Returns true if the conversion succeeds. The conversion always succeeds + // except for the following cases: + // - the input value is special and no infinity_symbol or nan_symbol has + // been provided to the constructor, + // - 'value' > 10^kMaxFixedDigitsBeforePoint, or + // - 'requested_digits' > kMaxFixedDigitsAfterPoint. + // The last two conditions imply that the result for non-special values never + // contains more than + // 1 + kMaxFixedDigitsBeforePoint + 1 + kMaxFixedDigitsAfterPoint characters + // (one additional character for the sign, and one for the decimal point). + // In addition, the buffer must be able to hold the trailing '\0' character. + bool ToFixed(double value, + int requested_digits, + StringBuilder* result_builder) const; + + // Computes a representation in exponential format with requested_digits + // after the decimal point. The last emitted digit is rounded. + // If requested_digits equals -1, then the shortest exponential representation + // is computed. + // + // Examples with EMIT_POSITIVE_EXPONENT_SIGN deactivated, and + // exponent_character set to 'e'. + // ToExponential(3.12, 1) -> "3.1e0" + // ToExponential(5.0, 3) -> "5.000e0" + // ToExponential(0.001, 2) -> "1.00e-3" + // ToExponential(3.1415, -1) -> "3.1415e0" + // ToExponential(3.1415, 4) -> "3.1415e0" + // ToExponential(3.1415, 3) -> "3.142e0" + // ToExponential(123456789000000, 3) -> "1.235e14" + // ToExponential(1000000000000000019884624838656.0, -1) -> "1e30" + // ToExponential(1000000000000000019884624838656.0, 32) -> + // "1.00000000000000001988462483865600e30" + // ToExponential(1234, 0) -> "1e3" + // + // Returns true if the conversion succeeds. The conversion always succeeds + // except for the following cases: + // - the input value is special and no infinity_symbol or nan_symbol has + // been provided to the constructor, + // - 'requested_digits' > kMaxExponentialDigits. + // + // The last condition implies that the result never contains more than + // kMaxExponentialDigits + 8 characters (the sign, the digit before the + // decimal point, the decimal point, the exponent character, the + // exponent's sign, and at most 3 exponent digits). + // In addition, the buffer must be able to hold the trailing '\0' character. + bool ToExponential(double value, + int requested_digits, + StringBuilder* result_builder) const; + + + // Computes 'precision' leading digits of the given 'value' and returns them + // either in exponential or decimal format, depending on + // max_{leading|trailing}_padding_zeroes_in_precision_mode (given to the + // constructor). + // The last computed digit is rounded. + // + // Example with max_leading_padding_zeroes_in_precision_mode = 6. + // ToPrecision(0.0000012345, 2) -> "0.0000012" + // ToPrecision(0.00000012345, 2) -> "1.2e-7" + // Similarly the converter may add up to + // max_trailing_padding_zeroes_in_precision_mode in precision mode to avoid + // returning an exponential representation. A zero added by the + // EMIT_TRAILING_ZERO_AFTER_POINT flag is counted for this limit. + // Examples for max_trailing_padding_zeroes_in_precision_mode = 1: + // ToPrecision(230.0, 2) -> "230" + // ToPrecision(230.0, 2) -> "230." with EMIT_TRAILING_DECIMAL_POINT. + // ToPrecision(230.0, 2) -> "2.3e2" with EMIT_TRAILING_ZERO_AFTER_POINT. + // Examples for max_trailing_padding_zeroes_in_precision_mode = 3, and no + // EMIT_TRAILING_ZERO_AFTER_POINT: + // ToPrecision(123450.0, 6) -> "123450" + // ToPrecision(123450.0, 5) -> "123450" + // ToPrecision(123450.0, 4) -> "123500" + // ToPrecision(123450.0, 3) -> "123000" + // ToPrecision(123450.0, 2) -> "1.2e5" + // + // Returns true if the conversion succeeds. The conversion always succeeds + // except for the following cases: + // - the input value is special and no infinity_symbol or nan_symbol has + // been provided to the constructor, + // - precision < kMinPericisionDigits + // - precision > kMaxPrecisionDigits + // + // The last condition implies that the result never contains more than + // kMaxPrecisionDigits + 7 characters (the sign, the decimal point, the + // exponent character, the exponent's sign, and at most 3 exponent digits). + // In addition, the buffer must be able to hold the trailing '\0' character. + bool ToPrecision(double value, + int precision, + StringBuilder* result_builder) const; +#endif // not needed for ICU + + enum DtoaMode { + // Produce the shortest correct representation. + // For example the output of 0.299999999999999988897 is (the less accurate + // but correct) 0.3. + SHORTEST, + // Same as SHORTEST, but for single-precision floats. + SHORTEST_SINGLE, + // Produce a fixed number of digits after the decimal point. + // For instance fixed(0.1, 4) becomes 0.1000 + // If the input number is big, the output will be big. + FIXED, + // Fixed number of digits (independent of the decimal point). + PRECISION + }; + + // Converts the given double 'v' to digit characters. 'v' must not be NaN, + // +Infinity, or -Infinity. In SHORTEST_SINGLE-mode this restriction also + // applies to 'v' after it has been casted to a single-precision float. That + // is, in this mode static_cast(v) must not be NaN, +Infinity or + // -Infinity. + // + // The result should be interpreted as buffer * 10^(point-length). + // + // The digits are written to the buffer in the platform's charset, which is + // often UTF-8 (with ASCII-range digits) but may be another charset, such + // as EBCDIC. + // + // The output depends on the given mode: + // - SHORTEST: produce the least amount of digits for which the internal + // identity requirement is still satisfied. If the digits are printed + // (together with the correct exponent) then reading this number will give + // 'v' again. The buffer will choose the representation that is closest to + // 'v'. If there are two at the same distance, than the one farther away + // from 0 is chosen (halfway cases - ending with 5 - are rounded up). + // In this mode the 'requested_digits' parameter is ignored. + // - SHORTEST_SINGLE: same as SHORTEST but with single-precision. + // - FIXED: produces digits necessary to print a given number with + // 'requested_digits' digits after the decimal point. The produced digits + // might be too short in which case the caller has to fill the remainder + // with '0's. + // Example: toFixed(0.001, 5) is allowed to return buffer="1", point=-2. + // Halfway cases are rounded towards +/-Infinity (away from 0). The call + // toFixed(0.15, 2) thus returns buffer="2", point=0. + // The returned buffer may contain digits that would be truncated from the + // shortest representation of the input. + // - PRECISION: produces 'requested_digits' where the first digit is not '0'. + // Even though the length of produced digits usually equals + // 'requested_digits', the function is allowed to return fewer digits, in + // which case the caller has to fill the missing digits with '0's. + // Halfway cases are again rounded away from 0. + // DoubleToAscii expects the given buffer to be big enough to hold all + // digits and a terminating null-character. In SHORTEST-mode it expects a + // buffer of at least kBase10MaximalLength + 1. In all other modes the + // requested_digits parameter and the padding-zeroes limit the size of the + // output. Don't forget the decimal point, the exponent character and the + // terminating null-character when computing the maximal output size. + // The given length is only used in debug mode to ensure the buffer is big + // enough. + // ICU PATCH: Export this as U_I18N_API for unit tests. + static void U_I18N_API DoubleToAscii(double v, + DtoaMode mode, + int requested_digits, + char* buffer, + int buffer_length, + bool* sign, + int* length, + int* point); + +#if 0 // not needed for ICU + private: + // Implementation for ToShortest and ToShortestSingle. + bool ToShortestIeeeNumber(double value, + StringBuilder* result_builder, + DtoaMode mode) const; + + // If the value is a special value (NaN or Infinity) constructs the + // corresponding string using the configured infinity/nan-symbol. + // If either of them is nullptr or the value is not special then the + // function returns false. + bool HandleSpecialValues(double value, StringBuilder* result_builder) const; + // Constructs an exponential representation (i.e. 1.234e56). + // The given exponent assumes a decimal point after the first decimal digit. + void CreateExponentialRepresentation(const char* decimal_digits, + int length, + int exponent, + StringBuilder* result_builder) const; + // Creates a decimal representation (i.e 1234.5678). + void CreateDecimalRepresentation(const char* decimal_digits, + int length, + int decimal_point, + int digits_after_point, + StringBuilder* result_builder) const; + + const int flags_; + const char* const infinity_symbol_; + const char* const nan_symbol_; + const char exponent_character_; + const int decimal_in_shortest_low_; + const int decimal_in_shortest_high_; + const int max_leading_padding_zeroes_in_precision_mode_; + const int max_trailing_padding_zeroes_in_precision_mode_; + const int min_exponent_width_; +#endif // not needed for ICU + + DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS(DoubleToStringConverter); +}; + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_DOUBLE_TO_STRING_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-fast-dtoa.cpp b/intl/icu/source/i18n/double-conversion-fast-dtoa.cpp new file mode 100644 index 0000000000..06e4cf1255 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-fast-dtoa.cpp @@ -0,0 +1,683 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2012 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-fast-dtoa.h" + +#include "double-conversion-cached-powers.h" +#include "double-conversion-diy-fp.h" +#include "double-conversion-ieee.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +// The minimal and maximal target exponent define the range of w's binary +// exponent, where 'w' is the result of multiplying the input by a cached power +// of ten. +// +// A different range might be chosen on a different platform, to optimize digit +// generation, but a smaller range requires more powers of ten to be cached. +static const int kMinimalTargetExponent = -60; +static const int kMaximalTargetExponent = -32; + + +// Adjusts the last digit of the generated number, and screens out generated +// solutions that may be inaccurate. A solution may be inaccurate if it is +// outside the safe interval, or if we cannot prove that it is closer to the +// input than a neighboring representation of the same length. +// +// Input: * buffer containing the digits of too_high / 10^kappa +// * the buffer's length +// * distance_too_high_w == (too_high - w).f() * unit +// * unsafe_interval == (too_high - too_low).f() * unit +// * rest = (too_high - buffer * 10^kappa).f() * unit +// * ten_kappa = 10^kappa * unit +// * unit = the common multiplier +// Output: returns true if the buffer is guaranteed to contain the closest +// representable number to the input. +// Modifies the generated digits in the buffer to approach (round towards) w. +static bool RoundWeed(Vector buffer, + int length, + uint64_t distance_too_high_w, + uint64_t unsafe_interval, + uint64_t rest, + uint64_t ten_kappa, + uint64_t unit) { + uint64_t small_distance = distance_too_high_w - unit; + uint64_t big_distance = distance_too_high_w + unit; + // Let w_low = too_high - big_distance, and + // w_high = too_high - small_distance. + // Note: w_low < w < w_high + // + // The real w (* unit) must lie somewhere inside the interval + // ]w_low; w_high[ (often written as "(w_low; w_high)") + + // Basically the buffer currently contains a number in the unsafe interval + // ]too_low; too_high[ with too_low < w < too_high + // + // too_high - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // ^v 1 unit ^ ^ ^ ^ + // boundary_high --------------------- . . . . + // ^v 1 unit . . . . + // - - - - - - - - - - - - - - - - - - - + - - + - - - - - - . . + // . . ^ . . + // . big_distance . . . + // . . . . rest + // small_distance . . . . + // v . . . . + // w_high - - - - - - - - - - - - - - - - - - . . . . + // ^v 1 unit . . . . + // w ---------------------------------------- . . . . + // ^v 1 unit v . . . + // w_low - - - - - - - - - - - - - - - - - - - - - . . . + // . . v + // buffer --------------------------------------------------+-------+-------- + // . . + // safe_interval . + // v . + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - . + // ^v 1 unit . + // boundary_low ------------------------- unsafe_interval + // ^v 1 unit v + // too_low - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // + // + // Note that the value of buffer could lie anywhere inside the range too_low + // to too_high. + // + // boundary_low, boundary_high and w are approximations of the real boundaries + // and v (the input number). They are guaranteed to be precise up to one unit. + // In fact the error is guaranteed to be strictly less than one unit. + // + // Anything that lies outside the unsafe interval is guaranteed not to round + // to v when read again. + // Anything that lies inside the safe interval is guaranteed to round to v + // when read again. + // If the number inside the buffer lies inside the unsafe interval but not + // inside the safe interval then we simply do not know and bail out (returning + // false). + // + // Similarly we have to take into account the imprecision of 'w' when finding + // the closest representation of 'w'. If we have two potential + // representations, and one is closer to both w_low and w_high, then we know + // it is closer to the actual value v. + // + // By generating the digits of too_high we got the largest (closest to + // too_high) buffer that is still in the unsafe interval. In the case where + // w_high < buffer < too_high we try to decrement the buffer. + // This way the buffer approaches (rounds towards) w. + // There are 3 conditions that stop the decrementation process: + // 1) the buffer is already below w_high + // 2) decrementing the buffer would make it leave the unsafe interval + // 3) decrementing the buffer would yield a number below w_high and farther + // away than the current number. In other words: + // (buffer{-1} < w_high) && w_high - buffer{-1} > buffer - w_high + // Instead of using the buffer directly we use its distance to too_high. + // Conceptually rest ~= too_high - buffer + // We need to do the following tests in this order to avoid over- and + // underflows. + DOUBLE_CONVERSION_ASSERT(rest <= unsafe_interval); + while (rest < small_distance && // Negated condition 1 + unsafe_interval - rest >= ten_kappa && // Negated condition 2 + (rest + ten_kappa < small_distance || // buffer{-1} > w_high + small_distance - rest >= rest + ten_kappa - small_distance)) { + buffer[length - 1]--; + rest += ten_kappa; + } + + // We have approached w+ as much as possible. We now test if approaching w- + // would require changing the buffer. If yes, then we have two possible + // representations close to w, but we cannot decide which one is closer. + if (rest < big_distance && + unsafe_interval - rest >= ten_kappa && + (rest + ten_kappa < big_distance || + big_distance - rest > rest + ten_kappa - big_distance)) { + return false; + } + + // Weeding test. + // The safe interval is [too_low + 2 ulp; too_high - 2 ulp] + // Since too_low = too_high - unsafe_interval this is equivalent to + // [too_high - unsafe_interval + 4 ulp; too_high - 2 ulp] + // Conceptually we have: rest ~= too_high - buffer + return (2 * unit <= rest) && (rest <= unsafe_interval - 4 * unit); +} + + +// Rounds the buffer upwards if the result is closer to v by possibly adding +// 1 to the buffer. If the precision of the calculation is not sufficient to +// round correctly, return false. +// The rounding might shift the whole buffer in which case the kappa is +// adjusted. For example "99", kappa = 3 might become "10", kappa = 4. +// +// If 2*rest > ten_kappa then the buffer needs to be round up. +// rest can have an error of +/- 1 unit. This function accounts for the +// imprecision and returns false, if the rounding direction cannot be +// unambiguously determined. +// +// Precondition: rest < ten_kappa. +static bool RoundWeedCounted(Vector buffer, + int length, + uint64_t rest, + uint64_t ten_kappa, + uint64_t unit, + int* kappa) { + DOUBLE_CONVERSION_ASSERT(rest < ten_kappa); + // The following tests are done in a specific order to avoid overflows. They + // will work correctly with any uint64 values of rest < ten_kappa and unit. + // + // If the unit is too big, then we don't know which way to round. For example + // a unit of 50 means that the real number lies within rest +/- 50. If + // 10^kappa == 40 then there is no way to tell which way to round. + if (unit >= ten_kappa) return false; + // Even if unit is just half the size of 10^kappa we are already completely + // lost. (And after the previous test we know that the expression will not + // over/underflow.) + if (ten_kappa - unit <= unit) return false; + // If 2 * (rest + unit) <= 10^kappa we can safely round down. + if ((ten_kappa - rest > rest) && (ten_kappa - 2 * rest >= 2 * unit)) { + return true; + } + // If 2 * (rest - unit) >= 10^kappa, then we can safely round up. + if ((rest > unit) && (ten_kappa - (rest - unit) <= (rest - unit))) { + // Increment the last digit recursively until we find a non '9' digit. + buffer[length - 1]++; + for (int i = length - 1; i > 0; --i) { + if (buffer[i] != '0' + 10) break; + buffer[i] = '0'; + buffer[i - 1]++; + } + // If the first digit is now '0'+ 10 we had a buffer with all '9's. With the + // exception of the first digit all digits are now '0'. Simply switch the + // first digit to '1' and adjust the kappa. Example: "99" becomes "10" and + // the power (the kappa) is increased. + if (buffer[0] == '0' + 10) { + buffer[0] = '1'; + (*kappa) += 1; + } + return true; + } + return false; +} + +// Returns the biggest power of ten that is less than or equal to the given +// number. We furthermore receive the maximum number of bits 'number' has. +// +// Returns power == 10^(exponent_plus_one-1) such that +// power <= number < power * 10. +// If number_bits == 0 then 0^(0-1) is returned. +// The number of bits must be <= 32. +// Precondition: number < (1 << (number_bits + 1)). + +// Inspired by the method for finding an integer log base 10 from here: +// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10 +static unsigned int const kSmallPowersOfTen[] = + {0, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, + 1000000000}; + +static void BiggestPowerTen(uint32_t number, + int number_bits, + uint32_t* power, + int* exponent_plus_one) { + DOUBLE_CONVERSION_ASSERT(number < (1u << (number_bits + 1))); + // 1233/4096 is approximately 1/lg(10). + int exponent_plus_one_guess = ((number_bits + 1) * 1233 >> 12); + // We increment to skip over the first entry in the kPowersOf10 table. + // Note: kPowersOf10[i] == 10^(i-1). + exponent_plus_one_guess++; + // We don't have any guarantees that 2^number_bits <= number. + if (number < kSmallPowersOfTen[exponent_plus_one_guess]) { + exponent_plus_one_guess--; + } + *power = kSmallPowersOfTen[exponent_plus_one_guess]; + *exponent_plus_one = exponent_plus_one_guess; +} + +// Generates the digits of input number w. +// w is a floating-point number (DiyFp), consisting of a significand and an +// exponent. Its exponent is bounded by kMinimalTargetExponent and +// kMaximalTargetExponent. +// Hence -60 <= w.e() <= -32. +// +// Returns false if it fails, in which case the generated digits in the buffer +// should not be used. +// Preconditions: +// * low, w and high are correct up to 1 ulp (unit in the last place). That +// is, their error must be less than a unit of their last digits. +// * low.e() == w.e() == high.e() +// * low < w < high, and taking into account their error: low~ <= high~ +// * kMinimalTargetExponent <= w.e() <= kMaximalTargetExponent +// Postconditions: returns false if procedure fails. +// otherwise: +// * buffer is not null-terminated, but len contains the number of digits. +// * buffer contains the shortest possible decimal digit-sequence +// such that LOW < buffer * 10^kappa < HIGH, where LOW and HIGH are the +// correct values of low and high (without their error). +// * if more than one decimal representation gives the minimal number of +// decimal digits then the one closest to W (where W is the correct value +// of w) is chosen. +// Remark: this procedure takes into account the imprecision of its input +// numbers. If the precision is not enough to guarantee all the postconditions +// then false is returned. This usually happens rarely (~0.5%). +// +// Say, for the sake of example, that +// w.e() == -48, and w.f() == 0x1234567890abcdef +// w's value can be computed by w.f() * 2^w.e() +// We can obtain w's integral digits by simply shifting w.f() by -w.e(). +// -> w's integral part is 0x1234 +// w's fractional part is therefore 0x567890abcdef. +// Printing w's integral part is easy (simply print 0x1234 in decimal). +// In order to print its fraction we repeatedly multiply the fraction by 10 and +// get each digit. Example the first digit after the point would be computed by +// (0x567890abcdef * 10) >> 48. -> 3 +// The whole thing becomes slightly more complicated because we want to stop +// once we have enough digits. That is, once the digits inside the buffer +// represent 'w' we can stop. Everything inside the interval low - high +// represents w. However we have to pay attention to low, high and w's +// imprecision. +static bool DigitGen(DiyFp low, + DiyFp w, + DiyFp high, + Vector buffer, + int* length, + int* kappa) { + DOUBLE_CONVERSION_ASSERT(low.e() == w.e() && w.e() == high.e()); + DOUBLE_CONVERSION_ASSERT(low.f() + 1 <= high.f() - 1); + DOUBLE_CONVERSION_ASSERT(kMinimalTargetExponent <= w.e() && w.e() <= kMaximalTargetExponent); + // low, w and high are imprecise, but by less than one ulp (unit in the last + // place). + // If we remove (resp. add) 1 ulp from low (resp. high) we are certain that + // the new numbers are outside of the interval we want the final + // representation to lie in. + // Inversely adding (resp. removing) 1 ulp from low (resp. high) would yield + // numbers that are certain to lie in the interval. We will use this fact + // later on. + // We will now start by generating the digits within the uncertain + // interval. Later we will weed out representations that lie outside the safe + // interval and thus _might_ lie outside the correct interval. + uint64_t unit = 1; + DiyFp too_low = DiyFp(low.f() - unit, low.e()); + DiyFp too_high = DiyFp(high.f() + unit, high.e()); + // too_low and too_high are guaranteed to lie outside the interval we want the + // generated number in. + DiyFp unsafe_interval = DiyFp::Minus(too_high, too_low); + // We now cut the input number into two parts: the integral digits and the + // fractionals. We will not write any decimal separator though, but adapt + // kappa instead. + // Reminder: we are currently computing the digits (stored inside the buffer) + // such that: too_low < buffer * 10^kappa < too_high + // We use too_high for the digit_generation and stop as soon as possible. + // If we stop early we effectively round down. + DiyFp one = DiyFp(static_cast(1) << -w.e(), w.e()); + // Division by one is a shift. + uint32_t integrals = static_cast(too_high.f() >> -one.e()); + // Modulo by one is an and. + uint64_t fractionals = too_high.f() & (one.f() - 1); + uint32_t divisor; + int divisor_exponent_plus_one; + BiggestPowerTen(integrals, DiyFp::kSignificandSize - (-one.e()), + &divisor, &divisor_exponent_plus_one); + *kappa = divisor_exponent_plus_one; + *length = 0; + // Loop invariant: buffer = too_high / 10^kappa (integer division) + // The invariant holds for the first iteration: kappa has been initialized + // with the divisor exponent + 1. And the divisor is the biggest power of ten + // that is smaller than integrals. + while (*kappa > 0) { + int digit = integrals / divisor; + DOUBLE_CONVERSION_ASSERT(digit <= 9); + buffer[*length] = static_cast('0' + digit); + (*length)++; + integrals %= divisor; + (*kappa)--; + // Note that kappa now equals the exponent of the divisor and that the + // invariant thus holds again. + uint64_t rest = + (static_cast(integrals) << -one.e()) + fractionals; + // Invariant: too_high = buffer * 10^kappa + DiyFp(rest, one.e()) + // Reminder: unsafe_interval.e() == one.e() + if (rest < unsafe_interval.f()) { + // Rounding down (by not emitting the remaining digits) yields a number + // that lies within the unsafe interval. + return RoundWeed(buffer, *length, DiyFp::Minus(too_high, w).f(), + unsafe_interval.f(), rest, + static_cast(divisor) << -one.e(), unit); + } + divisor /= 10; + } + + // The integrals have been generated. We are at the point of the decimal + // separator. In the following loop we simply multiply the remaining digits by + // 10 and divide by one. We just need to pay attention to multiply associated + // data (like the interval or 'unit'), too. + // Note that the multiplication by 10 does not overflow, because w.e >= -60 + // and thus one.e >= -60. + DOUBLE_CONVERSION_ASSERT(one.e() >= -60); + DOUBLE_CONVERSION_ASSERT(fractionals < one.f()); + DOUBLE_CONVERSION_ASSERT(DOUBLE_CONVERSION_UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF) / 10 >= one.f()); + for (;;) { + fractionals *= 10; + unit *= 10; + unsafe_interval.set_f(unsafe_interval.f() * 10); + // Integer division by one. + int digit = static_cast(fractionals >> -one.e()); + DOUBLE_CONVERSION_ASSERT(digit <= 9); + buffer[*length] = static_cast('0' + digit); + (*length)++; + fractionals &= one.f() - 1; // Modulo by one. + (*kappa)--; + if (fractionals < unsafe_interval.f()) { + return RoundWeed(buffer, *length, DiyFp::Minus(too_high, w).f() * unit, + unsafe_interval.f(), fractionals, one.f(), unit); + } + } +} + + + +// Generates (at most) requested_digits digits of input number w. +// w is a floating-point number (DiyFp), consisting of a significand and an +// exponent. Its exponent is bounded by kMinimalTargetExponent and +// kMaximalTargetExponent. +// Hence -60 <= w.e() <= -32. +// +// Returns false if it fails, in which case the generated digits in the buffer +// should not be used. +// Preconditions: +// * w is correct up to 1 ulp (unit in the last place). That +// is, its error must be strictly less than a unit of its last digit. +// * kMinimalTargetExponent <= w.e() <= kMaximalTargetExponent +// +// Postconditions: returns false if procedure fails. +// otherwise: +// * buffer is not null-terminated, but length contains the number of +// digits. +// * the representation in buffer is the most precise representation of +// requested_digits digits. +// * buffer contains at most requested_digits digits of w. If there are less +// than requested_digits digits then some trailing '0's have been removed. +// * kappa is such that +// w = buffer * 10^kappa + eps with |eps| < 10^kappa / 2. +// +// Remark: This procedure takes into account the imprecision of its input +// numbers. If the precision is not enough to guarantee all the postconditions +// then false is returned. This usually happens rarely, but the failure-rate +// increases with higher requested_digits. +static bool DigitGenCounted(DiyFp w, + int requested_digits, + Vector buffer, + int* length, + int* kappa) { + DOUBLE_CONVERSION_ASSERT(kMinimalTargetExponent <= w.e() && w.e() <= kMaximalTargetExponent); + DOUBLE_CONVERSION_ASSERT(kMinimalTargetExponent >= -60); + DOUBLE_CONVERSION_ASSERT(kMaximalTargetExponent <= -32); + // w is assumed to have an error less than 1 unit. Whenever w is scaled we + // also scale its error. + uint64_t w_error = 1; + // We cut the input number into two parts: the integral digits and the + // fractional digits. We don't emit any decimal separator, but adapt kappa + // instead. Example: instead of writing "1.2" we put "12" into the buffer and + // increase kappa by 1. + DiyFp one = DiyFp(static_cast(1) << -w.e(), w.e()); + // Division by one is a shift. + uint32_t integrals = static_cast(w.f() >> -one.e()); + // Modulo by one is an and. + uint64_t fractionals = w.f() & (one.f() - 1); + uint32_t divisor; + int divisor_exponent_plus_one; + BiggestPowerTen(integrals, DiyFp::kSignificandSize - (-one.e()), + &divisor, &divisor_exponent_plus_one); + *kappa = divisor_exponent_plus_one; + *length = 0; + + // Loop invariant: buffer = w / 10^kappa (integer division) + // The invariant holds for the first iteration: kappa has been initialized + // with the divisor exponent + 1. And the divisor is the biggest power of ten + // that is smaller than 'integrals'. + while (*kappa > 0) { + int digit = integrals / divisor; + DOUBLE_CONVERSION_ASSERT(digit <= 9); + buffer[*length] = static_cast('0' + digit); + (*length)++; + requested_digits--; + integrals %= divisor; + (*kappa)--; + // Note that kappa now equals the exponent of the divisor and that the + // invariant thus holds again. + if (requested_digits == 0) break; + divisor /= 10; + } + + if (requested_digits == 0) { + uint64_t rest = + (static_cast(integrals) << -one.e()) + fractionals; + return RoundWeedCounted(buffer, *length, rest, + static_cast(divisor) << -one.e(), w_error, + kappa); + } + + // The integrals have been generated. We are at the point of the decimal + // separator. In the following loop we simply multiply the remaining digits by + // 10 and divide by one. We just need to pay attention to multiply associated + // data (the 'unit'), too. + // Note that the multiplication by 10 does not overflow, because w.e >= -60 + // and thus one.e >= -60. + DOUBLE_CONVERSION_ASSERT(one.e() >= -60); + DOUBLE_CONVERSION_ASSERT(fractionals < one.f()); + DOUBLE_CONVERSION_ASSERT(DOUBLE_CONVERSION_UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF) / 10 >= one.f()); + while (requested_digits > 0 && fractionals > w_error) { + fractionals *= 10; + w_error *= 10; + // Integer division by one. + int digit = static_cast(fractionals >> -one.e()); + DOUBLE_CONVERSION_ASSERT(digit <= 9); + buffer[*length] = static_cast('0' + digit); + (*length)++; + requested_digits--; + fractionals &= one.f() - 1; // Modulo by one. + (*kappa)--; + } + if (requested_digits != 0) return false; + return RoundWeedCounted(buffer, *length, fractionals, one.f(), w_error, + kappa); +} + + +// Provides a decimal representation of v. +// Returns true if it succeeds, otherwise the result cannot be trusted. +// There will be *length digits inside the buffer (not null-terminated). +// If the function returns true then +// v == (double) (buffer * 10^decimal_exponent). +// The digits in the buffer are the shortest representation possible: no +// 0.09999999999999999 instead of 0.1. The shorter representation will even be +// chosen even if the longer one would be closer to v. +// The last digit will be closest to the actual v. That is, even if several +// digits might correctly yield 'v' when read again, the closest will be +// computed. +static bool Grisu3(double v, + FastDtoaMode mode, + Vector buffer, + int* length, + int* decimal_exponent) { + DiyFp w = Double(v).AsNormalizedDiyFp(); + // boundary_minus and boundary_plus are the boundaries between v and its + // closest floating-point neighbors. Any number strictly between + // boundary_minus and boundary_plus will round to v when convert to a double. + // Grisu3 will never output representations that lie exactly on a boundary. + DiyFp boundary_minus, boundary_plus; + if (mode == FAST_DTOA_SHORTEST) { + Double(v).NormalizedBoundaries(&boundary_minus, &boundary_plus); + } else { + DOUBLE_CONVERSION_ASSERT(mode == FAST_DTOA_SHORTEST_SINGLE); + float single_v = static_cast(v); + Single(single_v).NormalizedBoundaries(&boundary_minus, &boundary_plus); + } + DOUBLE_CONVERSION_ASSERT(boundary_plus.e() == w.e()); + DiyFp ten_mk; // Cached power of ten: 10^-k + int mk; // -k + int ten_mk_minimal_binary_exponent = + kMinimalTargetExponent - (w.e() + DiyFp::kSignificandSize); + int ten_mk_maximal_binary_exponent = + kMaximalTargetExponent - (w.e() + DiyFp::kSignificandSize); + PowersOfTenCache::GetCachedPowerForBinaryExponentRange( + ten_mk_minimal_binary_exponent, + ten_mk_maximal_binary_exponent, + &ten_mk, &mk); + DOUBLE_CONVERSION_ASSERT((kMinimalTargetExponent <= w.e() + ten_mk.e() + + DiyFp::kSignificandSize) && + (kMaximalTargetExponent >= w.e() + ten_mk.e() + + DiyFp::kSignificandSize)); + // Note that ten_mk is only an approximation of 10^-k. A DiyFp only contains a + // 64 bit significand and ten_mk is thus only precise up to 64 bits. + + // The DiyFp::Times procedure rounds its result, and ten_mk is approximated + // too. The variable scaled_w (as well as scaled_boundary_minus/plus) are now + // off by a small amount. + // In fact: scaled_w - w*10^k < 1ulp (unit in the last place) of scaled_w. + // In other words: let f = scaled_w.f() and e = scaled_w.e(), then + // (f-1) * 2^e < w*10^k < (f+1) * 2^e + DiyFp scaled_w = DiyFp::Times(w, ten_mk); + DOUBLE_CONVERSION_ASSERT(scaled_w.e() == + boundary_plus.e() + ten_mk.e() + DiyFp::kSignificandSize); + // In theory it would be possible to avoid some recomputations by computing + // the difference between w and boundary_minus/plus (a power of 2) and to + // compute scaled_boundary_minus/plus by subtracting/adding from + // scaled_w. However the code becomes much less readable and the speed + // enhancements are not terrific. + DiyFp scaled_boundary_minus = DiyFp::Times(boundary_minus, ten_mk); + DiyFp scaled_boundary_plus = DiyFp::Times(boundary_plus, ten_mk); + + // DigitGen will generate the digits of scaled_w. Therefore we have + // v == (double) (scaled_w * 10^-mk). + // Set decimal_exponent == -mk and pass it to DigitGen. If scaled_w is not an + // integer than it will be updated. For instance if scaled_w == 1.23 then + // the buffer will be filled with "123" and the decimal_exponent will be + // decreased by 2. + int kappa; + bool result = DigitGen(scaled_boundary_minus, scaled_w, scaled_boundary_plus, + buffer, length, &kappa); + *decimal_exponent = -mk + kappa; + return result; +} + + +// The "counted" version of grisu3 (see above) only generates requested_digits +// number of digits. This version does not generate the shortest representation, +// and with enough requested digits 0.1 will at some point print as 0.9999999... +// Grisu3 is too imprecise for real halfway cases (1.5 will not work) and +// therefore the rounding strategy for halfway cases is irrelevant. +static bool Grisu3Counted(double v, + int requested_digits, + Vector buffer, + int* length, + int* decimal_exponent) { + DiyFp w = Double(v).AsNormalizedDiyFp(); + DiyFp ten_mk; // Cached power of ten: 10^-k + int mk; // -k + int ten_mk_minimal_binary_exponent = + kMinimalTargetExponent - (w.e() + DiyFp::kSignificandSize); + int ten_mk_maximal_binary_exponent = + kMaximalTargetExponent - (w.e() + DiyFp::kSignificandSize); + PowersOfTenCache::GetCachedPowerForBinaryExponentRange( + ten_mk_minimal_binary_exponent, + ten_mk_maximal_binary_exponent, + &ten_mk, &mk); + DOUBLE_CONVERSION_ASSERT((kMinimalTargetExponent <= w.e() + ten_mk.e() + + DiyFp::kSignificandSize) && + (kMaximalTargetExponent >= w.e() + ten_mk.e() + + DiyFp::kSignificandSize)); + // Note that ten_mk is only an approximation of 10^-k. A DiyFp only contains a + // 64 bit significand and ten_mk is thus only precise up to 64 bits. + + // The DiyFp::Times procedure rounds its result, and ten_mk is approximated + // too. The variable scaled_w (as well as scaled_boundary_minus/plus) are now + // off by a small amount. + // In fact: scaled_w - w*10^k < 1ulp (unit in the last place) of scaled_w. + // In other words: let f = scaled_w.f() and e = scaled_w.e(), then + // (f-1) * 2^e < w*10^k < (f+1) * 2^e + DiyFp scaled_w = DiyFp::Times(w, ten_mk); + + // We now have (double) (scaled_w * 10^-mk). + // DigitGen will generate the first requested_digits digits of scaled_w and + // return together with a kappa such that scaled_w ~= buffer * 10^kappa. (It + // will not always be exactly the same since DigitGenCounted only produces a + // limited number of digits.) + int kappa; + bool result = DigitGenCounted(scaled_w, requested_digits, + buffer, length, &kappa); + *decimal_exponent = -mk + kappa; + return result; +} + + +bool FastDtoa(double v, + FastDtoaMode mode, + int requested_digits, + Vector buffer, + int* length, + int* decimal_point) { + DOUBLE_CONVERSION_ASSERT(v > 0); + DOUBLE_CONVERSION_ASSERT(!Double(v).IsSpecial()); + + bool result = false; + int decimal_exponent = 0; + switch (mode) { + case FAST_DTOA_SHORTEST: + case FAST_DTOA_SHORTEST_SINGLE: + result = Grisu3(v, mode, buffer, length, &decimal_exponent); + break; + case FAST_DTOA_PRECISION: + result = Grisu3Counted(v, requested_digits, + buffer, length, &decimal_exponent); + break; + default: + DOUBLE_CONVERSION_UNREACHABLE(); + } + if (result) { + *decimal_point = *length + decimal_exponent; + buffer[*length] = '\0'; + } + return result; +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-fast-dtoa.h b/intl/icu/source/i18n/double-conversion-fast-dtoa.h new file mode 100644 index 0000000000..58a6470052 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-fast-dtoa.h @@ -0,0 +1,106 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_FAST_DTOA_H_ +#define DOUBLE_CONVERSION_FAST_DTOA_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +enum FastDtoaMode { + // Computes the shortest representation of the given input. The returned + // result will be the most accurate number of this length. Longer + // representations might be more accurate. + FAST_DTOA_SHORTEST, + // Same as FAST_DTOA_SHORTEST but for single-precision floats. + FAST_DTOA_SHORTEST_SINGLE, + // Computes a representation where the precision (number of digits) is + // given as input. The precision is independent of the decimal point. + FAST_DTOA_PRECISION +}; + +// FastDtoa will produce at most kFastDtoaMaximalLength digits. This does not +// include the terminating '\0' character. +static const int kFastDtoaMaximalLength = 17; +// Same for single-precision numbers. +static const int kFastDtoaMaximalSingleLength = 9; + +// Provides a decimal representation of v. +// The result should be interpreted as buffer * 10^(point - length). +// +// Precondition: +// * v must be a strictly positive finite double. +// +// Returns true if it succeeds, otherwise the result can not be trusted. +// There will be *length digits inside the buffer followed by a null terminator. +// If the function returns true and mode equals +// - FAST_DTOA_SHORTEST, then +// the parameter requested_digits is ignored. +// The result satisfies +// v == (double) (buffer * 10^(point - length)). +// The digits in the buffer are the shortest representation possible. E.g. +// if 0.099999999999 and 0.1 represent the same double then "1" is returned +// with point = 0. +// The last digit will be closest to the actual v. That is, even if several +// digits might correctly yield 'v' when read again, the buffer will contain +// the one closest to v. +// - FAST_DTOA_PRECISION, then +// the buffer contains requested_digits digits. +// the difference v - (buffer * 10^(point-length)) is closest to zero for +// all possible representations of requested_digits digits. +// If there are two values that are equally close, then FastDtoa returns +// false. +// For both modes the buffer must be large enough to hold the result. +bool FastDtoa(double d, + FastDtoaMode mode, + int requested_digits, + Vector buffer, + int* length, + int* decimal_point); + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_FAST_DTOA_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-ieee.h b/intl/icu/source/i18n/double-conversion-ieee.h new file mode 100644 index 0000000000..2940acb169 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-ieee.h @@ -0,0 +1,465 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2012 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_DOUBLE_H_ +#define DOUBLE_CONVERSION_DOUBLE_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-diy-fp.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +// We assume that doubles and uint64_t have the same endianness. +static uint64_t double_to_uint64(double d) { return BitCast(d); } +static double uint64_to_double(uint64_t d64) { return BitCast(d64); } +static uint32_t float_to_uint32(float f) { return BitCast(f); } +static float uint32_to_float(uint32_t d32) { return BitCast(d32); } + +// Helper functions for doubles. +class Double { + public: + static const uint64_t kSignMask = DOUBLE_CONVERSION_UINT64_2PART_C(0x80000000, 00000000); + static const uint64_t kExponentMask = DOUBLE_CONVERSION_UINT64_2PART_C(0x7FF00000, 00000000); + static const uint64_t kSignificandMask = DOUBLE_CONVERSION_UINT64_2PART_C(0x000FFFFF, FFFFFFFF); + static const uint64_t kHiddenBit = DOUBLE_CONVERSION_UINT64_2PART_C(0x00100000, 00000000); + static const uint64_t kQuietNanBit = DOUBLE_CONVERSION_UINT64_2PART_C(0x00080000, 00000000); + static const int kPhysicalSignificandSize = 52; // Excludes the hidden bit. + static const int kSignificandSize = 53; + static const int kExponentBias = 0x3FF + kPhysicalSignificandSize; + static const int kMaxExponent = 0x7FF - kExponentBias; + + Double() : d64_(0) {} + explicit Double(double d) : d64_(double_to_uint64(d)) {} + explicit Double(uint64_t d64) : d64_(d64) {} + explicit Double(DiyFp diy_fp) + : d64_(DiyFpToUint64(diy_fp)) {} + + // The value encoded by this Double must be greater or equal to +0.0. + // It must not be special (infinity, or NaN). + DiyFp AsDiyFp() const { + DOUBLE_CONVERSION_ASSERT(Sign() > 0); + DOUBLE_CONVERSION_ASSERT(!IsSpecial()); + return DiyFp(Significand(), Exponent()); + } + + // The value encoded by this Double must be strictly greater than 0. + DiyFp AsNormalizedDiyFp() const { + DOUBLE_CONVERSION_ASSERT(value() > 0.0); + uint64_t f = Significand(); + int e = Exponent(); + + // The current double could be a denormal. + while ((f & kHiddenBit) == 0) { + f <<= 1; + e--; + } + // Do the final shifts in one go. + f <<= DiyFp::kSignificandSize - kSignificandSize; + e -= DiyFp::kSignificandSize - kSignificandSize; + return DiyFp(f, e); + } + + // Returns the double's bit as uint64. + uint64_t AsUint64() const { + return d64_; + } + + // Returns the next greater double. Returns +infinity on input +infinity. + double NextDouble() const { + if (d64_ == kInfinity) return Double(kInfinity).value(); + if (Sign() < 0 && Significand() == 0) { + // -0.0 + return 0.0; + } + if (Sign() < 0) { + return Double(d64_ - 1).value(); + } else { + return Double(d64_ + 1).value(); + } + } + + double PreviousDouble() const { + if (d64_ == (kInfinity | kSignMask)) return -Infinity(); + if (Sign() < 0) { + return Double(d64_ + 1).value(); + } else { + if (Significand() == 0) return -0.0; + return Double(d64_ - 1).value(); + } + } + + int Exponent() const { + if (IsDenormal()) return kDenormalExponent; + + uint64_t d64 = AsUint64(); + int biased_e = + static_cast((d64 & kExponentMask) >> kPhysicalSignificandSize); + return biased_e - kExponentBias; + } + + uint64_t Significand() const { + uint64_t d64 = AsUint64(); + uint64_t significand = d64 & kSignificandMask; + if (!IsDenormal()) { + return significand + kHiddenBit; + } else { + return significand; + } + } + + // Returns true if the double is a denormal. + bool IsDenormal() const { + uint64_t d64 = AsUint64(); + return (d64 & kExponentMask) == 0; + } + + // We consider denormals not to be special. + // Hence only Infinity and NaN are special. + bool IsSpecial() const { + uint64_t d64 = AsUint64(); + return (d64 & kExponentMask) == kExponentMask; + } + + bool IsNan() const { + uint64_t d64 = AsUint64(); + return ((d64 & kExponentMask) == kExponentMask) && + ((d64 & kSignificandMask) != 0); + } + + bool IsQuietNan() const { +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + return IsNan() && ((AsUint64() & kQuietNanBit) == 0); +#else + return IsNan() && ((AsUint64() & kQuietNanBit) != 0); +#endif + } + + bool IsSignalingNan() const { +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + return IsNan() && ((AsUint64() & kQuietNanBit) != 0); +#else + return IsNan() && ((AsUint64() & kQuietNanBit) == 0); +#endif + } + + + bool IsInfinite() const { + uint64_t d64 = AsUint64(); + return ((d64 & kExponentMask) == kExponentMask) && + ((d64 & kSignificandMask) == 0); + } + + int Sign() const { + uint64_t d64 = AsUint64(); + return (d64 & kSignMask) == 0? 1: -1; + } + + // Precondition: the value encoded by this Double must be greater or equal + // than +0.0. + DiyFp UpperBoundary() const { + DOUBLE_CONVERSION_ASSERT(Sign() > 0); + return DiyFp(Significand() * 2 + 1, Exponent() - 1); + } + + // Computes the two boundaries of this. + // The bigger boundary (m_plus) is normalized. The lower boundary has the same + // exponent as m_plus. + // Precondition: the value encoded by this Double must be greater than 0. + void NormalizedBoundaries(DiyFp* out_m_minus, DiyFp* out_m_plus) const { + DOUBLE_CONVERSION_ASSERT(value() > 0.0); + DiyFp v = this->AsDiyFp(); + DiyFp m_plus = DiyFp::Normalize(DiyFp((v.f() << 1) + 1, v.e() - 1)); + DiyFp m_minus; + if (LowerBoundaryIsCloser()) { + m_minus = DiyFp((v.f() << 2) - 1, v.e() - 2); + } else { + m_minus = DiyFp((v.f() << 1) - 1, v.e() - 1); + } + m_minus.set_f(m_minus.f() << (m_minus.e() - m_plus.e())); + m_minus.set_e(m_plus.e()); + *out_m_plus = m_plus; + *out_m_minus = m_minus; + } + + bool LowerBoundaryIsCloser() const { + // The boundary is closer if the significand is of the form f == 2^p-1 then + // the lower boundary is closer. + // Think of v = 1000e10 and v- = 9999e9. + // Then the boundary (== (v - v-)/2) is not just at a distance of 1e9 but + // at a distance of 1e8. + // The only exception is for the smallest normal: the largest denormal is + // at the same distance as its successor. + // Note: denormals have the same exponent as the smallest normals. + bool physical_significand_is_zero = ((AsUint64() & kSignificandMask) == 0); + return physical_significand_is_zero && (Exponent() != kDenormalExponent); + } + + double value() const { return uint64_to_double(d64_); } + + // Returns the significand size for a given order of magnitude. + // If v = f*2^e with 2^p-1 <= f <= 2^p then p+e is v's order of magnitude. + // This function returns the number of significant binary digits v will have + // once it's encoded into a double. In almost all cases this is equal to + // kSignificandSize. The only exceptions are denormals. They start with + // leading zeroes and their effective significand-size is hence smaller. + static int SignificandSizeForOrderOfMagnitude(int order) { + if (order >= (kDenormalExponent + kSignificandSize)) { + return kSignificandSize; + } + if (order <= kDenormalExponent) return 0; + return order - kDenormalExponent; + } + + static double Infinity() { + return Double(kInfinity).value(); + } + + static double NaN() { + return Double(kNaN).value(); + } + + private: + static const int kDenormalExponent = -kExponentBias + 1; + static const uint64_t kInfinity = DOUBLE_CONVERSION_UINT64_2PART_C(0x7FF00000, 00000000); +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + static const uint64_t kNaN = DOUBLE_CONVERSION_UINT64_2PART_C(0x7FF7FFFF, FFFFFFFF); +#else + static const uint64_t kNaN = DOUBLE_CONVERSION_UINT64_2PART_C(0x7FF80000, 00000000); +#endif + + + const uint64_t d64_; + + static uint64_t DiyFpToUint64(DiyFp diy_fp) { + uint64_t significand = diy_fp.f(); + int exponent = diy_fp.e(); + while (significand > kHiddenBit + kSignificandMask) { + significand >>= 1; + exponent++; + } + if (exponent >= kMaxExponent) { + return kInfinity; + } + if (exponent < kDenormalExponent) { + return 0; + } + while (exponent > kDenormalExponent && (significand & kHiddenBit) == 0) { + significand <<= 1; + exponent--; + } + uint64_t biased_exponent; + if (exponent == kDenormalExponent && (significand & kHiddenBit) == 0) { + biased_exponent = 0; + } else { + biased_exponent = static_cast(exponent + kExponentBias); + } + return (significand & kSignificandMask) | + (biased_exponent << kPhysicalSignificandSize); + } + + DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN(Double); +}; + +class Single { + public: + static const uint32_t kSignMask = 0x80000000; + static const uint32_t kExponentMask = 0x7F800000; + static const uint32_t kSignificandMask = 0x007FFFFF; + static const uint32_t kHiddenBit = 0x00800000; + static const uint32_t kQuietNanBit = 0x00400000; + static const int kPhysicalSignificandSize = 23; // Excludes the hidden bit. + static const int kSignificandSize = 24; + + Single() : d32_(0) {} + explicit Single(float f) : d32_(float_to_uint32(f)) {} + explicit Single(uint32_t d32) : d32_(d32) {} + + // The value encoded by this Single must be greater or equal to +0.0. + // It must not be special (infinity, or NaN). + DiyFp AsDiyFp() const { + DOUBLE_CONVERSION_ASSERT(Sign() > 0); + DOUBLE_CONVERSION_ASSERT(!IsSpecial()); + return DiyFp(Significand(), Exponent()); + } + + // Returns the single's bit as uint64. + uint32_t AsUint32() const { + return d32_; + } + + int Exponent() const { + if (IsDenormal()) return kDenormalExponent; + + uint32_t d32 = AsUint32(); + int biased_e = + static_cast((d32 & kExponentMask) >> kPhysicalSignificandSize); + return biased_e - kExponentBias; + } + + uint32_t Significand() const { + uint32_t d32 = AsUint32(); + uint32_t significand = d32 & kSignificandMask; + if (!IsDenormal()) { + return significand + kHiddenBit; + } else { + return significand; + } + } + + // Returns true if the single is a denormal. + bool IsDenormal() const { + uint32_t d32 = AsUint32(); + return (d32 & kExponentMask) == 0; + } + + // We consider denormals not to be special. + // Hence only Infinity and NaN are special. + bool IsSpecial() const { + uint32_t d32 = AsUint32(); + return (d32 & kExponentMask) == kExponentMask; + } + + bool IsNan() const { + uint32_t d32 = AsUint32(); + return ((d32 & kExponentMask) == kExponentMask) && + ((d32 & kSignificandMask) != 0); + } + + bool IsQuietNan() const { +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + return IsNan() && ((AsUint32() & kQuietNanBit) == 0); +#else + return IsNan() && ((AsUint32() & kQuietNanBit) != 0); +#endif + } + + bool IsSignalingNan() const { +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + return IsNan() && ((AsUint32() & kQuietNanBit) != 0); +#else + return IsNan() && ((AsUint32() & kQuietNanBit) == 0); +#endif + } + + + bool IsInfinite() const { + uint32_t d32 = AsUint32(); + return ((d32 & kExponentMask) == kExponentMask) && + ((d32 & kSignificandMask) == 0); + } + + int Sign() const { + uint32_t d32 = AsUint32(); + return (d32 & kSignMask) == 0? 1: -1; + } + + // Computes the two boundaries of this. + // The bigger boundary (m_plus) is normalized. The lower boundary has the same + // exponent as m_plus. + // Precondition: the value encoded by this Single must be greater than 0. + void NormalizedBoundaries(DiyFp* out_m_minus, DiyFp* out_m_plus) const { + DOUBLE_CONVERSION_ASSERT(value() > 0.0); + DiyFp v = this->AsDiyFp(); + DiyFp m_plus = DiyFp::Normalize(DiyFp((v.f() << 1) + 1, v.e() - 1)); + DiyFp m_minus; + if (LowerBoundaryIsCloser()) { + m_minus = DiyFp((v.f() << 2) - 1, v.e() - 2); + } else { + m_minus = DiyFp((v.f() << 1) - 1, v.e() - 1); + } + m_minus.set_f(m_minus.f() << (m_minus.e() - m_plus.e())); + m_minus.set_e(m_plus.e()); + *out_m_plus = m_plus; + *out_m_minus = m_minus; + } + + // Precondition: the value encoded by this Single must be greater or equal + // than +0.0. + DiyFp UpperBoundary() const { + DOUBLE_CONVERSION_ASSERT(Sign() > 0); + return DiyFp(Significand() * 2 + 1, Exponent() - 1); + } + + bool LowerBoundaryIsCloser() const { + // The boundary is closer if the significand is of the form f == 2^p-1 then + // the lower boundary is closer. + // Think of v = 1000e10 and v- = 9999e9. + // Then the boundary (== (v - v-)/2) is not just at a distance of 1e9 but + // at a distance of 1e8. + // The only exception is for the smallest normal: the largest denormal is + // at the same distance as its successor. + // Note: denormals have the same exponent as the smallest normals. + bool physical_significand_is_zero = ((AsUint32() & kSignificandMask) == 0); + return physical_significand_is_zero && (Exponent() != kDenormalExponent); + } + + float value() const { return uint32_to_float(d32_); } + + static float Infinity() { + return Single(kInfinity).value(); + } + + static float NaN() { + return Single(kNaN).value(); + } + + private: + static const int kExponentBias = 0x7F + kPhysicalSignificandSize; + static const int kDenormalExponent = -kExponentBias + 1; + static const int kMaxExponent = 0xFF - kExponentBias; + static const uint32_t kInfinity = 0x7F800000; +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + static const uint32_t kNaN = 0x7FBFFFFF; +#else + static const uint32_t kNaN = 0x7FC00000; +#endif + + const uint32_t d32_; + + DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN(Single); +}; + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_DOUBLE_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-string-to-double.cpp b/intl/icu/source/i18n/double-conversion-string-to-double.cpp new file mode 100644 index 0000000000..727fff24e1 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-string-to-double.cpp @@ -0,0 +1,843 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +// ICU PATCH: Do not include std::locale. + +#include +// #include +#include + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-string-to-double.h" + +#include "double-conversion-ieee.h" +#include "double-conversion-strtod.h" +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +#ifdef _MSC_VER +# if _MSC_VER >= 1900 +// Fix MSVC >= 2015 (_MSC_VER == 1900) warning +// C4244: 'argument': conversion from 'const uc16' to 'char', possible loss of data +// against Advance and friends, when instantiated with **it as char, not uc16. + __pragma(warning(disable: 4244)) +# endif +# if _MSC_VER <= 1700 // VS2012, see IsDecimalDigitForRadix warning fix, below +# define VS2012_RADIXWARN +# endif +#endif + +namespace double_conversion { + +namespace { + +inline char ToLower(char ch) { +#if 0 // do not include std::locale in ICU + static const std::ctype& cType = + std::use_facet >(std::locale::classic()); + return cType.tolower(ch); +#else + (void)ch; + DOUBLE_CONVERSION_UNREACHABLE(); +#endif +} + +inline char Pass(char ch) { + return ch; +} + +template +static inline bool ConsumeSubStringImpl(Iterator* current, + Iterator end, + const char* substring, + Converter converter) { + DOUBLE_CONVERSION_ASSERT(converter(**current) == *substring); + for (substring++; *substring != '\0'; substring++) { + ++*current; + if (*current == end || converter(**current) != *substring) { + return false; + } + } + ++*current; + return true; +} + +// Consumes the given substring from the iterator. +// Returns false, if the substring does not match. +template +static bool ConsumeSubString(Iterator* current, + Iterator end, + const char* substring, + bool allow_case_insensitivity) { + if (allow_case_insensitivity) { + return ConsumeSubStringImpl(current, end, substring, ToLower); + } else { + return ConsumeSubStringImpl(current, end, substring, Pass); + } +} + +// Consumes first character of the str is equal to ch +inline bool ConsumeFirstCharacter(char ch, + const char* str, + bool case_insensitivity) { + return case_insensitivity ? ToLower(ch) == str[0] : ch == str[0]; +} +} // namespace + +// Maximum number of significant digits in decimal representation. +// The longest possible double in decimal representation is +// (2^53 - 1) * 2 ^ -1074 that is (2 ^ 53 - 1) * 5 ^ 1074 / 10 ^ 1074 +// (768 digits). If we parse a number whose first digits are equal to a +// mean of 2 adjacent doubles (that could have up to 769 digits) the result +// must be rounded to the bigger one unless the tail consists of zeros, so +// we don't need to preserve all the digits. +const int kMaxSignificantDigits = 772; + + +static const char kWhitespaceTable7[] = { 32, 13, 10, 9, 11, 12 }; +static const int kWhitespaceTable7Length = DOUBLE_CONVERSION_ARRAY_SIZE(kWhitespaceTable7); + + +static const uc16 kWhitespaceTable16[] = { + 160, 8232, 8233, 5760, 6158, 8192, 8193, 8194, 8195, + 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8239, 8287, 12288, 65279 +}; +static const int kWhitespaceTable16Length = DOUBLE_CONVERSION_ARRAY_SIZE(kWhitespaceTable16); + + +static bool isWhitespace(int x) { + if (x < 128) { + for (int i = 0; i < kWhitespaceTable7Length; i++) { + if (kWhitespaceTable7[i] == x) return true; + } + } else { + for (int i = 0; i < kWhitespaceTable16Length; i++) { + if (kWhitespaceTable16[i] == x) return true; + } + } + return false; +} + + +// Returns true if a nonspace found and false if the end has reached. +template +static inline bool AdvanceToNonspace(Iterator* current, Iterator end) { + while (*current != end) { + if (!isWhitespace(**current)) return true; + ++*current; + } + return false; +} + + +static bool isDigit(int x, int radix) { + return (x >= '0' && x <= '9' && x < '0' + radix) + || (radix > 10 && x >= 'a' && x < 'a' + radix - 10) + || (radix > 10 && x >= 'A' && x < 'A' + radix - 10); +} + + +static double SignedZero(bool sign) { + return sign ? -0.0 : 0.0; +} + + +// Returns true if 'c' is a decimal digit that is valid for the given radix. +// +// The function is small and could be inlined, but VS2012 emitted a warning +// because it constant-propagated the radix and concluded that the last +// condition was always true. Moving it into a separate function and +// suppressing optimisation keeps the compiler from warning. +#ifdef VS2012_RADIXWARN +#pragma optimize("",off) +static bool IsDecimalDigitForRadix(int c, int radix) { + return '0' <= c && c <= '9' && (c - '0') < radix; +} +#pragma optimize("",on) +#else +static bool inline IsDecimalDigitForRadix(int c, int radix) { + return '0' <= c && c <= '9' && (c - '0') < radix; +} +#endif +// Returns true if 'c' is a character digit that is valid for the given radix. +// The 'a_character' should be 'a' or 'A'. +// +// The function is small and could be inlined, but VS2012 emitted a warning +// because it constant-propagated the radix and concluded that the first +// condition was always false. By moving it into a separate function the +// compiler wouldn't warn anymore. +static bool IsCharacterDigitForRadix(int c, int radix, char a_character) { + return radix > 10 && c >= a_character && c < a_character + radix - 10; +} + +// Returns true, when the iterator is equal to end. +template +static bool Advance (Iterator* it, uc16 separator, int base, Iterator& end) { + if (separator == StringToDoubleConverter::kNoSeparator) { + ++(*it); + return *it == end; + } + if (!isDigit(**it, base)) { + ++(*it); + return *it == end; + } + ++(*it); + if (*it == end) return true; + if (*it + 1 == end) return false; + if (**it == separator && isDigit(*(*it + 1), base)) { + ++(*it); + } + return *it == end; +} + +// Checks whether the string in the range start-end is a hex-float string. +// This function assumes that the leading '0x'/'0X' is already consumed. +// +// Hex float strings are of one of the following forms: +// - hex_digits+ 'p' ('+'|'-')? exponent_digits+ +// - hex_digits* '.' hex_digits+ 'p' ('+'|'-')? exponent_digits+ +// - hex_digits+ '.' 'p' ('+'|'-')? exponent_digits+ +template +static bool IsHexFloatString(Iterator start, + Iterator end, + uc16 separator, + bool allow_trailing_junk) { + DOUBLE_CONVERSION_ASSERT(start != end); + + Iterator current = start; + + bool saw_digit = false; + while (isDigit(*current, 16)) { + saw_digit = true; + if (Advance(¤t, separator, 16, end)) return false; + } + if (*current == '.') { + if (Advance(¤t, separator, 16, end)) return false; + while (isDigit(*current, 16)) { + saw_digit = true; + if (Advance(¤t, separator, 16, end)) return false; + } + } + if (!saw_digit) return false; + if (*current != 'p' && *current != 'P') return false; + if (Advance(¤t, separator, 16, end)) return false; + if (*current == '+' || *current == '-') { + if (Advance(¤t, separator, 16, end)) return false; + } + if (!isDigit(*current, 10)) return false; + if (Advance(¤t, separator, 16, end)) return true; + while (isDigit(*current, 10)) { + if (Advance(¤t, separator, 16, end)) return true; + } + return allow_trailing_junk || !AdvanceToNonspace(¤t, end); +} + + +// Parsing integers with radix 2, 4, 8, 16, 32. Assumes current != end. +// +// If parse_as_hex_float is true, then the string must be a valid +// hex-float. +template +static double RadixStringToIeee(Iterator* current, + Iterator end, + bool sign, + uc16 separator, + bool parse_as_hex_float, + bool allow_trailing_junk, + double junk_string_value, + bool read_as_double, + bool* result_is_junk) { + DOUBLE_CONVERSION_ASSERT(*current != end); + DOUBLE_CONVERSION_ASSERT(!parse_as_hex_float || + IsHexFloatString(*current, end, separator, allow_trailing_junk)); + + const int kDoubleSize = Double::kSignificandSize; + const int kSingleSize = Single::kSignificandSize; + const int kSignificandSize = read_as_double? kDoubleSize: kSingleSize; + + *result_is_junk = true; + + int64_t number = 0; + int exponent = 0; + const int radix = (1 << radix_log_2); + // Whether we have encountered a '.' and are parsing the decimal digits. + // Only relevant if parse_as_hex_float is true. + bool post_decimal = false; + + // Skip leading 0s. + while (**current == '0') { + if (Advance(current, separator, radix, end)) { + *result_is_junk = false; + return SignedZero(sign); + } + } + + while (true) { + int digit; + if (IsDecimalDigitForRadix(**current, radix)) { + digit = static_cast(**current) - '0'; + if (post_decimal) exponent -= radix_log_2; + } else if (IsCharacterDigitForRadix(**current, radix, 'a')) { + digit = static_cast(**current) - 'a' + 10; + if (post_decimal) exponent -= radix_log_2; + } else if (IsCharacterDigitForRadix(**current, radix, 'A')) { + digit = static_cast(**current) - 'A' + 10; + if (post_decimal) exponent -= radix_log_2; + } else if (parse_as_hex_float && **current == '.') { + post_decimal = true; + Advance(current, separator, radix, end); + DOUBLE_CONVERSION_ASSERT(*current != end); + continue; + } else if (parse_as_hex_float && (**current == 'p' || **current == 'P')) { + break; + } else { + if (allow_trailing_junk || !AdvanceToNonspace(current, end)) { + break; + } else { + return junk_string_value; + } + } + + number = number * radix + digit; + int overflow = static_cast(number >> kSignificandSize); + if (overflow != 0) { + // Overflow occurred. Need to determine which direction to round the + // result. + int overflow_bits_count = 1; + while (overflow > 1) { + overflow_bits_count++; + overflow >>= 1; + } + + int dropped_bits_mask = ((1 << overflow_bits_count) - 1); + int dropped_bits = static_cast(number) & dropped_bits_mask; + number >>= overflow_bits_count; + exponent += overflow_bits_count; + + bool zero_tail = true; + for (;;) { + if (Advance(current, separator, radix, end)) break; + if (parse_as_hex_float && **current == '.') { + // Just run over the '.'. We are just trying to see whether there is + // a non-zero digit somewhere. + Advance(current, separator, radix, end); + DOUBLE_CONVERSION_ASSERT(*current != end); + post_decimal = true; + } + if (!isDigit(**current, radix)) break; + zero_tail = zero_tail && **current == '0'; + if (!post_decimal) exponent += radix_log_2; + } + + if (!parse_as_hex_float && + !allow_trailing_junk && + AdvanceToNonspace(current, end)) { + return junk_string_value; + } + + int middle_value = (1 << (overflow_bits_count - 1)); + if (dropped_bits > middle_value) { + number++; // Rounding up. + } else if (dropped_bits == middle_value) { + // Rounding to even to consistency with decimals: half-way case rounds + // up if significant part is odd and down otherwise. + if ((number & 1) != 0 || !zero_tail) { + number++; // Rounding up. + } + } + + // Rounding up may cause overflow. + if ((number & ((int64_t)1 << kSignificandSize)) != 0) { + exponent++; + number >>= 1; + } + break; + } + if (Advance(current, separator, radix, end)) break; + } + + DOUBLE_CONVERSION_ASSERT(number < ((int64_t)1 << kSignificandSize)); + DOUBLE_CONVERSION_ASSERT(static_cast(static_cast(number)) == number); + + *result_is_junk = false; + + if (parse_as_hex_float) { + DOUBLE_CONVERSION_ASSERT(**current == 'p' || **current == 'P'); + Advance(current, separator, radix, end); + DOUBLE_CONVERSION_ASSERT(*current != end); + bool is_negative = false; + if (**current == '+') { + Advance(current, separator, radix, end); + DOUBLE_CONVERSION_ASSERT(*current != end); + } else if (**current == '-') { + is_negative = true; + Advance(current, separator, radix, end); + DOUBLE_CONVERSION_ASSERT(*current != end); + } + int written_exponent = 0; + while (IsDecimalDigitForRadix(**current, 10)) { + // No need to read exponents if they are too big. That could potentially overflow + // the `written_exponent` variable. + if (abs(written_exponent) <= 100 * Double::kMaxExponent) { + written_exponent = 10 * written_exponent + **current - '0'; + } + if (Advance(current, separator, radix, end)) break; + } + if (is_negative) written_exponent = -written_exponent; + exponent += written_exponent; + } + + if (exponent == 0 || number == 0) { + if (sign) { + if (number == 0) return -0.0; + number = -number; + } + return static_cast(number); + } + + DOUBLE_CONVERSION_ASSERT(number != 0); + double result = Double(DiyFp(number, exponent)).value(); + return sign ? -result : result; +} + +template +double StringToDoubleConverter::StringToIeee( + Iterator input, + int length, + bool read_as_double, + int* processed_characters_count) const { + Iterator current = input; + Iterator end = input + length; + + *processed_characters_count = 0; + + const bool allow_trailing_junk = (flags_ & ALLOW_TRAILING_JUNK) != 0; + const bool allow_leading_spaces = (flags_ & ALLOW_LEADING_SPACES) != 0; + const bool allow_trailing_spaces = (flags_ & ALLOW_TRAILING_SPACES) != 0; + const bool allow_spaces_after_sign = (flags_ & ALLOW_SPACES_AFTER_SIGN) != 0; + const bool allow_case_insensitivity = (flags_ & ALLOW_CASE_INSENSITIVITY) != 0; + + // To make sure that iterator dereferencing is valid the following + // convention is used: + // 1. Each '++current' statement is followed by check for equality to 'end'. + // 2. If AdvanceToNonspace returned false then current == end. + // 3. If 'current' becomes equal to 'end' the function returns or goes to + // 'parsing_done'. + // 4. 'current' is not dereferenced after the 'parsing_done' label. + // 5. Code before 'parsing_done' may rely on 'current != end'. + if (current == end) return empty_string_value_; + + if (allow_leading_spaces || allow_trailing_spaces) { + if (!AdvanceToNonspace(¤t, end)) { + *processed_characters_count = static_cast(current - input); + return empty_string_value_; + } + if (!allow_leading_spaces && (input != current)) { + // No leading spaces allowed, but AdvanceToNonspace moved forward. + return junk_string_value_; + } + } + + // Exponent will be adjusted if insignificant digits of the integer part + // or insignificant leading zeros of the fractional part are dropped. + int exponent = 0; + int significant_digits = 0; + int insignificant_digits = 0; + bool nonzero_digit_dropped = false; + + bool sign = false; + + if (*current == '+' || *current == '-') { + sign = (*current == '-'); + ++current; + Iterator next_non_space = current; + // Skip following spaces (if allowed). + if (!AdvanceToNonspace(&next_non_space, end)) return junk_string_value_; + if (!allow_spaces_after_sign && (current != next_non_space)) { + return junk_string_value_; + } + current = next_non_space; + } + + if (infinity_symbol_ != DOUBLE_CONVERSION_NULLPTR) { + if (ConsumeFirstCharacter(*current, infinity_symbol_, allow_case_insensitivity)) { + if (!ConsumeSubString(¤t, end, infinity_symbol_, allow_case_insensitivity)) { + return junk_string_value_; + } + + if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) { + return junk_string_value_; + } + if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) { + return junk_string_value_; + } + + *processed_characters_count = static_cast(current - input); + return sign ? -Double::Infinity() : Double::Infinity(); + } + } + + if (nan_symbol_ != DOUBLE_CONVERSION_NULLPTR) { + if (ConsumeFirstCharacter(*current, nan_symbol_, allow_case_insensitivity)) { + if (!ConsumeSubString(¤t, end, nan_symbol_, allow_case_insensitivity)) { + return junk_string_value_; + } + + if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) { + return junk_string_value_; + } + if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) { + return junk_string_value_; + } + + *processed_characters_count = static_cast(current - input); + return sign ? -Double::NaN() : Double::NaN(); + } + } + + bool leading_zero = false; + if (*current == '0') { + if (Advance(¤t, separator_, 10, end)) { + *processed_characters_count = static_cast(current - input); + return SignedZero(sign); + } + + leading_zero = true; + + // It could be hexadecimal value. + if (((flags_ & ALLOW_HEX) || (flags_ & ALLOW_HEX_FLOATS)) && + (*current == 'x' || *current == 'X')) { + ++current; + + if (current == end) return junk_string_value_; // "0x" + + bool parse_as_hex_float = (flags_ & ALLOW_HEX_FLOATS) && + IsHexFloatString(current, end, separator_, allow_trailing_junk); + + if (!parse_as_hex_float && !isDigit(*current, 16)) { + return junk_string_value_; + } + + bool result_is_junk; + double result = RadixStringToIeee<4>(¤t, + end, + sign, + separator_, + parse_as_hex_float, + allow_trailing_junk, + junk_string_value_, + read_as_double, + &result_is_junk); + if (!result_is_junk) { + if (allow_trailing_spaces) AdvanceToNonspace(¤t, end); + *processed_characters_count = static_cast(current - input); + } + return result; + } + + // Ignore leading zeros in the integer part. + while (*current == '0') { + if (Advance(¤t, separator_, 10, end)) { + *processed_characters_count = static_cast(current - input); + return SignedZero(sign); + } + } + } + + bool octal = leading_zero && (flags_ & ALLOW_OCTALS) != 0; + + // The longest form of simplified number is: "-.1eXXX\0". + const int kBufferSize = kMaxSignificantDigits + 10; + DOUBLE_CONVERSION_STACK_UNINITIALIZED char + buffer[kBufferSize]; // NOLINT: size is known at compile time. + int buffer_pos = 0; + + // Copy significant digits of the integer part (if any) to the buffer. + while (*current >= '0' && *current <= '9') { + if (significant_digits < kMaxSignificantDigits) { + DOUBLE_CONVERSION_ASSERT(buffer_pos < kBufferSize); + buffer[buffer_pos++] = static_cast(*current); + significant_digits++; + // Will later check if it's an octal in the buffer. + } else { + insignificant_digits++; // Move the digit into the exponential part. + nonzero_digit_dropped = nonzero_digit_dropped || *current != '0'; + } + octal = octal && *current < '8'; + if (Advance(¤t, separator_, 10, end)) goto parsing_done; + } + + if (significant_digits == 0) { + octal = false; + } + + if (*current == '.') { + if (octal && !allow_trailing_junk) return junk_string_value_; + if (octal) goto parsing_done; + + if (Advance(¤t, separator_, 10, end)) { + if (significant_digits == 0 && !leading_zero) { + return junk_string_value_; + } else { + goto parsing_done; + } + } + + if (significant_digits == 0) { + // octal = false; + // Integer part consists of 0 or is absent. Significant digits start after + // leading zeros (if any). + while (*current == '0') { + if (Advance(¤t, separator_, 10, end)) { + *processed_characters_count = static_cast(current - input); + return SignedZero(sign); + } + exponent--; // Move this 0 into the exponent. + } + } + + // There is a fractional part. + // We don't emit a '.', but adjust the exponent instead. + while (*current >= '0' && *current <= '9') { + if (significant_digits < kMaxSignificantDigits) { + DOUBLE_CONVERSION_ASSERT(buffer_pos < kBufferSize); + buffer[buffer_pos++] = static_cast(*current); + significant_digits++; + exponent--; + } else { + // Ignore insignificant digits in the fractional part. + nonzero_digit_dropped = nonzero_digit_dropped || *current != '0'; + } + if (Advance(¤t, separator_, 10, end)) goto parsing_done; + } + } + + if (!leading_zero && exponent == 0 && significant_digits == 0) { + // If leading_zeros is true then the string contains zeros. + // If exponent < 0 then string was [+-]\.0*... + // If significant_digits != 0 the string is not equal to 0. + // Otherwise there are no digits in the string. + return junk_string_value_; + } + + // Parse exponential part. + if (*current == 'e' || *current == 'E') { + if (octal && !allow_trailing_junk) return junk_string_value_; + if (octal) goto parsing_done; + Iterator junk_begin = current; + ++current; + if (current == end) { + if (allow_trailing_junk) { + current = junk_begin; + goto parsing_done; + } else { + return junk_string_value_; + } + } + char exponen_sign = '+'; + if (*current == '+' || *current == '-') { + exponen_sign = static_cast(*current); + ++current; + if (current == end) { + if (allow_trailing_junk) { + current = junk_begin; + goto parsing_done; + } else { + return junk_string_value_; + } + } + } + + if (current == end || *current < '0' || *current > '9') { + if (allow_trailing_junk) { + current = junk_begin; + goto parsing_done; + } else { + return junk_string_value_; + } + } + + const int max_exponent = INT_MAX / 2; + DOUBLE_CONVERSION_ASSERT(-max_exponent / 2 <= exponent && exponent <= max_exponent / 2); + int num = 0; + do { + // Check overflow. + int digit = *current - '0'; + if (num >= max_exponent / 10 + && !(num == max_exponent / 10 && digit <= max_exponent % 10)) { + num = max_exponent; + } else { + num = num * 10 + digit; + } + ++current; + } while (current != end && *current >= '0' && *current <= '9'); + + exponent += (exponen_sign == '-' ? -num : num); + } + + if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) { + return junk_string_value_; + } + if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) { + return junk_string_value_; + } + if (allow_trailing_spaces) { + AdvanceToNonspace(¤t, end); + } + + parsing_done: + exponent += insignificant_digits; + + if (octal) { + double result; + bool result_is_junk; + char* start = buffer; + result = RadixStringToIeee<3>(&start, + buffer + buffer_pos, + sign, + separator_, + false, // Don't parse as hex_float. + allow_trailing_junk, + junk_string_value_, + read_as_double, + &result_is_junk); + DOUBLE_CONVERSION_ASSERT(!result_is_junk); + *processed_characters_count = static_cast(current - input); + return result; + } + + if (nonzero_digit_dropped) { + buffer[buffer_pos++] = '1'; + exponent--; + } + + DOUBLE_CONVERSION_ASSERT(buffer_pos < kBufferSize); + buffer[buffer_pos] = '\0'; + + // Code above ensures there are no leading zeros and the buffer has fewer than + // kMaxSignificantDecimalDigits characters. Trim trailing zeros. + Vector chars(buffer, buffer_pos); + chars = TrimTrailingZeros(chars); + exponent += buffer_pos - chars.length(); + + double converted; + if (read_as_double) { + converted = StrtodTrimmed(chars, exponent); + } else { + converted = StrtofTrimmed(chars, exponent); + } + *processed_characters_count = static_cast(current - input); + return sign? -converted: converted; +} + + +double StringToDoubleConverter::StringToDouble( + const char* buffer, + int length, + int* processed_characters_count) const { + return StringToIeee(buffer, length, true, processed_characters_count); +} + + +double StringToDoubleConverter::StringToDouble( + const uc16* buffer, + int length, + int* processed_characters_count) const { + return StringToIeee(buffer, length, true, processed_characters_count); +} + + +float StringToDoubleConverter::StringToFloat( + const char* buffer, + int length, + int* processed_characters_count) const { + return static_cast(StringToIeee(buffer, length, false, + processed_characters_count)); +} + + +float StringToDoubleConverter::StringToFloat( + const uc16* buffer, + int length, + int* processed_characters_count) const { + return static_cast(StringToIeee(buffer, length, false, + processed_characters_count)); +} + + +template<> +double StringToDoubleConverter::StringTo( + const char* buffer, + int length, + int* processed_characters_count) const { + return StringToDouble(buffer, length, processed_characters_count); +} + + +template<> +float StringToDoubleConverter::StringTo( + const char* buffer, + int length, + int* processed_characters_count) const { + return StringToFloat(buffer, length, processed_characters_count); +} + + +template<> +double StringToDoubleConverter::StringTo( + const uc16* buffer, + int length, + int* processed_characters_count) const { + return StringToDouble(buffer, length, processed_characters_count); +} + + +template<> +float StringToDoubleConverter::StringTo( + const uc16* buffer, + int length, + int* processed_characters_count) const { + return StringToFloat(buffer, length, processed_characters_count); +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-string-to-double.h b/intl/icu/source/i18n/double-conversion-string-to-double.h new file mode 100644 index 0000000000..1d4e3dddde --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-string-to-double.h @@ -0,0 +1,256 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2012 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_STRING_TO_DOUBLE_H_ +#define DOUBLE_CONVERSION_STRING_TO_DOUBLE_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +class StringToDoubleConverter { + public: + // Enumeration for allowing octals and ignoring junk when converting + // strings to numbers. + enum Flags { + NO_FLAGS = 0, + ALLOW_HEX = 1, + ALLOW_OCTALS = 2, + ALLOW_TRAILING_JUNK = 4, + ALLOW_LEADING_SPACES = 8, + ALLOW_TRAILING_SPACES = 16, + ALLOW_SPACES_AFTER_SIGN = 32, + ALLOW_CASE_INSENSITIVITY = 64, + ALLOW_CASE_INSENSIBILITY = 64, // Deprecated + ALLOW_HEX_FLOATS = 128, + }; + + static const uc16 kNoSeparator = '\0'; + + // Flags should be a bit-or combination of the possible Flags-enum. + // - NO_FLAGS: no special flags. + // - ALLOW_HEX: recognizes the prefix "0x". Hex numbers may only be integers. + // Ex: StringToDouble("0x1234") -> 4660.0 + // In StringToDouble("0x1234.56") the characters ".56" are trailing + // junk. The result of the call is hence dependent on + // the ALLOW_TRAILING_JUNK flag and/or the junk value. + // With this flag "0x" is a junk-string. Even with ALLOW_TRAILING_JUNK, + // the string will not be parsed as "0" followed by junk. + // + // - ALLOW_OCTALS: recognizes the prefix "0" for octals: + // If a sequence of octal digits starts with '0', then the number is + // read as octal integer. Octal numbers may only be integers. + // Ex: StringToDouble("01234") -> 668.0 + // StringToDouble("012349") -> 12349.0 // Not a sequence of octal + // // digits. + // In StringToDouble("01234.56") the characters ".56" are trailing + // junk. The result of the call is hence dependent on + // the ALLOW_TRAILING_JUNK flag and/or the junk value. + // In StringToDouble("01234e56") the characters "e56" are trailing + // junk, too. + // - ALLOW_TRAILING_JUNK: ignore trailing characters that are not part of + // a double literal. + // - ALLOW_LEADING_SPACES: skip over leading whitespace, including spaces, + // new-lines, and tabs. + // - ALLOW_TRAILING_SPACES: ignore trailing whitespace. + // - ALLOW_SPACES_AFTER_SIGN: ignore whitespace after the sign. + // Ex: StringToDouble("- 123.2") -> -123.2. + // StringToDouble("+ 123.2") -> 123.2 + // - ALLOW_CASE_INSENSITIVITY: ignore case of characters for special values: + // infinity and nan. + // - ALLOW_HEX_FLOATS: allows hexadecimal float literals. + // This *must* start with "0x" and separate the exponent with "p". + // Examples: 0x1.2p3 == 9.0 + // 0x10.1p0 == 16.0625 + // ALLOW_HEX and ALLOW_HEX_FLOATS are indented. + // + // empty_string_value is returned when an empty string is given as input. + // If ALLOW_LEADING_SPACES or ALLOW_TRAILING_SPACES are set, then a string + // containing only spaces is converted to the 'empty_string_value', too. + // + // junk_string_value is returned when + // a) ALLOW_TRAILING_JUNK is not set, and a junk character (a character not + // part of a double-literal) is found. + // b) ALLOW_TRAILING_JUNK is set, but the string does not start with a + // double literal. + // + // infinity_symbol and nan_symbol are strings that are used to detect + // inputs that represent infinity and NaN. They can be null, in which case + // they are ignored. + // The conversion routine first reads any possible signs. Then it compares the + // following character of the input-string with the first character of + // the infinity, and nan-symbol. If either matches, the function assumes, that + // a match has been found, and expects the following input characters to match + // the remaining characters of the special-value symbol. + // This means that the following restrictions apply to special-value symbols: + // - they must not start with signs ('+', or '-'), + // - they must not have the same first character. + // - they must not start with digits. + // + // If the separator character is not kNoSeparator, then that specific + // character is ignored when in between two valid digits of the significant. + // It is not allowed to appear in the exponent. + // It is not allowed to lead or trail the number. + // It is not allowed to appear twice next to each other. + // + // Examples: + // flags = ALLOW_HEX | ALLOW_TRAILING_JUNK, + // empty_string_value = 0.0, + // junk_string_value = NaN, + // infinity_symbol = "infinity", + // nan_symbol = "nan": + // StringToDouble("0x1234") -> 4660.0. + // StringToDouble("0x1234K") -> 4660.0. + // StringToDouble("") -> 0.0 // empty_string_value. + // StringToDouble(" ") -> NaN // junk_string_value. + // StringToDouble(" 1") -> NaN // junk_string_value. + // StringToDouble("0x") -> NaN // junk_string_value. + // StringToDouble("-123.45") -> -123.45. + // StringToDouble("--123.45") -> NaN // junk_string_value. + // StringToDouble("123e45") -> 123e45. + // StringToDouble("123E45") -> 123e45. + // StringToDouble("123e+45") -> 123e45. + // StringToDouble("123E-45") -> 123e-45. + // StringToDouble("123e") -> 123.0 // trailing junk ignored. + // StringToDouble("123e-") -> 123.0 // trailing junk ignored. + // StringToDouble("+NaN") -> NaN // NaN string literal. + // StringToDouble("-infinity") -> -inf. // infinity literal. + // StringToDouble("Infinity") -> NaN // junk_string_value. + // + // flags = ALLOW_OCTAL | ALLOW_LEADING_SPACES, + // empty_string_value = 0.0, + // junk_string_value = NaN, + // infinity_symbol = nullptr, + // nan_symbol = nullptr: + // StringToDouble("0x1234") -> NaN // junk_string_value. + // StringToDouble("01234") -> 668.0. + // StringToDouble("") -> 0.0 // empty_string_value. + // StringToDouble(" ") -> 0.0 // empty_string_value. + // StringToDouble(" 1") -> 1.0 + // StringToDouble("0x") -> NaN // junk_string_value. + // StringToDouble("0123e45") -> NaN // junk_string_value. + // StringToDouble("01239E45") -> 1239e45. + // StringToDouble("-infinity") -> NaN // junk_string_value. + // StringToDouble("NaN") -> NaN // junk_string_value. + // + // flags = NO_FLAGS, + // separator = ' ': + // StringToDouble("1 2 3 4") -> 1234.0 + // StringToDouble("1 2") -> NaN // junk_string_value + // StringToDouble("1 000 000.0") -> 1000000.0 + // StringToDouble("1.000 000") -> 1.0 + // StringToDouble("1.0e1 000") -> NaN // junk_string_value + StringToDoubleConverter(int flags, + double empty_string_value, + double junk_string_value, + const char* infinity_symbol, + const char* nan_symbol, + uc16 separator = kNoSeparator) + : flags_(flags), + empty_string_value_(empty_string_value), + junk_string_value_(junk_string_value), + infinity_symbol_(infinity_symbol), + nan_symbol_(nan_symbol), + separator_(separator) { + } + + // Performs the conversion. + // The output parameter 'processed_characters_count' is set to the number + // of characters that have been processed to read the number. + // Spaces than are processed with ALLOW_{LEADING|TRAILING}_SPACES are included + // in the 'processed_characters_count'. Trailing junk is never included. + double StringToDouble(const char* buffer, + int length, + int* processed_characters_count) const; + + // Same as StringToDouble above but for 16 bit characters. + double StringToDouble(const uc16* buffer, + int length, + int* processed_characters_count) const; + + // Same as StringToDouble but reads a float. + // Note that this is not equivalent to static_cast(StringToDouble(...)) + // due to potential double-rounding. + float StringToFloat(const char* buffer, + int length, + int* processed_characters_count) const; + + // Same as StringToFloat above but for 16 bit characters. + float StringToFloat(const uc16* buffer, + int length, + int* processed_characters_count) const; + + // Same as StringToDouble for T = double, and StringToFloat for T = float. + template + T StringTo(const char* buffer, + int length, + int* processed_characters_count) const; + + // Same as StringTo above but for 16 bit characters. + template + T StringTo(const uc16* buffer, + int length, + int* processed_characters_count) const; + + private: + const int flags_; + const double empty_string_value_; + const double junk_string_value_; + const char* const infinity_symbol_; + const char* const nan_symbol_; + const uc16 separator_; + + template + double StringToIeee(Iterator start_pointer, + int length, + bool read_as_double, + int* processed_characters_count) const; + + DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS(StringToDoubleConverter); +}; + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_STRING_TO_DOUBLE_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-strtod.cpp b/intl/icu/source/i18n/double-conversion-strtod.cpp new file mode 100644 index 0000000000..eea8203281 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-strtod.cpp @@ -0,0 +1,628 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include +#include + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-bignum.h" +#include "double-conversion-cached-powers.h" +#include "double-conversion-ieee.h" +#include "double-conversion-strtod.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +#if defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS) +// 2^53 = 9007199254740992. +// Any integer with at most 15 decimal digits will hence fit into a double +// (which has a 53bit significand) without loss of precision. +static const int kMaxExactDoubleIntegerDecimalDigits = 15; +#endif // #if defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS) +// 2^64 = 18446744073709551616 > 10^19 +static const int kMaxUint64DecimalDigits = 19; + +// Max double: 1.7976931348623157 x 10^308 +// Min non-zero double: 4.9406564584124654 x 10^-324 +// Any x >= 10^309 is interpreted as +infinity. +// Any x <= 10^-324 is interpreted as 0. +// Note that 2.5e-324 (despite being smaller than the min double) will be read +// as non-zero (equal to the min non-zero double). +static const int kMaxDecimalPower = 309; +static const int kMinDecimalPower = -324; + +// 2^64 = 18446744073709551616 +static const uint64_t kMaxUint64 = DOUBLE_CONVERSION_UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF); + + +#if defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS) +static const double exact_powers_of_ten[] = { + 1.0, // 10^0 + 10.0, + 100.0, + 1000.0, + 10000.0, + 100000.0, + 1000000.0, + 10000000.0, + 100000000.0, + 1000000000.0, + 10000000000.0, // 10^10 + 100000000000.0, + 1000000000000.0, + 10000000000000.0, + 100000000000000.0, + 1000000000000000.0, + 10000000000000000.0, + 100000000000000000.0, + 1000000000000000000.0, + 10000000000000000000.0, + 100000000000000000000.0, // 10^20 + 1000000000000000000000.0, + // 10^22 = 0x21e19e0c9bab2400000 = 0x878678326eac9 * 2^22 + 10000000000000000000000.0 +}; +static const int kExactPowersOfTenSize = DOUBLE_CONVERSION_ARRAY_SIZE(exact_powers_of_ten); +#endif // #if defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS) + +// Maximum number of significant digits in the decimal representation. +// In fact the value is 772 (see conversions.cc), but to give us some margin +// we round up to 780. +static const int kMaxSignificantDecimalDigits = 780; + +static Vector TrimLeadingZeros(Vector buffer) { + for (int i = 0; i < buffer.length(); i++) { + if (buffer[i] != '0') { + return buffer.SubVector(i, buffer.length()); + } + } + return Vector(buffer.start(), 0); +} + +static void CutToMaxSignificantDigits(Vector buffer, + int exponent, + char* significant_buffer, + int* significant_exponent) { + for (int i = 0; i < kMaxSignificantDecimalDigits - 1; ++i) { + significant_buffer[i] = buffer[i]; + } + // The input buffer has been trimmed. Therefore the last digit must be + // different from '0'. + DOUBLE_CONVERSION_ASSERT(buffer[buffer.length() - 1] != '0'); + // Set the last digit to be non-zero. This is sufficient to guarantee + // correct rounding. + significant_buffer[kMaxSignificantDecimalDigits - 1] = '1'; + *significant_exponent = + exponent + (buffer.length() - kMaxSignificantDecimalDigits); +} + + +// Trims the buffer and cuts it to at most kMaxSignificantDecimalDigits. +// If possible the input-buffer is reused, but if the buffer needs to be +// modified (due to cutting), then the input needs to be copied into the +// buffer_copy_space. +static void TrimAndCut(Vector buffer, int exponent, + char* buffer_copy_space, int space_size, + Vector* trimmed, int* updated_exponent) { + Vector left_trimmed = TrimLeadingZeros(buffer); + Vector right_trimmed = TrimTrailingZeros(left_trimmed); + exponent += left_trimmed.length() - right_trimmed.length(); + if (right_trimmed.length() > kMaxSignificantDecimalDigits) { + (void) space_size; // Mark variable as used. + DOUBLE_CONVERSION_ASSERT(space_size >= kMaxSignificantDecimalDigits); + CutToMaxSignificantDigits(right_trimmed, exponent, + buffer_copy_space, updated_exponent); + *trimmed = Vector(buffer_copy_space, + kMaxSignificantDecimalDigits); + } else { + *trimmed = right_trimmed; + *updated_exponent = exponent; + } +} + + +// Reads digits from the buffer and converts them to a uint64. +// Reads in as many digits as fit into a uint64. +// When the string starts with "1844674407370955161" no further digit is read. +// Since 2^64 = 18446744073709551616 it would still be possible read another +// digit if it was less or equal than 6, but this would complicate the code. +static uint64_t ReadUint64(Vector buffer, + int* number_of_read_digits) { + uint64_t result = 0; + int i = 0; + while (i < buffer.length() && result <= (kMaxUint64 / 10 - 1)) { + int digit = buffer[i++] - '0'; + DOUBLE_CONVERSION_ASSERT(0 <= digit && digit <= 9); + result = 10 * result + digit; + } + *number_of_read_digits = i; + return result; +} + + +// Reads a DiyFp from the buffer. +// The returned DiyFp is not necessarily normalized. +// If remaining_decimals is zero then the returned DiyFp is accurate. +// Otherwise it has been rounded and has error of at most 1/2 ulp. +static void ReadDiyFp(Vector buffer, + DiyFp* result, + int* remaining_decimals) { + int read_digits; + uint64_t significand = ReadUint64(buffer, &read_digits); + if (buffer.length() == read_digits) { + *result = DiyFp(significand, 0); + *remaining_decimals = 0; + } else { + // Round the significand. + if (buffer[read_digits] >= '5') { + significand++; + } + // Compute the binary exponent. + int exponent = 0; + *result = DiyFp(significand, exponent); + *remaining_decimals = buffer.length() - read_digits; + } +} + + +static bool DoubleStrtod(Vector trimmed, + int exponent, + double* result) { +#if !defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS) + // Avoid "unused parameter" warnings + (void) trimmed; + (void) exponent; + (void) result; + // On x86 the floating-point stack can be 64 or 80 bits wide. If it is + // 80 bits wide (as is the case on Linux) then double-rounding occurs and the + // result is not accurate. + // We know that Windows32 uses 64 bits and is therefore accurate. + return false; +#else + if (trimmed.length() <= kMaxExactDoubleIntegerDecimalDigits) { + int read_digits; + // The trimmed input fits into a double. + // If the 10^exponent (resp. 10^-exponent) fits into a double too then we + // can compute the result-double simply by multiplying (resp. dividing) the + // two numbers. + // This is possible because IEEE guarantees that floating-point operations + // return the best possible approximation. + if (exponent < 0 && -exponent < kExactPowersOfTenSize) { + // 10^-exponent fits into a double. + *result = static_cast(ReadUint64(trimmed, &read_digits)); + DOUBLE_CONVERSION_ASSERT(read_digits == trimmed.length()); + *result /= exact_powers_of_ten[-exponent]; + return true; + } + if (0 <= exponent && exponent < kExactPowersOfTenSize) { + // 10^exponent fits into a double. + *result = static_cast(ReadUint64(trimmed, &read_digits)); + DOUBLE_CONVERSION_ASSERT(read_digits == trimmed.length()); + *result *= exact_powers_of_ten[exponent]; + return true; + } + int remaining_digits = + kMaxExactDoubleIntegerDecimalDigits - trimmed.length(); + if ((0 <= exponent) && + (exponent - remaining_digits < kExactPowersOfTenSize)) { + // The trimmed string was short and we can multiply it with + // 10^remaining_digits. As a result the remaining exponent now fits + // into a double too. + *result = static_cast(ReadUint64(trimmed, &read_digits)); + DOUBLE_CONVERSION_ASSERT(read_digits == trimmed.length()); + *result *= exact_powers_of_ten[remaining_digits]; + *result *= exact_powers_of_ten[exponent - remaining_digits]; + return true; + } + } + return false; +#endif +} + + +// Returns 10^exponent as an exact DiyFp. +// The given exponent must be in the range [1; kDecimalExponentDistance[. +static DiyFp AdjustmentPowerOfTen(int exponent) { + DOUBLE_CONVERSION_ASSERT(0 < exponent); + DOUBLE_CONVERSION_ASSERT(exponent < PowersOfTenCache::kDecimalExponentDistance); + // Simply hardcode the remaining powers for the given decimal exponent + // distance. + DOUBLE_CONVERSION_ASSERT(PowersOfTenCache::kDecimalExponentDistance == 8); + switch (exponent) { + case 1: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0xa0000000, 00000000), -60); + case 2: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0xc8000000, 00000000), -57); + case 3: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0xfa000000, 00000000), -54); + case 4: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0x9c400000, 00000000), -50); + case 5: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0xc3500000, 00000000), -47); + case 6: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0xf4240000, 00000000), -44); + case 7: return DiyFp(DOUBLE_CONVERSION_UINT64_2PART_C(0x98968000, 00000000), -40); + default: + DOUBLE_CONVERSION_UNREACHABLE(); + } +} + + +// If the function returns true then the result is the correct double. +// Otherwise it is either the correct double or the double that is just below +// the correct double. +static bool DiyFpStrtod(Vector buffer, + int exponent, + double* result) { + DiyFp input; + int remaining_decimals; + ReadDiyFp(buffer, &input, &remaining_decimals); + // Since we may have dropped some digits the input is not accurate. + // If remaining_decimals is different than 0 than the error is at most + // .5 ulp (unit in the last place). + // We don't want to deal with fractions and therefore keep a common + // denominator. + const int kDenominatorLog = 3; + const int kDenominator = 1 << kDenominatorLog; + // Move the remaining decimals into the exponent. + exponent += remaining_decimals; + uint64_t error = (remaining_decimals == 0 ? 0 : kDenominator / 2); + + int old_e = input.e(); + input.Normalize(); + error <<= old_e - input.e(); + + DOUBLE_CONVERSION_ASSERT(exponent <= PowersOfTenCache::kMaxDecimalExponent); + if (exponent < PowersOfTenCache::kMinDecimalExponent) { + *result = 0.0; + return true; + } + DiyFp cached_power; + int cached_decimal_exponent; + PowersOfTenCache::GetCachedPowerForDecimalExponent(exponent, + &cached_power, + &cached_decimal_exponent); + + if (cached_decimal_exponent != exponent) { + int adjustment_exponent = exponent - cached_decimal_exponent; + DiyFp adjustment_power = AdjustmentPowerOfTen(adjustment_exponent); + input.Multiply(adjustment_power); + if (kMaxUint64DecimalDigits - buffer.length() >= adjustment_exponent) { + // The product of input with the adjustment power fits into a 64 bit + // integer. + DOUBLE_CONVERSION_ASSERT(DiyFp::kSignificandSize == 64); + } else { + // The adjustment power is exact. There is hence only an error of 0.5. + error += kDenominator / 2; + } + } + + input.Multiply(cached_power); + // The error introduced by a multiplication of a*b equals + // error_a + error_b + error_a*error_b/2^64 + 0.5 + // Substituting a with 'input' and b with 'cached_power' we have + // error_b = 0.5 (all cached powers have an error of less than 0.5 ulp), + // error_ab = 0 or 1 / kDenominator > error_a*error_b/ 2^64 + int error_b = kDenominator / 2; + int error_ab = (error == 0 ? 0 : 1); // We round up to 1. + int fixed_error = kDenominator / 2; + error += error_b + error_ab + fixed_error; + + old_e = input.e(); + input.Normalize(); + error <<= old_e - input.e(); + + // See if the double's significand changes if we add/subtract the error. + int order_of_magnitude = DiyFp::kSignificandSize + input.e(); + int effective_significand_size = + Double::SignificandSizeForOrderOfMagnitude(order_of_magnitude); + int precision_digits_count = + DiyFp::kSignificandSize - effective_significand_size; + if (precision_digits_count + kDenominatorLog >= DiyFp::kSignificandSize) { + // This can only happen for very small denormals. In this case the + // half-way multiplied by the denominator exceeds the range of an uint64. + // Simply shift everything to the right. + int shift_amount = (precision_digits_count + kDenominatorLog) - + DiyFp::kSignificandSize + 1; + input.set_f(input.f() >> shift_amount); + input.set_e(input.e() + shift_amount); + // We add 1 for the lost precision of error, and kDenominator for + // the lost precision of input.f(). + error = (error >> shift_amount) + 1 + kDenominator; + precision_digits_count -= shift_amount; + } + // We use uint64_ts now. This only works if the DiyFp uses uint64_ts too. + DOUBLE_CONVERSION_ASSERT(DiyFp::kSignificandSize == 64); + DOUBLE_CONVERSION_ASSERT(precision_digits_count < 64); + uint64_t one64 = 1; + uint64_t precision_bits_mask = (one64 << precision_digits_count) - 1; + uint64_t precision_bits = input.f() & precision_bits_mask; + uint64_t half_way = one64 << (precision_digits_count - 1); + precision_bits *= kDenominator; + half_way *= kDenominator; + DiyFp rounded_input(input.f() >> precision_digits_count, + input.e() + precision_digits_count); + if (precision_bits >= half_way + error) { + rounded_input.set_f(rounded_input.f() + 1); + } + // If the last_bits are too close to the half-way case than we are too + // inaccurate and round down. In this case we return false so that we can + // fall back to a more precise algorithm. + + *result = Double(rounded_input).value(); + if (half_way - error < precision_bits && precision_bits < half_way + error) { + // Too imprecise. The caller will have to fall back to a slower version. + // However the returned number is guaranteed to be either the correct + // double, or the next-lower double. + return false; + } else { + return true; + } +} + + +// Returns +// - -1 if buffer*10^exponent < diy_fp. +// - 0 if buffer*10^exponent == diy_fp. +// - +1 if buffer*10^exponent > diy_fp. +// Preconditions: +// buffer.length() + exponent <= kMaxDecimalPower + 1 +// buffer.length() + exponent > kMinDecimalPower +// buffer.length() <= kMaxDecimalSignificantDigits +static int CompareBufferWithDiyFp(Vector buffer, + int exponent, + DiyFp diy_fp) { + DOUBLE_CONVERSION_ASSERT(buffer.length() + exponent <= kMaxDecimalPower + 1); + DOUBLE_CONVERSION_ASSERT(buffer.length() + exponent > kMinDecimalPower); + DOUBLE_CONVERSION_ASSERT(buffer.length() <= kMaxSignificantDecimalDigits); + // Make sure that the Bignum will be able to hold all our numbers. + // Our Bignum implementation has a separate field for exponents. Shifts will + // consume at most one bigit (< 64 bits). + // ln(10) == 3.3219... + DOUBLE_CONVERSION_ASSERT(((kMaxDecimalPower + 1) * 333 / 100) < Bignum::kMaxSignificantBits); + Bignum buffer_bignum; + Bignum diy_fp_bignum; + buffer_bignum.AssignDecimalString(buffer); + diy_fp_bignum.AssignUInt64(diy_fp.f()); + if (exponent >= 0) { + buffer_bignum.MultiplyByPowerOfTen(exponent); + } else { + diy_fp_bignum.MultiplyByPowerOfTen(-exponent); + } + if (diy_fp.e() > 0) { + diy_fp_bignum.ShiftLeft(diy_fp.e()); + } else { + buffer_bignum.ShiftLeft(-diy_fp.e()); + } + return Bignum::Compare(buffer_bignum, diy_fp_bignum); +} + + +// Returns true if the guess is the correct double. +// Returns false, when guess is either correct or the next-lower double. +static bool ComputeGuess(Vector trimmed, int exponent, + double* guess) { + if (trimmed.length() == 0) { + *guess = 0.0; + return true; + } + if (exponent + trimmed.length() - 1 >= kMaxDecimalPower) { + *guess = Double::Infinity(); + return true; + } + if (exponent + trimmed.length() <= kMinDecimalPower) { + *guess = 0.0; + return true; + } + + if (DoubleStrtod(trimmed, exponent, guess) || + DiyFpStrtod(trimmed, exponent, guess)) { + return true; + } + if (*guess == Double::Infinity()) { + return true; + } + return false; +} + +#if U_DEBUG // needed for ICU only in debug mode +static bool IsDigit(const char d) { + return ('0' <= d) && (d <= '9'); +} + +static bool IsNonZeroDigit(const char d) { + return ('1' <= d) && (d <= '9'); +} + +#ifdef __has_cpp_attribute +#if __has_cpp_attribute(maybe_unused) +[[maybe_unused]] +#endif +#endif +static bool AssertTrimmedDigits(const Vector& buffer) { + for(int i = 0; i < buffer.length(); ++i) { + if(!IsDigit(buffer[i])) { + return false; + } + } + return (buffer.length() == 0) || (IsNonZeroDigit(buffer[0]) && IsNonZeroDigit(buffer[buffer.length()-1])); +} +#endif // needed for ICU only in debug mode + +double StrtodTrimmed(Vector trimmed, int exponent) { + DOUBLE_CONVERSION_ASSERT(trimmed.length() <= kMaxSignificantDecimalDigits); + DOUBLE_CONVERSION_ASSERT(AssertTrimmedDigits(trimmed)); + double guess; + const bool is_correct = ComputeGuess(trimmed, exponent, &guess); + if (is_correct) { + return guess; + } + DiyFp upper_boundary = Double(guess).UpperBoundary(); + int comparison = CompareBufferWithDiyFp(trimmed, exponent, upper_boundary); + if (comparison < 0) { + return guess; + } else if (comparison > 0) { + return Double(guess).NextDouble(); + } else if ((Double(guess).Significand() & 1) == 0) { + // Round towards even. + return guess; + } else { + return Double(guess).NextDouble(); + } +} + +double Strtod(Vector buffer, int exponent) { + char copy_buffer[kMaxSignificantDecimalDigits]; + Vector trimmed; + int updated_exponent; + TrimAndCut(buffer, exponent, copy_buffer, kMaxSignificantDecimalDigits, + &trimmed, &updated_exponent); + return StrtodTrimmed(trimmed, updated_exponent); +} + +static float SanitizedDoubletof(double d) { + DOUBLE_CONVERSION_ASSERT(d >= 0.0); + // ASAN has a sanitize check that disallows casting doubles to floats if + // they are too big. + // https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html#available-checks + // The behavior should be covered by IEEE 754, but some projects use this + // flag, so work around it. + float max_finite = 3.4028234663852885981170418348451692544e+38; + // The half-way point between the max-finite and infinity value. + // Since infinity has an even significand everything equal or greater than + // this value should become infinity. + double half_max_finite_infinity = + 3.40282356779733661637539395458142568448e+38; + if (d >= max_finite) { + if (d >= half_max_finite_infinity) { + return Single::Infinity(); + } else { + return max_finite; + } + } else { + return static_cast(d); + } +} + +float Strtof(Vector buffer, int exponent) { + char copy_buffer[kMaxSignificantDecimalDigits]; + Vector trimmed; + int updated_exponent; + TrimAndCut(buffer, exponent, copy_buffer, kMaxSignificantDecimalDigits, + &trimmed, &updated_exponent); + exponent = updated_exponent; + return StrtofTrimmed(trimmed, exponent); +} + +float StrtofTrimmed(Vector trimmed, int exponent) { + DOUBLE_CONVERSION_ASSERT(trimmed.length() <= kMaxSignificantDecimalDigits); + DOUBLE_CONVERSION_ASSERT(AssertTrimmedDigits(trimmed)); + + double double_guess; + bool is_correct = ComputeGuess(trimmed, exponent, &double_guess); + + float float_guess = SanitizedDoubletof(double_guess); + if (float_guess == double_guess) { + // This shortcut triggers for integer values. + return float_guess; + } + + // We must catch double-rounding. Say the double has been rounded up, and is + // now a boundary of a float, and rounds up again. This is why we have to + // look at previous too. + // Example (in decimal numbers): + // input: 12349 + // high-precision (4 digits): 1235 + // low-precision (3 digits): + // when read from input: 123 + // when rounded from high precision: 124. + // To do this we simply look at the neighbors of the correct result and see + // if they would round to the same float. If the guess is not correct we have + // to look at four values (since two different doubles could be the correct + // double). + + double double_next = Double(double_guess).NextDouble(); + double double_previous = Double(double_guess).PreviousDouble(); + + float f1 = SanitizedDoubletof(double_previous); + float f2 = float_guess; + float f3 = SanitizedDoubletof(double_next); + float f4; + if (is_correct) { + f4 = f3; + } else { + double double_next2 = Double(double_next).NextDouble(); + f4 = SanitizedDoubletof(double_next2); + } + (void) f2; // Mark variable as used. + DOUBLE_CONVERSION_ASSERT(f1 <= f2 && f2 <= f3 && f3 <= f4); + + // If the guess doesn't lie near a single-precision boundary we can simply + // return its float-value. + if (f1 == f4) { + return float_guess; + } + + DOUBLE_CONVERSION_ASSERT((f1 != f2 && f2 == f3 && f3 == f4) || + (f1 == f2 && f2 != f3 && f3 == f4) || + (f1 == f2 && f2 == f3 && f3 != f4)); + + // guess and next are the two possible candidates (in the same way that + // double_guess was the lower candidate for a double-precision guess). + float guess = f1; + float next = f4; + DiyFp upper_boundary; + if (guess == 0.0f) { + float min_float = 1e-45f; + upper_boundary = Double(static_cast(min_float) / 2).AsDiyFp(); + } else { + upper_boundary = Single(guess).UpperBoundary(); + } + int comparison = CompareBufferWithDiyFp(trimmed, exponent, upper_boundary); + if (comparison < 0) { + return guess; + } else if (comparison > 0) { + return next; + } else if ((Single(guess).Significand() & 1) == 0) { + // Round towards even. + return guess; + } else { + return next; + } +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-strtod.h b/intl/icu/source/i18n/double-conversion-strtod.h new file mode 100644 index 0000000000..abfe00a333 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-strtod.h @@ -0,0 +1,82 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_STRTOD_H_ +#define DOUBLE_CONVERSION_STRTOD_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-utils.h" + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +// The buffer must only contain digits in the range [0-9]. It must not +// contain a dot or a sign. It must not start with '0', and must not be empty. +double Strtod(Vector buffer, int exponent); + +// The buffer must only contain digits in the range [0-9]. It must not +// contain a dot or a sign. It must not start with '0', and must not be empty. +float Strtof(Vector buffer, int exponent); + +// Same as Strtod, but assumes that 'trimmed' is already trimmed, as if run +// through TrimAndCut. That is, 'trimmed' must have no leading or trailing +// zeros, must not be a lone zero, and must not have 'too many' digits. +double StrtodTrimmed(Vector trimmed, int exponent); + +// Same as Strtof, but assumes that 'trimmed' is already trimmed, as if run +// through TrimAndCut. That is, 'trimmed' must have no leading or trailing +// zeros, must not be a lone zero, and must not have 'too many' digits. +float StrtofTrimmed(Vector trimmed, int exponent); + +inline Vector TrimTrailingZeros(Vector buffer) { + for (int i = buffer.length() - 1; i >= 0; --i) { + if (buffer[i] != '0') { + return buffer.SubVector(0, i + 1); + } + } + return Vector(buffer.start(), 0); +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_STRTOD_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion-utils.h b/intl/icu/source/i18n/double-conversion-utils.h new file mode 100644 index 0000000000..303668f931 --- /dev/null +++ b/intl/icu/source/i18n/double-conversion-utils.h @@ -0,0 +1,435 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2010 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_UTILS_H_ +#define DOUBLE_CONVERSION_UTILS_H_ + +// Use DOUBLE_CONVERSION_NON_PREFIXED_MACROS to get unprefixed macros as was +// the case in double-conversion releases prior to 3.1.6 + +#include +#include + +// For pre-C++11 compatibility +#if __cplusplus >= 201103L +#define DOUBLE_CONVERSION_NULLPTR nullptr +#else +#define DOUBLE_CONVERSION_NULLPTR NULL +#endif + +// ICU PATCH: Use U_ASSERT instead of +#include "uassert.h" +#ifndef DOUBLE_CONVERSION_ASSERT +#define DOUBLE_CONVERSION_ASSERT(condition) \ + U_ASSERT(condition) +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(ASSERT) +#define ASSERT DOUBLE_CONVERSION_ASSERT +#endif + +#ifndef DOUBLE_CONVERSION_UNIMPLEMENTED +#define DOUBLE_CONVERSION_UNIMPLEMENTED() (abort()) +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(UNIMPLEMENTED) +#define UNIMPLEMENTED DOUBLE_CONVERSION_UNIMPLEMENTED +#endif + +#ifndef DOUBLE_CONVERSION_NO_RETURN +#ifdef _MSC_VER +#define DOUBLE_CONVERSION_NO_RETURN __declspec(noreturn) +#else +#define DOUBLE_CONVERSION_NO_RETURN __attribute__((noreturn)) +#endif +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(NO_RETURN) +#define NO_RETURN DOUBLE_CONVERSION_NO_RETURN +#endif + +#ifndef DOUBLE_CONVERSION_UNREACHABLE +#ifdef _MSC_VER +void DOUBLE_CONVERSION_NO_RETURN abort_noreturn(); +inline void abort_noreturn() { abort(); } +#define DOUBLE_CONVERSION_UNREACHABLE() (abort_noreturn()) +#else +#define DOUBLE_CONVERSION_UNREACHABLE() (abort()) +#endif +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(UNREACHABLE) +#define UNREACHABLE DOUBLE_CONVERSION_UNREACHABLE +#endif + +// Not all compilers support __has_attribute and combining a check for both +// ifdef and __has_attribute on the same preprocessor line isn't portable. +#ifdef __has_attribute +# define DOUBLE_CONVERSION_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define DOUBLE_CONVERSION_HAS_ATTRIBUTE(x) 0 +#endif + +#ifndef DOUBLE_CONVERSION_UNUSED +#if DOUBLE_CONVERSION_HAS_ATTRIBUTE(unused) +#define DOUBLE_CONVERSION_UNUSED __attribute__((unused)) +#else +#define DOUBLE_CONVERSION_UNUSED +#endif +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(UNUSED) +#define UNUSED DOUBLE_CONVERSION_UNUSED +#endif + +#if DOUBLE_CONVERSION_HAS_ATTRIBUTE(uninitialized) +#define DOUBLE_CONVERSION_STACK_UNINITIALIZED __attribute__((uninitialized)) +#else +#define DOUBLE_CONVERSION_STACK_UNINITIALIZED +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(STACK_UNINITIALIZED) +#define STACK_UNINITIALIZED DOUBLE_CONVERSION_STACK_UNINITIALIZED +#endif + +// Double operations detection based on target architecture. +// Linux uses a 80bit wide floating point stack on x86. This induces double +// rounding, which in turn leads to wrong results. +// An easy way to test if the floating-point operations are correct is to +// evaluate: 89255.0/1e22. If the floating-point stack is 64 bits wide then +// the result is equal to 89255e-22. +// The best way to test this, is to create a division-function and to compare +// the output of the division with the expected result. (Inlining must be +// disabled.) +// On Linux,x86 89255e-22 != Div_double(89255.0/1e22) +// +// For example: +/* +// -- in div.c +double Div_double(double x, double y) { return x / y; } + +// -- in main.c +double Div_double(double x, double y); // Forward declaration. + +int main(int argc, char** argv) { + return Div_double(89255.0, 1e22) == 89255e-22; +} +*/ +// Run as follows ./main || echo "correct" +// +// If it prints "correct" then the architecture should be here, in the "correct" section. +#if defined(_M_X64) || defined(__x86_64__) || \ + defined(__ARMEL__) || defined(__avr32__) || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(__hppa__) || defined(__ia64__) || \ + defined(__mips__) || \ + defined(__loongarch__) || \ + defined(__nios2__) || defined(__ghs) || \ + defined(__powerpc__) || defined(__ppc__) || defined(__ppc64__) || \ + defined(_POWER) || defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \ + defined(__sparc__) || defined(__sparc) || defined(__s390__) || \ + defined(__SH4__) || defined(__alpha__) || \ + defined(_MIPS_ARCH_MIPS32R2) || defined(__ARMEB__) ||\ + defined(__AARCH64EL__) || defined(__aarch64__) || defined(__AARCH64EB__) || \ + defined(__riscv) || defined(__e2k__) || \ + defined(__or1k__) || defined(__arc__) || defined(__ARC64__) || \ + defined(__microblaze__) || defined(__XTENSA__) || \ + defined(__EMSCRIPTEN__) || defined(__wasm32__) +#define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1 +#elif defined(__mc68000__) || \ + defined(__pnacl__) || defined(__native_client__) +#undef DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS +#elif defined(_M_IX86) || defined(__i386__) || defined(__i386) +#if defined(_WIN32) +// Windows uses a 64bit wide floating point stack. +#define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1 +#else +#undef DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS +#endif // _WIN32 +#else +#error Target architecture was not detected as supported by Double-Conversion. +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(CORRECT_DOUBLE_OPERATIONS) +#define CORRECT_DOUBLE_OPERATIONS DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef short int16_t; // NOLINT +typedef unsigned short uint16_t; // NOLINT +typedef int int32_t; +typedef unsigned int uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +// intptr_t and friends are defined in crtdefs.h through stdio.h. + +#else + +#include + +#endif + +typedef uint16_t uc16; + +// The following macro works on both 32 and 64-bit platforms. +// Usage: instead of writing 0x1234567890123456 +// write DOUBLE_CONVERSION_UINT64_2PART_C(0x12345678,90123456); +#define DOUBLE_CONVERSION_UINT64_2PART_C(a, b) (((static_cast(a) << 32) + 0x##b##u)) +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(UINT64_2PART_C) +#define UINT64_2PART_C DOUBLE_CONVERSION_UINT64_2PART_C +#endif + +// The expression DOUBLE_CONVERSION_ARRAY_SIZE(a) is a compile-time constant of type +// size_t which represents the number of elements of the given +// array. You should only use DOUBLE_CONVERSION_ARRAY_SIZE on statically allocated +// arrays. +#ifndef DOUBLE_CONVERSION_ARRAY_SIZE +#define DOUBLE_CONVERSION_ARRAY_SIZE(a) \ + ((sizeof(a) / sizeof(*(a))) / \ + static_cast(!(sizeof(a) % sizeof(*(a))))) +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(ARRAY_SIZE) +#define ARRAY_SIZE DOUBLE_CONVERSION_ARRAY_SIZE +#endif + +// A macro to disallow the evil copy constructor and operator= functions +// This should be used in the private: declarations for a class +#ifndef DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN +#define DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(DC_DISALLOW_COPY_AND_ASSIGN) +#define DC_DISALLOW_COPY_AND_ASSIGN DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN +#endif + +// A macro to disallow all the implicit constructors, namely the +// default constructor, copy constructor and operator= functions. +// +// This should be used in the private: declarations for a class +// that wants to prevent anyone from instantiating it. This is +// especially useful for classes containing only static methods. +#ifndef DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS +#define DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ + TypeName(); \ + DOUBLE_CONVERSION_DISALLOW_COPY_AND_ASSIGN(TypeName) +#endif +#if defined(DOUBLE_CONVERSION_NON_PREFIXED_MACROS) && !defined(DC_DISALLOW_IMPLICIT_CONSTRUCTORS) +#define DC_DISALLOW_IMPLICIT_CONSTRUCTORS DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS +#endif + +// ICU PATCH: Wrap in ICU namespace +U_NAMESPACE_BEGIN + +namespace double_conversion { + +inline int StrLength(const char* string) { + size_t length = strlen(string); + DOUBLE_CONVERSION_ASSERT(length == static_cast(static_cast(length))); + return static_cast(length); +} + +// This is a simplified version of V8's Vector class. +template +class Vector { + public: + Vector() : start_(DOUBLE_CONVERSION_NULLPTR), length_(0) {} + Vector(T* data, int len) : start_(data), length_(len) { + DOUBLE_CONVERSION_ASSERT(len == 0 || (len > 0 && data != DOUBLE_CONVERSION_NULLPTR)); + } + + // Returns a vector using the same backing storage as this one, + // spanning from and including 'from', to but not including 'to'. + Vector SubVector(int from, int to) { + DOUBLE_CONVERSION_ASSERT(to <= length_); + DOUBLE_CONVERSION_ASSERT(from < to); + DOUBLE_CONVERSION_ASSERT(0 <= from); + return Vector(start() + from, to - from); + } + + // Returns the length of the vector. + int length() const { return length_; } + + // Returns whether or not the vector is empty. + bool is_empty() const { return length_ == 0; } + + // Returns the pointer to the start of the data in the vector. + T* start() const { return start_; } + + // Access individual vector elements - checks bounds in debug mode. + T& operator[](int index) const { + DOUBLE_CONVERSION_ASSERT(0 <= index && index < length_); + return start_[index]; + } + + T& first() { return start_[0]; } + + T& last() { return start_[length_ - 1]; } + + void pop_back() { + DOUBLE_CONVERSION_ASSERT(!is_empty()); + --length_; + } + + private: + T* start_; + int length_; +}; + + +// Helper class for building result strings in a character buffer. The +// purpose of the class is to use safe operations that checks the +// buffer bounds on all operations in debug mode. +class StringBuilder { + public: + StringBuilder(char* buffer, int buffer_size) + : buffer_(buffer, buffer_size), position_(0) { } + + ~StringBuilder() { if (!is_finalized()) Finalize(); } + + int size() const { return buffer_.length(); } + + // Get the current position in the builder. + int position() const { + DOUBLE_CONVERSION_ASSERT(!is_finalized()); + return position_; + } + + // Reset the position. + void Reset() { position_ = 0; } + + // Add a single character to the builder. It is not allowed to add + // 0-characters; use the Finalize() method to terminate the string + // instead. + void AddCharacter(char c) { + DOUBLE_CONVERSION_ASSERT(c != '\0'); + DOUBLE_CONVERSION_ASSERT(!is_finalized() && position_ < buffer_.length()); + buffer_[position_++] = c; + } + + // Add an entire string to the builder. Uses strlen() internally to + // compute the length of the input string. + void AddString(const char* s) { + AddSubstring(s, StrLength(s)); + } + + // Add the first 'n' characters of the given string 's' to the + // builder. The input string must have enough characters. + void AddSubstring(const char* s, int n) { + DOUBLE_CONVERSION_ASSERT(!is_finalized() && position_ + n < buffer_.length()); + DOUBLE_CONVERSION_ASSERT(static_cast(n) <= strlen(s)); + memmove(&buffer_[position_], s, static_cast(n)); + position_ += n; + } + + + // Add character padding to the builder. If count is non-positive, + // nothing is added to the builder. + void AddPadding(char c, int count) { + for (int i = 0; i < count; i++) { + AddCharacter(c); + } + } + + // Finalize the string by 0-terminating it and returning the buffer. + char* Finalize() { + DOUBLE_CONVERSION_ASSERT(!is_finalized() && position_ < buffer_.length()); + buffer_[position_] = '\0'; + // Make sure nobody managed to add a 0-character to the + // buffer while building the string. + DOUBLE_CONVERSION_ASSERT(strlen(buffer_.start()) == static_cast(position_)); + position_ = -1; + DOUBLE_CONVERSION_ASSERT(is_finalized()); + return buffer_.start(); + } + + private: + Vector buffer_; + int position_; + + bool is_finalized() const { return position_ < 0; } + + DOUBLE_CONVERSION_DISALLOW_IMPLICIT_CONSTRUCTORS(StringBuilder); +}; + +// The type-based aliasing rule allows the compiler to assume that pointers of +// different types (for some definition of different) never alias each other. +// Thus the following code does not work: +// +// float f = foo(); +// int fbits = *(int*)(&f); +// +// The compiler 'knows' that the int pointer can't refer to f since the types +// don't match, so the compiler may cache f in a register, leaving random data +// in fbits. Using C++ style casts makes no difference, however a pointer to +// char data is assumed to alias any other pointer. This is the 'memcpy +// exception'. +// +// Bit_cast uses the memcpy exception to move the bits from a variable of one +// type of a variable of another type. Of course the end result is likely to +// be implementation dependent. Most compilers (gcc-4.2 and MSVC 2005) +// will completely optimize BitCast away. +// +// There is an additional use for BitCast. +// Recent gccs will warn when they see casts that may result in breakage due to +// the type-based aliasing rule. If you have checked that there is no breakage +// you can use BitCast to cast one pointer type to another. This confuses gcc +// enough that it can no longer see that you have cast one pointer type to +// another thus avoiding the warning. +template +Dest BitCast(const Source& source) { + // Compile time assertion: sizeof(Dest) == sizeof(Source) + // A compile error here means your Dest and Source have different sizes. +#if __cplusplus >= 201103L + static_assert(sizeof(Dest) == sizeof(Source), + "source and destination size mismatch"); +#else + DOUBLE_CONVERSION_UNUSED + typedef char VerifySizesAreEqual[sizeof(Dest) == sizeof(Source) ? 1 : -1]; +#endif + + Dest dest; + memmove(&dest, &source, sizeof(dest)); + return dest; +} + +template +Dest BitCast(Source* source) { + return BitCast(reinterpret_cast(source)); +} + +} // namespace double_conversion + +// ICU PATCH: Close ICU namespace +U_NAMESPACE_END + +#endif // DOUBLE_CONVERSION_UTILS_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/double-conversion.h b/intl/icu/source/i18n/double-conversion.h new file mode 100644 index 0000000000..eddc38763b --- /dev/null +++ b/intl/icu/source/i18n/double-conversion.h @@ -0,0 +1,46 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// From the double-conversion library. Original license: +// +// Copyright 2012 the V8 project authors. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ICU PATCH: ifdef around UCONFIG_NO_FORMATTING +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#ifndef DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_ +#define DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_ + +// ICU PATCH: Customize header file paths for ICU. + +#include "double-conversion-string-to-double.h" +#include "double-conversion-double-to-string.h" + +#endif // DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_ +#endif // ICU PATCH: close #if !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/dt_impl.h b/intl/icu/source/i18n/dt_impl.h new file mode 100644 index 0000000000..a21b68ce1e --- /dev/null +++ b/intl/icu/source/i18n/dt_impl.h @@ -0,0 +1,92 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File dt_impl.h +* +******************************************************************************* +*/ + + +#ifndef DT_IMPL_H__ +#define DT_IMPL_H__ + +/** + * \file + * \brief C++ API: Defines macros for interval format implementation + */ + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/unistr.h" + + +#define QUOTE ((char16_t)0x0027) +#define LOW_LINE ((char16_t)0x005F) +#define COLON ((char16_t)0x003A) +#define LEFT_CURLY_BRACKET ((char16_t)0x007B) +#define RIGHT_CURLY_BRACKET ((char16_t)0x007D) +#define SPACE ((char16_t)0x0020) +#define EN_DASH ((char16_t)0x2013) +#define SOLIDUS ((char16_t)0x002F) +#define PERCENT ((char16_t)0x0025) + +#define DIGIT_ZERO ((char16_t)0x0030) +#define DIGIT_ONE ((char16_t)0x0031) + +#define LOW_A ((char16_t)0x0061) +#define LOW_B ((char16_t)0x0062) +#define LOW_C ((char16_t)0x0063) +#define LOW_D ((char16_t)0x0064) +#define LOW_E ((char16_t)0x0065) +#define LOW_F ((char16_t)0x0066) +#define LOW_G ((char16_t)0x0067) +#define LOW_H ((char16_t)0x0068) +#define LOW_I ((char16_t)0x0069) +#define LOW_J ((char16_t)0x006a) +#define LOW_K ((char16_t)0x006B) +#define LOW_L ((char16_t)0x006C) +#define LOW_M ((char16_t)0x006D) +#define LOW_N ((char16_t)0x006E) +#define LOW_O ((char16_t)0x006F) +#define LOW_P ((char16_t)0x0070) +#define LOW_Q ((char16_t)0x0071) +#define LOW_R ((char16_t)0x0072) +#define LOW_S ((char16_t)0x0073) +#define LOW_T ((char16_t)0x0074) +#define LOW_U ((char16_t)0x0075) +#define LOW_V ((char16_t)0x0076) +#define LOW_W ((char16_t)0x0077) +#define LOW_Y ((char16_t)0x0079) +#define LOW_Z ((char16_t)0x007A) + +#define CAP_A ((char16_t)0x0041) +#define CAP_C ((char16_t)0x0043) +#define CAP_D ((char16_t)0x0044) +#define CAP_E ((char16_t)0x0045) +#define CAP_F ((char16_t)0x0046) +#define CAP_G ((char16_t)0x0047) +#define CAP_H ((char16_t)0x0048) +#define CAP_K ((char16_t)0x004B) +#define CAP_L ((char16_t)0x004C) +#define CAP_M ((char16_t)0x004D) +#define CAP_N ((char16_t)0x004E) +#define CAP_O ((char16_t)0x004F) +#define CAP_P ((char16_t)0x0050) +#define CAP_Q ((char16_t)0x0051) +#define CAP_S ((char16_t)0x0053) +#define CAP_T ((char16_t)0x0054) +#define CAP_U ((char16_t)0x0055) +#define CAP_V ((char16_t)0x0056) +#define CAP_W ((char16_t)0x0057) +#define CAP_Y ((char16_t)0x0059) +#define CAP_Z ((char16_t)0x005A) + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif +//eof diff --git a/intl/icu/source/i18n/dtfmtsym.cpp b/intl/icu/source/i18n/dtfmtsym.cpp new file mode 100644 index 0000000000..943f6e21d2 --- /dev/null +++ b/intl/icu/source/i18n/dtfmtsym.cpp @@ -0,0 +1,2556 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2016, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File DTFMTSYM.CPP +* +* Modification History: +* +* Date Name Description +* 02/19/97 aliu Converted from java. +* 07/21/98 stephen Added getZoneIndex +* Changed weekdays/short weekdays to be one-based +* 06/14/99 stephen Removed SimpleDateFormat::fgTimeZoneDataSuffix +* 11/16/99 weiv Added 'Y' and 'e' to fgPatternChars +* 03/27/00 weiv Keeping resource bundle around! +* 06/30/05 emmons Added eraNames, narrow month/day, standalone context +* 10/12/05 emmons Added setters for eraNames, month/day by width/context +******************************************************************************* +*/ + +#include + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#include "unicode/ustring.h" +#include "unicode/localpointer.h" +#include "unicode/dtfmtsym.h" +#include "unicode/smpdtfmt.h" +#include "unicode/msgfmt.h" +#include "unicode/numsys.h" +#include "unicode/tznames.h" +#include "cpputils.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "charstr.h" +#include "dt_impl.h" +#include "locbased.h" +#include "gregoimp.h" +#include "hash.h" +#include "uassert.h" +#include "uresimp.h" +#include "ureslocs.h" +#include "uvector.h" +#include "shareddateformatsymbols.h" +#include "unicode/calendar.h" +#include "unifiedcache.h" + +// ***************************************************************************** +// class DateFormatSymbols +// ***************************************************************************** + +/** + * These are static arrays we use only in the case where we have no + * resource data. + */ + +#if UDAT_HAS_PATTERN_CHAR_FOR_TIME_SEPARATOR +#define PATTERN_CHARS_LEN 38 +#else +#define PATTERN_CHARS_LEN 37 +#endif + +/** + * Unlocalized date-time pattern characters. For example: 'y', 'd', etc. All + * locales use the same these unlocalized pattern characters. + */ +static const char16_t gPatternChars[] = { + // if UDAT_HAS_PATTERN_CHAR_FOR_TIME_SEPARATOR: + // GyMdkHmsSEDFwWahKzYeugAZvcLQqVUOXxrbB: + // else: + // GyMdkHmsSEDFwWahKzYeugAZvcLQqVUOXxrbB + + 0x47, 0x79, 0x4D, 0x64, 0x6B, 0x48, 0x6D, 0x73, 0x53, 0x45, + 0x44, 0x46, 0x77, 0x57, 0x61, 0x68, 0x4B, 0x7A, 0x59, 0x65, + 0x75, 0x67, 0x41, 0x5A, 0x76, 0x63, 0x4c, 0x51, 0x71, 0x56, + 0x55, 0x4F, 0x58, 0x78, 0x72, 0x62, 0x42, +#if UDAT_HAS_PATTERN_CHAR_FOR_TIME_SEPARATOR + 0x3a, +#endif + 0 +}; + +//------------------------------------------------------ +// Strings of last resort. These are only used if we have no resource +// files. They aren't designed for actual use, just for backup. + +// These are the month names and abbreviations of last resort. +static const char16_t gLastResortMonthNames[13][3] = +{ + {0x0030, 0x0031, 0x0000}, /* "01" */ + {0x0030, 0x0032, 0x0000}, /* "02" */ + {0x0030, 0x0033, 0x0000}, /* "03" */ + {0x0030, 0x0034, 0x0000}, /* "04" */ + {0x0030, 0x0035, 0x0000}, /* "05" */ + {0x0030, 0x0036, 0x0000}, /* "06" */ + {0x0030, 0x0037, 0x0000}, /* "07" */ + {0x0030, 0x0038, 0x0000}, /* "08" */ + {0x0030, 0x0039, 0x0000}, /* "09" */ + {0x0031, 0x0030, 0x0000}, /* "10" */ + {0x0031, 0x0031, 0x0000}, /* "11" */ + {0x0031, 0x0032, 0x0000}, /* "12" */ + {0x0031, 0x0033, 0x0000} /* "13" */ +}; + +// These are the weekday names and abbreviations of last resort. +static const char16_t gLastResortDayNames[8][2] = +{ + {0x0030, 0x0000}, /* "0" */ + {0x0031, 0x0000}, /* "1" */ + {0x0032, 0x0000}, /* "2" */ + {0x0033, 0x0000}, /* "3" */ + {0x0034, 0x0000}, /* "4" */ + {0x0035, 0x0000}, /* "5" */ + {0x0036, 0x0000}, /* "6" */ + {0x0037, 0x0000} /* "7" */ +}; + +// These are the quarter names and abbreviations of last resort. +static const char16_t gLastResortQuarters[4][2] = +{ + {0x0031, 0x0000}, /* "1" */ + {0x0032, 0x0000}, /* "2" */ + {0x0033, 0x0000}, /* "3" */ + {0x0034, 0x0000}, /* "4" */ +}; + +// These are the am/pm and BC/AD markers of last resort. +static const char16_t gLastResortAmPmMarkers[2][3] = +{ + {0x0041, 0x004D, 0x0000}, /* "AM" */ + {0x0050, 0x004D, 0x0000} /* "PM" */ +}; + +static const char16_t gLastResortEras[2][3] = +{ + {0x0042, 0x0043, 0x0000}, /* "BC" */ + {0x0041, 0x0044, 0x0000} /* "AD" */ +}; + +/* Sizes for the last resort string arrays */ +typedef enum LastResortSize { + kMonthNum = 13, + kMonthLen = 3, + + kDayNum = 8, + kDayLen = 2, + + kAmPmNum = 2, + kAmPmLen = 3, + + kQuarterNum = 4, + kQuarterLen = 2, + + kEraNum = 2, + kEraLen = 3, + + kZoneNum = 5, + kZoneLen = 4, + + kGmtHourNum = 4, + kGmtHourLen = 10 +} LastResortSize; + +U_NAMESPACE_BEGIN + +SharedDateFormatSymbols::~SharedDateFormatSymbols() { +} + +template<> U_I18N_API +const SharedDateFormatSymbols * + LocaleCacheKey::createObject( + const void * /*unusedContext*/, UErrorCode &status) const { + char type[256]; + Calendar::getCalendarTypeFromLocale(fLoc, type, UPRV_LENGTHOF(type), status); + if (U_FAILURE(status)) { + return nullptr; + } + SharedDateFormatSymbols *shared + = new SharedDateFormatSymbols(fLoc, type, status); + if (shared == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if (U_FAILURE(status)) { + delete shared; + return nullptr; + } + shared->addRef(); + return shared; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DateFormatSymbols) + +#define kSUPPLEMENTAL "supplementalData" + +/** + * These are the tags we expect to see in normal resource bundle files associated + * with a locale and calendar + */ +static const char gCalendarTag[]="calendar"; +static const char gGregorianTag[]="gregorian"; +static const char gErasTag[]="eras"; +static const char gCyclicNameSetsTag[]="cyclicNameSets"; +static const char gNameSetYearsTag[]="years"; +static const char gNameSetZodiacsTag[]="zodiacs"; +static const char gMonthNamesTag[]="monthNames"; +static const char gMonthPatternsTag[]="monthPatterns"; +static const char gDayNamesTag[]="dayNames"; +static const char gNamesWideTag[]="wide"; +static const char gNamesAbbrTag[]="abbreviated"; +static const char gNamesShortTag[]="short"; +static const char gNamesNarrowTag[]="narrow"; +static const char gNamesAllTag[]="all"; +static const char gNamesFormatTag[]="format"; +static const char gNamesStandaloneTag[]="stand-alone"; +static const char gNamesNumericTag[]="numeric"; +static const char gAmPmMarkersTag[]="AmPmMarkers"; +static const char gAmPmMarkersAbbrTag[]="AmPmMarkersAbbr"; +static const char gAmPmMarkersNarrowTag[]="AmPmMarkersNarrow"; +static const char gQuartersTag[]="quarters"; +static const char gNumberElementsTag[]="NumberElements"; +static const char gSymbolsTag[]="symbols"; +static const char gTimeSeparatorTag[]="timeSeparator"; +static const char gDayPeriodTag[]="dayPeriod"; + +// static const char gZoneStringsTag[]="zoneStrings"; + +// static const char gLocalPatternCharsTag[]="localPatternChars"; + +static const char gContextTransformsTag[]="contextTransforms"; + +/** + * Jitterbug 2974: MSVC has a bug whereby new X[0] behaves badly. + * Work around this. + */ +static inline UnicodeString* newUnicodeStringArray(size_t count) { + return new UnicodeString[count ? count : 1]; +} + +//------------------------------------------------------ + +DateFormatSymbols * U_EXPORT2 +DateFormatSymbols::createForLocale( + const Locale& locale, UErrorCode &status) { + const SharedDateFormatSymbols *shared = nullptr; + UnifiedCache::getByLocale(locale, shared, status); + if (U_FAILURE(status)) { + return nullptr; + } + DateFormatSymbols *result = new DateFormatSymbols(shared->get()); + shared->removeRef(); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return result; +} + +DateFormatSymbols::DateFormatSymbols(const Locale& locale, + UErrorCode& status) + : UObject() +{ + initializeData(locale, nullptr, status); +} + +DateFormatSymbols::DateFormatSymbols(UErrorCode& status) + : UObject() +{ + initializeData(Locale::getDefault(), nullptr, status, true); +} + + +DateFormatSymbols::DateFormatSymbols(const Locale& locale, + const char *type, + UErrorCode& status) + : UObject() +{ + initializeData(locale, type, status); +} + +DateFormatSymbols::DateFormatSymbols(const char *type, UErrorCode& status) + : UObject() +{ + initializeData(Locale::getDefault(), type, status, true); +} + +DateFormatSymbols::DateFormatSymbols(const DateFormatSymbols& other) + : UObject(other) +{ + copyData(other); +} + +void +DateFormatSymbols::assignArray(UnicodeString*& dstArray, + int32_t& dstCount, + const UnicodeString* srcArray, + int32_t srcCount) +{ + // assignArray() is only called by copyData() and initializeData(), which in turn + // implements the copy constructor and the assignment operator. + // All strings in a DateFormatSymbols object are created in one of the following + // three ways that all allow to safely use UnicodeString::fastCopyFrom(): + // - readonly-aliases from resource bundles + // - readonly-aliases or allocated strings from constants + // - safely cloned strings (with owned buffers) from setXYZ() functions + // + // Note that this is true for as long as DateFormatSymbols can be constructed + // only from a locale bundle or set via the cloning API, + // *and* for as long as all the strings are in *private* fields, preventing + // a subclass from creating these strings in an "unsafe" way (with respect to fastCopyFrom()). + if(srcArray == nullptr) { + // Do not attempt to copy bogus input (which will crash). + // Note that this assignArray method already had the potential to return a null dstArray; + // see handling below for "if(dstArray != nullptr)". + dstCount = 0; + dstArray = nullptr; + return; + } + dstCount = srcCount; + dstArray = newUnicodeStringArray(srcCount); + if(dstArray != nullptr) { + int32_t i; + for(i=0; i= 0; i--) { + delete[] fZoneStrings[i]; + } + uprv_free(fZoneStrings); + fZoneStrings = nullptr; + } +} + +/** + * Copy all of the other's data to this. + */ +void +DateFormatSymbols::copyData(const DateFormatSymbols& other) { + UErrorCode status = U_ZERO_ERROR; + U_LOCALE_BASED(locBased, *this); + locBased.setLocaleIDs( + other.getLocale(ULOC_VALID_LOCALE, status), + other.getLocale(ULOC_ACTUAL_LOCALE, status)); + assignArray(fEras, fErasCount, other.fEras, other.fErasCount); + assignArray(fEraNames, fEraNamesCount, other.fEraNames, other.fEraNamesCount); + assignArray(fNarrowEras, fNarrowErasCount, other.fNarrowEras, other.fNarrowErasCount); + assignArray(fMonths, fMonthsCount, other.fMonths, other.fMonthsCount); + assignArray(fShortMonths, fShortMonthsCount, other.fShortMonths, other.fShortMonthsCount); + assignArray(fNarrowMonths, fNarrowMonthsCount, other.fNarrowMonths, other.fNarrowMonthsCount); + assignArray(fStandaloneMonths, fStandaloneMonthsCount, other.fStandaloneMonths, other.fStandaloneMonthsCount); + assignArray(fStandaloneShortMonths, fStandaloneShortMonthsCount, other.fStandaloneShortMonths, other.fStandaloneShortMonthsCount); + assignArray(fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount, other.fStandaloneNarrowMonths, other.fStandaloneNarrowMonthsCount); + assignArray(fWeekdays, fWeekdaysCount, other.fWeekdays, other.fWeekdaysCount); + assignArray(fShortWeekdays, fShortWeekdaysCount, other.fShortWeekdays, other.fShortWeekdaysCount); + assignArray(fShorterWeekdays, fShorterWeekdaysCount, other.fShorterWeekdays, other.fShorterWeekdaysCount); + assignArray(fNarrowWeekdays, fNarrowWeekdaysCount, other.fNarrowWeekdays, other.fNarrowWeekdaysCount); + assignArray(fStandaloneWeekdays, fStandaloneWeekdaysCount, other.fStandaloneWeekdays, other.fStandaloneWeekdaysCount); + assignArray(fStandaloneShortWeekdays, fStandaloneShortWeekdaysCount, other.fStandaloneShortWeekdays, other.fStandaloneShortWeekdaysCount); + assignArray(fStandaloneShorterWeekdays, fStandaloneShorterWeekdaysCount, other.fStandaloneShorterWeekdays, other.fStandaloneShorterWeekdaysCount); + assignArray(fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount, other.fStandaloneNarrowWeekdays, other.fStandaloneNarrowWeekdaysCount); + assignArray(fAmPms, fAmPmsCount, other.fAmPms, other.fAmPmsCount); + assignArray(fNarrowAmPms, fNarrowAmPmsCount, other.fNarrowAmPms, other.fNarrowAmPmsCount ); + fTimeSeparator.fastCopyFrom(other.fTimeSeparator); // fastCopyFrom() - see assignArray comments + assignArray(fQuarters, fQuartersCount, other.fQuarters, other.fQuartersCount); + assignArray(fShortQuarters, fShortQuartersCount, other.fShortQuarters, other.fShortQuartersCount); + assignArray(fNarrowQuarters, fNarrowQuartersCount, other.fNarrowQuarters, other.fNarrowQuartersCount); + assignArray(fStandaloneQuarters, fStandaloneQuartersCount, other.fStandaloneQuarters, other.fStandaloneQuartersCount); + assignArray(fStandaloneShortQuarters, fStandaloneShortQuartersCount, other.fStandaloneShortQuarters, other.fStandaloneShortQuartersCount); + assignArray(fStandaloneNarrowQuarters, fStandaloneNarrowQuartersCount, other.fStandaloneNarrowQuarters, other.fStandaloneNarrowQuartersCount); + assignArray(fWideDayPeriods, fWideDayPeriodsCount, + other.fWideDayPeriods, other.fWideDayPeriodsCount); + assignArray(fNarrowDayPeriods, fNarrowDayPeriodsCount, + other.fNarrowDayPeriods, other.fNarrowDayPeriodsCount); + assignArray(fAbbreviatedDayPeriods, fAbbreviatedDayPeriodsCount, + other.fAbbreviatedDayPeriods, other.fAbbreviatedDayPeriodsCount); + assignArray(fStandaloneWideDayPeriods, fStandaloneWideDayPeriodsCount, + other.fStandaloneWideDayPeriods, other.fStandaloneWideDayPeriodsCount); + assignArray(fStandaloneNarrowDayPeriods, fStandaloneNarrowDayPeriodsCount, + other.fStandaloneNarrowDayPeriods, other.fStandaloneNarrowDayPeriodsCount); + assignArray(fStandaloneAbbreviatedDayPeriods, fStandaloneAbbreviatedDayPeriodsCount, + other.fStandaloneAbbreviatedDayPeriods, other.fStandaloneAbbreviatedDayPeriodsCount); + if (other.fLeapMonthPatterns != nullptr) { + assignArray(fLeapMonthPatterns, fLeapMonthPatternsCount, other.fLeapMonthPatterns, other.fLeapMonthPatternsCount); + } else { + fLeapMonthPatterns = nullptr; + fLeapMonthPatternsCount = 0; + } + if (other.fShortYearNames != nullptr) { + assignArray(fShortYearNames, fShortYearNamesCount, other.fShortYearNames, other.fShortYearNamesCount); + } else { + fShortYearNames = nullptr; + fShortYearNamesCount = 0; + } + if (other.fShortZodiacNames != nullptr) { + assignArray(fShortZodiacNames, fShortZodiacNamesCount, other.fShortZodiacNames, other.fShortZodiacNamesCount); + } else { + fShortZodiacNames = nullptr; + fShortZodiacNamesCount = 0; + } + + if (other.fZoneStrings != nullptr) { + fZoneStringsColCount = other.fZoneStringsColCount; + fZoneStringsRowCount = other.fZoneStringsRowCount; + createZoneStrings((const UnicodeString**)other.fZoneStrings); + + } else { + fZoneStrings = nullptr; + fZoneStringsColCount = 0; + fZoneStringsRowCount = 0; + } + fZSFLocale = other.fZSFLocale; + // Other zone strings data is created on demand + fLocaleZoneStrings = nullptr; + + // fastCopyFrom() - see assignArray comments + fLocalPatternChars.fastCopyFrom(other.fLocalPatternChars); + + uprv_memcpy(fCapitalization, other.fCapitalization, sizeof(fCapitalization)); +} + +/** + * Assignment operator. + */ +DateFormatSymbols& DateFormatSymbols::operator=(const DateFormatSymbols& other) +{ + if (this == &other) { return *this; } // self-assignment: no-op + dispose(); + copyData(other); + + return *this; +} + +DateFormatSymbols::~DateFormatSymbols() +{ + dispose(); +} + +void DateFormatSymbols::dispose() +{ + delete[] fEras; + delete[] fEraNames; + delete[] fNarrowEras; + delete[] fMonths; + delete[] fShortMonths; + delete[] fNarrowMonths; + delete[] fStandaloneMonths; + delete[] fStandaloneShortMonths; + delete[] fStandaloneNarrowMonths; + delete[] fWeekdays; + delete[] fShortWeekdays; + delete[] fShorterWeekdays; + delete[] fNarrowWeekdays; + delete[] fStandaloneWeekdays; + delete[] fStandaloneShortWeekdays; + delete[] fStandaloneShorterWeekdays; + delete[] fStandaloneNarrowWeekdays; + delete[] fAmPms; + delete[] fNarrowAmPms; + delete[] fQuarters; + delete[] fShortQuarters; + delete[] fNarrowQuarters; + delete[] fStandaloneQuarters; + delete[] fStandaloneShortQuarters; + delete[] fStandaloneNarrowQuarters; + delete[] fLeapMonthPatterns; + delete[] fShortYearNames; + delete[] fShortZodiacNames; + delete[] fAbbreviatedDayPeriods; + delete[] fWideDayPeriods; + delete[] fNarrowDayPeriods; + delete[] fStandaloneAbbreviatedDayPeriods; + delete[] fStandaloneWideDayPeriods; + delete[] fStandaloneNarrowDayPeriods; + + disposeZoneStrings(); +} + +void DateFormatSymbols::disposeZoneStrings() +{ + if (fZoneStrings) { + for (int32_t row = 0; row < fZoneStringsRowCount; ++row) { + delete[] fZoneStrings[row]; + } + uprv_free(fZoneStrings); + } + if (fLocaleZoneStrings) { + for (int32_t row = 0; row < fZoneStringsRowCount; ++row) { + delete[] fLocaleZoneStrings[row]; + } + uprv_free(fLocaleZoneStrings); + } + + fZoneStrings = nullptr; + fLocaleZoneStrings = nullptr; + fZoneStringsRowCount = 0; + fZoneStringsColCount = 0; +} + +UBool +DateFormatSymbols::arrayCompare(const UnicodeString* array1, + const UnicodeString* array2, + int32_t count) +{ + if (array1 == array2) return true; + while (count>0) + { + --count; + if (array1[count] != array2[count]) return false; + } + return true; +} + +bool +DateFormatSymbols::operator==(const DateFormatSymbols& other) const +{ + // First do cheap comparisons + if (this == &other) { + return true; + } + if (fErasCount == other.fErasCount && + fEraNamesCount == other.fEraNamesCount && + fNarrowErasCount == other.fNarrowErasCount && + fMonthsCount == other.fMonthsCount && + fShortMonthsCount == other.fShortMonthsCount && + fNarrowMonthsCount == other.fNarrowMonthsCount && + fStandaloneMonthsCount == other.fStandaloneMonthsCount && + fStandaloneShortMonthsCount == other.fStandaloneShortMonthsCount && + fStandaloneNarrowMonthsCount == other.fStandaloneNarrowMonthsCount && + fWeekdaysCount == other.fWeekdaysCount && + fShortWeekdaysCount == other.fShortWeekdaysCount && + fShorterWeekdaysCount == other.fShorterWeekdaysCount && + fNarrowWeekdaysCount == other.fNarrowWeekdaysCount && + fStandaloneWeekdaysCount == other.fStandaloneWeekdaysCount && + fStandaloneShortWeekdaysCount == other.fStandaloneShortWeekdaysCount && + fStandaloneShorterWeekdaysCount == other.fStandaloneShorterWeekdaysCount && + fStandaloneNarrowWeekdaysCount == other.fStandaloneNarrowWeekdaysCount && + fAmPmsCount == other.fAmPmsCount && + fNarrowAmPmsCount == other.fNarrowAmPmsCount && + fQuartersCount == other.fQuartersCount && + fShortQuartersCount == other.fShortQuartersCount && + fNarrowQuartersCount == other.fNarrowQuartersCount && + fStandaloneQuartersCount == other.fStandaloneQuartersCount && + fStandaloneShortQuartersCount == other.fStandaloneShortQuartersCount && + fStandaloneNarrowQuartersCount == other.fStandaloneNarrowQuartersCount && + fLeapMonthPatternsCount == other.fLeapMonthPatternsCount && + fShortYearNamesCount == other.fShortYearNamesCount && + fShortZodiacNamesCount == other.fShortZodiacNamesCount && + fAbbreviatedDayPeriodsCount == other.fAbbreviatedDayPeriodsCount && + fWideDayPeriodsCount == other.fWideDayPeriodsCount && + fNarrowDayPeriodsCount == other.fNarrowDayPeriodsCount && + fStandaloneAbbreviatedDayPeriodsCount == other.fStandaloneAbbreviatedDayPeriodsCount && + fStandaloneWideDayPeriodsCount == other.fStandaloneWideDayPeriodsCount && + fStandaloneNarrowDayPeriodsCount == other.fStandaloneNarrowDayPeriodsCount && + (uprv_memcmp(fCapitalization, other.fCapitalization, sizeof(fCapitalization))==0)) + { + // Now compare the arrays themselves + if (arrayCompare(fEras, other.fEras, fErasCount) && + arrayCompare(fEraNames, other.fEraNames, fEraNamesCount) && + arrayCompare(fNarrowEras, other.fNarrowEras, fNarrowErasCount) && + arrayCompare(fMonths, other.fMonths, fMonthsCount) && + arrayCompare(fShortMonths, other.fShortMonths, fShortMonthsCount) && + arrayCompare(fNarrowMonths, other.fNarrowMonths, fNarrowMonthsCount) && + arrayCompare(fStandaloneMonths, other.fStandaloneMonths, fStandaloneMonthsCount) && + arrayCompare(fStandaloneShortMonths, other.fStandaloneShortMonths, fStandaloneShortMonthsCount) && + arrayCompare(fStandaloneNarrowMonths, other.fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount) && + arrayCompare(fWeekdays, other.fWeekdays, fWeekdaysCount) && + arrayCompare(fShortWeekdays, other.fShortWeekdays, fShortWeekdaysCount) && + arrayCompare(fShorterWeekdays, other.fShorterWeekdays, fShorterWeekdaysCount) && + arrayCompare(fNarrowWeekdays, other.fNarrowWeekdays, fNarrowWeekdaysCount) && + arrayCompare(fStandaloneWeekdays, other.fStandaloneWeekdays, fStandaloneWeekdaysCount) && + arrayCompare(fStandaloneShortWeekdays, other.fStandaloneShortWeekdays, fStandaloneShortWeekdaysCount) && + arrayCompare(fStandaloneShorterWeekdays, other.fStandaloneShorterWeekdays, fStandaloneShorterWeekdaysCount) && + arrayCompare(fStandaloneNarrowWeekdays, other.fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount) && + arrayCompare(fAmPms, other.fAmPms, fAmPmsCount) && + arrayCompare(fNarrowAmPms, other.fNarrowAmPms, fNarrowAmPmsCount) && + fTimeSeparator == other.fTimeSeparator && + arrayCompare(fQuarters, other.fQuarters, fQuartersCount) && + arrayCompare(fShortQuarters, other.fShortQuarters, fShortQuartersCount) && + arrayCompare(fNarrowQuarters, other.fNarrowQuarters, fNarrowQuartersCount) && + arrayCompare(fStandaloneQuarters, other.fStandaloneQuarters, fStandaloneQuartersCount) && + arrayCompare(fStandaloneShortQuarters, other.fStandaloneShortQuarters, fStandaloneShortQuartersCount) && + arrayCompare(fStandaloneNarrowQuarters, other.fStandaloneNarrowQuarters, fStandaloneNarrowQuartersCount) && + arrayCompare(fLeapMonthPatterns, other.fLeapMonthPatterns, fLeapMonthPatternsCount) && + arrayCompare(fShortYearNames, other.fShortYearNames, fShortYearNamesCount) && + arrayCompare(fShortZodiacNames, other.fShortZodiacNames, fShortZodiacNamesCount) && + arrayCompare(fAbbreviatedDayPeriods, other.fAbbreviatedDayPeriods, fAbbreviatedDayPeriodsCount) && + arrayCompare(fWideDayPeriods, other.fWideDayPeriods, fWideDayPeriodsCount) && + arrayCompare(fNarrowDayPeriods, other.fNarrowDayPeriods, fNarrowDayPeriodsCount) && + arrayCompare(fStandaloneAbbreviatedDayPeriods, other.fStandaloneAbbreviatedDayPeriods, + fStandaloneAbbreviatedDayPeriodsCount) && + arrayCompare(fStandaloneWideDayPeriods, other.fStandaloneWideDayPeriods, + fStandaloneWideDayPeriodsCount) && + arrayCompare(fStandaloneNarrowDayPeriods, other.fStandaloneNarrowDayPeriods, + fStandaloneWideDayPeriodsCount)) + { + // Compare the contents of fZoneStrings + if (fZoneStrings == nullptr && other.fZoneStrings == nullptr) { + if (fZSFLocale == other.fZSFLocale) { + return true; + } + } else if (fZoneStrings != nullptr && other.fZoneStrings != nullptr) { + if (fZoneStringsRowCount == other.fZoneStringsRowCount + && fZoneStringsColCount == other.fZoneStringsColCount) { + bool cmpres = true; + for (int32_t i = 0; (i < fZoneStringsRowCount) && cmpres; i++) { + cmpres = arrayCompare(fZoneStrings[i], other.fZoneStrings[i], fZoneStringsColCount); + } + return cmpres; + } + } + return false; + } + } + return false; +} + +//------------------------------------------------------ + +const UnicodeString* +DateFormatSymbols::getEras(int32_t &count) const +{ + count = fErasCount; + return fEras; +} + +const UnicodeString* +DateFormatSymbols::getEraNames(int32_t &count) const +{ + count = fEraNamesCount; + return fEraNames; +} + +const UnicodeString* +DateFormatSymbols::getNarrowEras(int32_t &count) const +{ + count = fNarrowErasCount; + return fNarrowEras; +} + +const UnicodeString* +DateFormatSymbols::getMonths(int32_t &count) const +{ + count = fMonthsCount; + return fMonths; +} + +const UnicodeString* +DateFormatSymbols::getShortMonths(int32_t &count) const +{ + count = fShortMonthsCount; + return fShortMonths; +} + +const UnicodeString* +DateFormatSymbols::getMonths(int32_t &count, DtContextType context, DtWidthType width ) const +{ + UnicodeString *returnValue = nullptr; + + switch (context) { + case FORMAT : + switch(width) { + case WIDE : + count = fMonthsCount; + returnValue = fMonths; + break; + case ABBREVIATED : + case SHORT : // no month data for this, defaults to ABBREVIATED + count = fShortMonthsCount; + returnValue = fShortMonths; + break; + case NARROW : + count = fNarrowMonthsCount; + returnValue = fNarrowMonths; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case STANDALONE : + switch(width) { + case WIDE : + count = fStandaloneMonthsCount; + returnValue = fStandaloneMonths; + break; + case ABBREVIATED : + case SHORT : // no month data for this, defaults to ABBREVIATED + count = fStandaloneShortMonthsCount; + returnValue = fStandaloneShortMonths; + break; + case NARROW : + count = fStandaloneNarrowMonthsCount; + returnValue = fStandaloneNarrowMonths; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } + return returnValue; +} + +const UnicodeString* +DateFormatSymbols::getWeekdays(int32_t &count) const +{ + count = fWeekdaysCount; + return fWeekdays; +} + +const UnicodeString* +DateFormatSymbols::getShortWeekdays(int32_t &count) const +{ + count = fShortWeekdaysCount; + return fShortWeekdays; +} + +const UnicodeString* +DateFormatSymbols::getWeekdays(int32_t &count, DtContextType context, DtWidthType width) const +{ + UnicodeString *returnValue = nullptr; + switch (context) { + case FORMAT : + switch(width) { + case WIDE : + count = fWeekdaysCount; + returnValue = fWeekdays; + break; + case ABBREVIATED : + count = fShortWeekdaysCount; + returnValue = fShortWeekdays; + break; + case SHORT : + count = fShorterWeekdaysCount; + returnValue = fShorterWeekdays; + break; + case NARROW : + count = fNarrowWeekdaysCount; + returnValue = fNarrowWeekdays; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case STANDALONE : + switch(width) { + case WIDE : + count = fStandaloneWeekdaysCount; + returnValue = fStandaloneWeekdays; + break; + case ABBREVIATED : + count = fStandaloneShortWeekdaysCount; + returnValue = fStandaloneShortWeekdays; + break; + case SHORT : + count = fStandaloneShorterWeekdaysCount; + returnValue = fStandaloneShorterWeekdays; + break; + case NARROW : + count = fStandaloneNarrowWeekdaysCount; + returnValue = fStandaloneNarrowWeekdays; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } + return returnValue; +} + +const UnicodeString* +DateFormatSymbols::getQuarters(int32_t &count, DtContextType context, DtWidthType width ) const +{ + UnicodeString *returnValue = nullptr; + + switch (context) { + case FORMAT : + switch(width) { + case WIDE : + count = fQuartersCount; + returnValue = fQuarters; + break; + case ABBREVIATED : + case SHORT : // no quarter data for this, defaults to ABBREVIATED + count = fShortQuartersCount; + returnValue = fShortQuarters; + break; + case NARROW : + count = fNarrowQuartersCount; + returnValue = fNarrowQuarters; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case STANDALONE : + switch(width) { + case WIDE : + count = fStandaloneQuartersCount; + returnValue = fStandaloneQuarters; + break; + case ABBREVIATED : + case SHORT : // no quarter data for this, defaults to ABBREVIATED + count = fStandaloneShortQuartersCount; + returnValue = fStandaloneShortQuarters; + break; + case NARROW : + count = fStandaloneNarrowQuartersCount; + returnValue = fStandaloneNarrowQuarters; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } + return returnValue; +} + +UnicodeString& +DateFormatSymbols::getTimeSeparatorString(UnicodeString& result) const +{ + // fastCopyFrom() - see assignArray comments + return result.fastCopyFrom(fTimeSeparator); +} + +const UnicodeString* +DateFormatSymbols::getAmPmStrings(int32_t &count) const +{ + count = fAmPmsCount; + return fAmPms; +} + +const UnicodeString* +DateFormatSymbols::getLeapMonthPatterns(int32_t &count) const +{ + count = fLeapMonthPatternsCount; + return fLeapMonthPatterns; +} + +const UnicodeString* +DateFormatSymbols::getYearNames(int32_t& count, + DtContextType /*ignored*/, DtWidthType /*ignored*/) const +{ + count = fShortYearNamesCount; + return fShortYearNames; +} + +void +DateFormatSymbols::setYearNames(const UnicodeString* yearNames, int32_t count, + DtContextType context, DtWidthType width) +{ + if (context == FORMAT && width == ABBREVIATED) { + if (fShortYearNames) { + delete[] fShortYearNames; + } + fShortYearNames = newUnicodeStringArray(count); + uprv_arrayCopy(yearNames, fShortYearNames, count); + fShortYearNamesCount = count; + } +} + +const UnicodeString* +DateFormatSymbols::getZodiacNames(int32_t& count, + DtContextType /*ignored*/, DtWidthType /*ignored*/) const +{ + count = fShortZodiacNamesCount; + return fShortZodiacNames; +} + +void +DateFormatSymbols::setZodiacNames(const UnicodeString* zodiacNames, int32_t count, + DtContextType context, DtWidthType width) +{ + if (context == FORMAT && width == ABBREVIATED) { + if (fShortZodiacNames) { + delete[] fShortZodiacNames; + } + fShortZodiacNames = newUnicodeStringArray(count); + uprv_arrayCopy(zodiacNames, fShortZodiacNames, count); + fShortZodiacNamesCount = count; + } +} + +//------------------------------------------------------ + +void +DateFormatSymbols::setEras(const UnicodeString* erasArray, int32_t count) +{ + // delete the old list if we own it + if (fEras) + delete[] fEras; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fEras = newUnicodeStringArray(count); + uprv_arrayCopy(erasArray,fEras, count); + fErasCount = count; +} + +void +DateFormatSymbols::setEraNames(const UnicodeString* eraNamesArray, int32_t count) +{ + // delete the old list if we own it + if (fEraNames) + delete[] fEraNames; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fEraNames = newUnicodeStringArray(count); + uprv_arrayCopy(eraNamesArray,fEraNames, count); + fEraNamesCount = count; +} + +void +DateFormatSymbols::setNarrowEras(const UnicodeString* narrowErasArray, int32_t count) +{ + // delete the old list if we own it + if (fNarrowEras) + delete[] fNarrowEras; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fNarrowEras = newUnicodeStringArray(count); + uprv_arrayCopy(narrowErasArray,fNarrowEras, count); + fNarrowErasCount = count; +} + +void +DateFormatSymbols::setMonths(const UnicodeString* monthsArray, int32_t count) +{ + // delete the old list if we own it + if (fMonths) + delete[] fMonths; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fMonths,count); + fMonthsCount = count; +} + +void +DateFormatSymbols::setShortMonths(const UnicodeString* shortMonthsArray, int32_t count) +{ + // delete the old list if we own it + if (fShortMonths) + delete[] fShortMonths; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fShortMonths = newUnicodeStringArray(count); + uprv_arrayCopy(shortMonthsArray,fShortMonths, count); + fShortMonthsCount = count; +} + +void +DateFormatSymbols::setMonths(const UnicodeString* monthsArray, int32_t count, DtContextType context, DtWidthType width) +{ + // delete the old list if we own it + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + + switch (context) { + case FORMAT : + switch (width) { + case WIDE : + if (fMonths) + delete[] fMonths; + fMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fMonths,count); + fMonthsCount = count; + break; + case ABBREVIATED : + if (fShortMonths) + delete[] fShortMonths; + fShortMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fShortMonths,count); + fShortMonthsCount = count; + break; + case NARROW : + if (fNarrowMonths) + delete[] fNarrowMonths; + fNarrowMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fNarrowMonths,count); + fNarrowMonthsCount = count; + break; + default : + break; + } + break; + case STANDALONE : + switch (width) { + case WIDE : + if (fStandaloneMonths) + delete[] fStandaloneMonths; + fStandaloneMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fStandaloneMonths,count); + fStandaloneMonthsCount = count; + break; + case ABBREVIATED : + if (fStandaloneShortMonths) + delete[] fStandaloneShortMonths; + fStandaloneShortMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fStandaloneShortMonths,count); + fStandaloneShortMonthsCount = count; + break; + case NARROW : + if (fStandaloneNarrowMonths) + delete[] fStandaloneNarrowMonths; + fStandaloneNarrowMonths = newUnicodeStringArray(count); + uprv_arrayCopy( monthsArray,fStandaloneNarrowMonths,count); + fStandaloneNarrowMonthsCount = count; + break; + default : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } +} + +void DateFormatSymbols::setWeekdays(const UnicodeString* weekdaysArray, int32_t count) +{ + // delete the old list if we own it + if (fWeekdays) + delete[] fWeekdays; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray,fWeekdays,count); + fWeekdaysCount = count; +} + +void +DateFormatSymbols::setShortWeekdays(const UnicodeString* shortWeekdaysArray, int32_t count) +{ + // delete the old list if we own it + if (fShortWeekdays) + delete[] fShortWeekdays; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fShortWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(shortWeekdaysArray, fShortWeekdays, count); + fShortWeekdaysCount = count; +} + +void +DateFormatSymbols::setWeekdays(const UnicodeString* weekdaysArray, int32_t count, DtContextType context, DtWidthType width) +{ + // delete the old list if we own it + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + + switch (context) { + case FORMAT : + switch (width) { + case WIDE : + if (fWeekdays) + delete[] fWeekdays; + fWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fWeekdays, count); + fWeekdaysCount = count; + break; + case ABBREVIATED : + if (fShortWeekdays) + delete[] fShortWeekdays; + fShortWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fShortWeekdays, count); + fShortWeekdaysCount = count; + break; + case SHORT : + if (fShorterWeekdays) + delete[] fShorterWeekdays; + fShorterWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fShorterWeekdays, count); + fShorterWeekdaysCount = count; + break; + case NARROW : + if (fNarrowWeekdays) + delete[] fNarrowWeekdays; + fNarrowWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fNarrowWeekdays, count); + fNarrowWeekdaysCount = count; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case STANDALONE : + switch (width) { + case WIDE : + if (fStandaloneWeekdays) + delete[] fStandaloneWeekdays; + fStandaloneWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fStandaloneWeekdays, count); + fStandaloneWeekdaysCount = count; + break; + case ABBREVIATED : + if (fStandaloneShortWeekdays) + delete[] fStandaloneShortWeekdays; + fStandaloneShortWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fStandaloneShortWeekdays, count); + fStandaloneShortWeekdaysCount = count; + break; + case SHORT : + if (fStandaloneShorterWeekdays) + delete[] fStandaloneShorterWeekdays; + fStandaloneShorterWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fStandaloneShorterWeekdays, count); + fStandaloneShorterWeekdaysCount = count; + break; + case NARROW : + if (fStandaloneNarrowWeekdays) + delete[] fStandaloneNarrowWeekdays; + fStandaloneNarrowWeekdays = newUnicodeStringArray(count); + uprv_arrayCopy(weekdaysArray, fStandaloneNarrowWeekdays, count); + fStandaloneNarrowWeekdaysCount = count; + break; + case DT_WIDTH_COUNT : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } +} + +void +DateFormatSymbols::setQuarters(const UnicodeString* quartersArray, int32_t count, DtContextType context, DtWidthType width) +{ + // delete the old list if we own it + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + + switch (context) { + case FORMAT : + switch (width) { + case WIDE : + if (fQuarters) + delete[] fQuarters; + fQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fQuarters,count); + fQuartersCount = count; + break; + case ABBREVIATED : + if (fShortQuarters) + delete[] fShortQuarters; + fShortQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fShortQuarters,count); + fShortQuartersCount = count; + break; + case NARROW : + if (fNarrowQuarters) + delete[] fNarrowQuarters; + fNarrowQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fNarrowQuarters,count); + fNarrowQuartersCount = count; + break; + default : + break; + } + break; + case STANDALONE : + switch (width) { + case WIDE : + if (fStandaloneQuarters) + delete[] fStandaloneQuarters; + fStandaloneQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fStandaloneQuarters,count); + fStandaloneQuartersCount = count; + break; + case ABBREVIATED : + if (fStandaloneShortQuarters) + delete[] fStandaloneShortQuarters; + fStandaloneShortQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fStandaloneShortQuarters,count); + fStandaloneShortQuartersCount = count; + break; + case NARROW : + if (fStandaloneNarrowQuarters) + delete[] fStandaloneNarrowQuarters; + fStandaloneNarrowQuarters = newUnicodeStringArray(count); + uprv_arrayCopy( quartersArray,fStandaloneNarrowQuarters,count); + fStandaloneNarrowQuartersCount = count; + break; + default : + break; + } + break; + case DT_CONTEXT_COUNT : + break; + } +} + +void +DateFormatSymbols::setAmPmStrings(const UnicodeString* amPmsArray, int32_t count) +{ + // delete the old list if we own it + if (fAmPms) delete[] fAmPms; + + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fAmPms = newUnicodeStringArray(count); + uprv_arrayCopy(amPmsArray,fAmPms,count); + fAmPmsCount = count; +} + +void +DateFormatSymbols::setTimeSeparatorString(const UnicodeString& newTimeSeparator) +{ + fTimeSeparator = newTimeSeparator; +} + +const UnicodeString** +DateFormatSymbols::getZoneStrings(int32_t& rowCount, int32_t& columnCount) const +{ + const UnicodeString **result = nullptr; + static UMutex LOCK; + + umtx_lock(&LOCK); + if (fZoneStrings == nullptr) { + if (fLocaleZoneStrings == nullptr) { + ((DateFormatSymbols*)this)->initZoneStringsArray(); + } + result = (const UnicodeString**)fLocaleZoneStrings; + } else { + result = (const UnicodeString**)fZoneStrings; + } + rowCount = fZoneStringsRowCount; + columnCount = fZoneStringsColCount; + umtx_unlock(&LOCK); + + return result; +} + +// For now, we include all zones +#define ZONE_SET UCAL_ZONE_TYPE_ANY + +// This code must be called within a synchronized block +void +DateFormatSymbols::initZoneStringsArray() { + if (fZoneStrings != nullptr || fLocaleZoneStrings != nullptr) { + return; + } + + UErrorCode status = U_ZERO_ERROR; + + StringEnumeration *tzids = nullptr; + UnicodeString ** zarray = nullptr; + TimeZoneNames *tzNames = nullptr; + int32_t rows = 0; + + static const UTimeZoneNameType TYPES[] = { + UTZNM_LONG_STANDARD, UTZNM_SHORT_STANDARD, + UTZNM_LONG_DAYLIGHT, UTZNM_SHORT_DAYLIGHT + }; + static const int32_t NUM_TYPES = 4; + + do { // dummy do-while + + tzids = TimeZone::createTimeZoneIDEnumeration(ZONE_SET, nullptr, nullptr, status); + rows = tzids->count(status); + if (U_FAILURE(status)) { + break; + } + + // Allocate array + int32_t size = rows * sizeof(UnicodeString*); + zarray = (UnicodeString**)uprv_malloc(size); + if (zarray == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + uprv_memset(zarray, 0, size); + + tzNames = TimeZoneNames::createInstance(fZSFLocale, status); + tzNames->loadAllDisplayNames(status); + if (U_FAILURE(status)) { break; } + + const UnicodeString *tzid; + int32_t i = 0; + UDate now = Calendar::getNow(); + UnicodeString tzDispName; + + while ((tzid = tzids->snext(status)) != 0) { + if (U_FAILURE(status)) { + break; + } + + zarray[i] = new UnicodeString[5]; + if (zarray[i] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + + zarray[i][0].setTo(*tzid); + tzNames->getDisplayNames(*tzid, TYPES, NUM_TYPES, now, zarray[i]+1, status); + i++; + } + + } while (false); + + if (U_FAILURE(status)) { + if (zarray) { + for (int32_t i = 0; i < rows; i++) { + if (zarray[i]) { + delete[] zarray[i]; + } + } + uprv_free(zarray); + zarray = nullptr; + } + } + + if (tzNames) { + delete tzNames; + } + if (tzids) { + delete tzids; + } + + fLocaleZoneStrings = zarray; + fZoneStringsRowCount = rows; + fZoneStringsColCount = 1 + NUM_TYPES; +} + +void +DateFormatSymbols::setZoneStrings(const UnicodeString* const *strings, int32_t rowCount, int32_t columnCount) +{ + // since deleting a 2-d array is a pain in the butt, we offload that task to + // a separate function + disposeZoneStrings(); + // we always own the new list, which we create here (we duplicate rather + // than adopting the list passed in) + fZoneStringsRowCount = rowCount; + fZoneStringsColCount = columnCount; + createZoneStrings((const UnicodeString**)strings); +} + +//------------------------------------------------------ + +const char16_t * U_EXPORT2 +DateFormatSymbols::getPatternUChars() +{ + return gPatternChars; +} + +UDateFormatField U_EXPORT2 +DateFormatSymbols::getPatternCharIndex(char16_t c) { + const char16_t *p = u_strchr(gPatternChars, c); + if (p == nullptr) { + return UDAT_FIELD_COUNT; + } else { + return static_cast(p - gPatternChars); + } +} + +static const uint64_t kNumericFieldsAlways = + ((uint64_t)1 << UDAT_YEAR_FIELD) | // y + ((uint64_t)1 << UDAT_DATE_FIELD) | // d + ((uint64_t)1 << UDAT_HOUR_OF_DAY1_FIELD) | // k + ((uint64_t)1 << UDAT_HOUR_OF_DAY0_FIELD) | // H + ((uint64_t)1 << UDAT_MINUTE_FIELD) | // m + ((uint64_t)1 << UDAT_SECOND_FIELD) | // s + ((uint64_t)1 << UDAT_FRACTIONAL_SECOND_FIELD) | // S + ((uint64_t)1 << UDAT_DAY_OF_YEAR_FIELD) | // D + ((uint64_t)1 << UDAT_DAY_OF_WEEK_IN_MONTH_FIELD) | // F + ((uint64_t)1 << UDAT_WEEK_OF_YEAR_FIELD) | // w + ((uint64_t)1 << UDAT_WEEK_OF_MONTH_FIELD) | // W + ((uint64_t)1 << UDAT_HOUR1_FIELD) | // h + ((uint64_t)1 << UDAT_HOUR0_FIELD) | // K + ((uint64_t)1 << UDAT_YEAR_WOY_FIELD) | // Y + ((uint64_t)1 << UDAT_EXTENDED_YEAR_FIELD) | // u + ((uint64_t)1 << UDAT_JULIAN_DAY_FIELD) | // g + ((uint64_t)1 << UDAT_MILLISECONDS_IN_DAY_FIELD) | // A + ((uint64_t)1 << UDAT_RELATED_YEAR_FIELD); // r + +static const uint64_t kNumericFieldsForCount12 = + ((uint64_t)1 << UDAT_MONTH_FIELD) | // M or MM + ((uint64_t)1 << UDAT_DOW_LOCAL_FIELD) | // e or ee + ((uint64_t)1 << UDAT_STANDALONE_DAY_FIELD) | // c or cc + ((uint64_t)1 << UDAT_STANDALONE_MONTH_FIELD) | // L or LL + ((uint64_t)1 << UDAT_QUARTER_FIELD) | // Q or QQ + ((uint64_t)1 << UDAT_STANDALONE_QUARTER_FIELD); // q or qq + +UBool U_EXPORT2 +DateFormatSymbols::isNumericField(UDateFormatField f, int32_t count) { + if (f == UDAT_FIELD_COUNT) { + return false; + } + uint64_t flag = ((uint64_t)1 << f); + return ((kNumericFieldsAlways & flag) != 0 || ((kNumericFieldsForCount12 & flag) != 0 && count < 3)); +} + +UBool U_EXPORT2 +DateFormatSymbols::isNumericPatternChar(char16_t c, int32_t count) { + return isNumericField(getPatternCharIndex(c), count); +} + +//------------------------------------------------------ + +UnicodeString& +DateFormatSymbols::getLocalPatternChars(UnicodeString& result) const +{ + // fastCopyFrom() - see assignArray comments + return result.fastCopyFrom(fLocalPatternChars); +} + +//------------------------------------------------------ + +void +DateFormatSymbols::setLocalPatternChars(const UnicodeString& newLocalPatternChars) +{ + fLocalPatternChars = newLocalPatternChars; +} + +//------------------------------------------------------ + +namespace { + +// Constants declarations +static const char16_t kCalendarAliasPrefixUChar[] = { + SOLIDUS, CAP_L, CAP_O, CAP_C, CAP_A, CAP_L, CAP_E, SOLIDUS, + LOW_C, LOW_A, LOW_L, LOW_E, LOW_N, LOW_D, LOW_A, LOW_R, SOLIDUS +}; +static const char16_t kGregorianTagUChar[] = { + LOW_G, LOW_R, LOW_E, LOW_G, LOW_O, LOW_R, LOW_I, LOW_A, LOW_N +}; +static const char16_t kVariantTagUChar[] = { + PERCENT, LOW_V, LOW_A, LOW_R, LOW_I, LOW_A, LOW_N, LOW_T +}; +static const char16_t kLeapTagUChar[] = { + LOW_L, LOW_E, LOW_A, LOW_P +}; +static const char16_t kCyclicNameSetsTagUChar[] = { + LOW_C, LOW_Y, LOW_C, LOW_L, LOW_I, LOW_C, CAP_N, LOW_A, LOW_M, LOW_E, CAP_S, LOW_E, LOW_T, LOW_S +}; +static const char16_t kYearsTagUChar[] = { + SOLIDUS, LOW_Y, LOW_E, LOW_A, LOW_R, LOW_S +}; +static const char16_t kZodiacsUChar[] = { + SOLIDUS, LOW_Z, LOW_O, LOW_D, LOW_I, LOW_A, LOW_C, LOW_S +}; +static const char16_t kDayPartsTagUChar[] = { + SOLIDUS, LOW_D, LOW_A, LOW_Y, CAP_P, LOW_A, LOW_R, LOW_T, LOW_S +}; +static const char16_t kFormatTagUChar[] = { + SOLIDUS, LOW_F, LOW_O, LOW_R, LOW_M, LOW_A, LOW_T +}; +static const char16_t kAbbrTagUChar[] = { + SOLIDUS, LOW_A, LOW_B, LOW_B, LOW_R, LOW_E, LOW_V, LOW_I, LOW_A, LOW_T, LOW_E, LOW_D +}; + +// ResourceSink to enumerate all calendar resources +struct CalendarDataSink : public ResourceSink { + + // Enum which specifies the type of alias received, or no alias + enum AliasType { + SAME_CALENDAR, + DIFFERENT_CALENDAR, + GREGORIAN, + NONE + }; + + // Data structures to store resources from the current resource bundle + Hashtable arrays; + Hashtable arraySizes; + Hashtable maps; + /** + * Whenever there are aliases, the same object will be added twice to 'map'. + * To avoid double deletion, 'maps' won't take ownership of the objects. Instead, + * 'mapRefs' will own them and will delete them when CalendarDataSink is deleted. + */ + MemoryPool mapRefs; + + // Paths and the aliases they point to + UVector aliasPathPairs; + + // Current and next calendar resource table which should be loaded + UnicodeString currentCalendarType; + UnicodeString nextCalendarType; + + // Resources to visit when enumerating fallback calendars + LocalPointer resourcesToVisit; + + // Alias' relative path populated whenever an alias is read + UnicodeString aliasRelativePath; + + // Initializes CalendarDataSink with default values + CalendarDataSink(UErrorCode& status) + : arrays(false, status), arraySizes(false, status), maps(false, status), + mapRefs(), + aliasPathPairs(uprv_deleteUObject, uhash_compareUnicodeString, status), + currentCalendarType(), nextCalendarType(), + resourcesToVisit(nullptr), aliasRelativePath() { + if (U_FAILURE(status)) { return; } + } + virtual ~CalendarDataSink(); + + // Configure the CalendarSink to visit all the resources + void visitAllResources() { + resourcesToVisit.adoptInstead(nullptr); + } + + // Actions to be done before enumerating + void preEnumerate(const UnicodeString &calendarType) { + currentCalendarType = calendarType; + nextCalendarType.setToBogus(); + aliasPathPairs.removeAllElements(); + } + + virtual void put(const char *key, ResourceValue &value, UBool, UErrorCode &errorCode) override { + if (U_FAILURE(errorCode)) { return; } + U_ASSERT(!currentCalendarType.isEmpty()); + + // Stores the resources to visit on the next calendar. + LocalPointer resourcesToVisitNext(nullptr); + ResourceTable calendarData = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + // Enumerate all resources for this calendar + for (int i = 0; calendarData.getKeyAndValue(i, key, value); i++) { + UnicodeString keyUString(key, -1, US_INV); + + // == Handle aliases == + AliasType aliasType = processAliasFromValue(keyUString, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + if (aliasType == GREGORIAN) { + // Ignore aliases to the gregorian calendar, all of its resources will be loaded anyway. + continue; + + } else if (aliasType == DIFFERENT_CALENDAR) { + // Whenever an alias to the next calendar (except gregorian) is encountered, register the + // calendar type it's pointing to + if (resourcesToVisitNext.isNull()) { + resourcesToVisitNext + .adoptInsteadAndCheckErrorCode(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, errorCode), + errorCode); + if (U_FAILURE(errorCode)) { return; } + } + LocalPointer aliasRelativePathCopy(aliasRelativePath.clone(), errorCode); + resourcesToVisitNext->adoptElement(aliasRelativePathCopy.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + continue; + + } else if (aliasType == SAME_CALENDAR) { + // Register same-calendar alias + if (arrays.get(aliasRelativePath) == nullptr && maps.get(aliasRelativePath) == nullptr) { + LocalPointer aliasRelativePathCopy(aliasRelativePath.clone(), errorCode); + aliasPathPairs.adoptElement(aliasRelativePathCopy.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + LocalPointer keyUStringCopy(keyUString.clone(), errorCode); + aliasPathPairs.adoptElement(keyUStringCopy.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + } + continue; + } + + // Only visit the resources that were referenced by an alias on the previous calendar + // (AmPmMarkersAbbr is an exception). + if (!resourcesToVisit.isNull() && !resourcesToVisit->isEmpty() && !resourcesToVisit->contains(&keyUString) + && uprv_strcmp(key, gAmPmMarkersAbbrTag) != 0) { continue; } + + // == Handle data == + if (uprv_strcmp(key, gAmPmMarkersTag) == 0 + || uprv_strcmp(key, gAmPmMarkersAbbrTag) == 0 + || uprv_strcmp(key, gAmPmMarkersNarrowTag) == 0) { + if (arrays.get(keyUString) == nullptr) { + ResourceArray resourceArray = value.getArray(errorCode); + int32_t arraySize = resourceArray.getSize(); + LocalArray stringArray(new UnicodeString[arraySize], errorCode); + value.getStringArray(stringArray.getAlias(), arraySize, errorCode); + arrays.put(keyUString, stringArray.orphan(), errorCode); + arraySizes.puti(keyUString, arraySize, errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } else if (uprv_strcmp(key, gErasTag) == 0 + || uprv_strcmp(key, gDayNamesTag) == 0 + || uprv_strcmp(key, gMonthNamesTag) == 0 + || uprv_strcmp(key, gQuartersTag) == 0 + || uprv_strcmp(key, gDayPeriodTag) == 0 + || uprv_strcmp(key, gMonthPatternsTag) == 0 + || uprv_strcmp(key, gCyclicNameSetsTag) == 0) { + processResource(keyUString, key, value, errorCode); + } + } + + // Apply same-calendar aliases + UBool modified; + do { + modified = false; + for (int32_t i = 0; i < aliasPathPairs.size();) { + UBool mod = false; + UnicodeString *alias = (UnicodeString*)aliasPathPairs[i]; + UnicodeString *aliasArray; + Hashtable *aliasMap; + if ((aliasArray = (UnicodeString*)arrays.get(*alias)) != nullptr) { + UnicodeString *path = (UnicodeString*)aliasPathPairs[i + 1]; + if (arrays.get(*path) == nullptr) { + // Clone the array + int32_t aliasArraySize = arraySizes.geti(*alias); + LocalArray aliasArrayCopy(new UnicodeString[aliasArraySize], errorCode); + if (U_FAILURE(errorCode)) { return; } + uprv_arrayCopy(aliasArray, aliasArrayCopy.getAlias(), aliasArraySize); + // Put the array on the 'arrays' map + arrays.put(*path, aliasArrayCopy.orphan(), errorCode); + arraySizes.puti(*path, aliasArraySize, errorCode); + } + if (U_FAILURE(errorCode)) { return; } + mod = true; + } else if ((aliasMap = (Hashtable*)maps.get(*alias)) != nullptr) { + UnicodeString *path = (UnicodeString*)aliasPathPairs[i + 1]; + if (maps.get(*path) == nullptr) { + maps.put(*path, aliasMap, errorCode); + } + if (U_FAILURE(errorCode)) { return; } + mod = true; + } + if (mod) { + aliasPathPairs.removeElementAt(i + 1); + aliasPathPairs.removeElementAt(i); + modified = true; + } else { + i += 2; + } + } + } while (modified && !aliasPathPairs.isEmpty()); + + // Set the resources to visit on the next calendar + if (!resourcesToVisitNext.isNull()) { + resourcesToVisit = std::move(resourcesToVisitNext); + } + } + + // Process the nested resource bundle tables + void processResource(UnicodeString &path, const char *key, ResourceValue &value, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) return; + + ResourceTable table = value.getTable(errorCode); + if (U_FAILURE(errorCode)) return; + Hashtable* stringMap = nullptr; + + // Iterate over all the elements of the table and add them to the map + for (int i = 0; table.getKeyAndValue(i, key, value); i++) { + UnicodeString keyUString(key, -1, US_INV); + + // Ignore '%variant' keys + if (keyUString.endsWith(kVariantTagUChar, UPRV_LENGTHOF(kVariantTagUChar))) { + continue; + } + + // == Handle String elements == + if (value.getType() == URES_STRING) { + // We are on a leaf, store the map elements into the stringMap + if (i == 0) { + // mapRefs will keep ownership of 'stringMap': + stringMap = mapRefs.create(false, errorCode); + if (stringMap == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + maps.put(path, stringMap, errorCode); + if (U_FAILURE(errorCode)) { return; } + stringMap->setValueDeleter(uprv_deleteUObject); + } + U_ASSERT(stringMap != nullptr); + int32_t valueStringSize; + const char16_t *valueString = value.getString(valueStringSize, errorCode); + if (U_FAILURE(errorCode)) { return; } + LocalPointer valueUString(new UnicodeString(true, valueString, valueStringSize), errorCode); + stringMap->put(keyUString, valueUString.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + continue; + } + U_ASSERT(stringMap == nullptr); + + // Store the current path's length and append the current key to the path. + int32_t pathLength = path.length(); + path.append(SOLIDUS).append(keyUString); + + // In cyclicNameSets ignore everything but years/format/abbreviated + // and zodiacs/format/abbreviated + if (path.startsWith(kCyclicNameSetsTagUChar, UPRV_LENGTHOF(kCyclicNameSetsTagUChar))) { + UBool skip = true; + int32_t startIndex = UPRV_LENGTHOF(kCyclicNameSetsTagUChar); + int32_t length = 0; + if (startIndex == path.length() + || path.compare(startIndex, (length = UPRV_LENGTHOF(kZodiacsUChar)), kZodiacsUChar, 0, UPRV_LENGTHOF(kZodiacsUChar)) == 0 + || path.compare(startIndex, (length = UPRV_LENGTHOF(kYearsTagUChar)), kYearsTagUChar, 0, UPRV_LENGTHOF(kYearsTagUChar)) == 0 + || path.compare(startIndex, (length = UPRV_LENGTHOF(kDayPartsTagUChar)), kDayPartsTagUChar, 0, UPRV_LENGTHOF(kDayPartsTagUChar)) == 0) { + startIndex += length; + length = 0; + if (startIndex == path.length() + || path.compare(startIndex, (length = UPRV_LENGTHOF(kFormatTagUChar)), kFormatTagUChar, 0, UPRV_LENGTHOF(kFormatTagUChar)) == 0) { + startIndex += length; + length = 0; + if (startIndex == path.length() + || path.compare(startIndex, (length = UPRV_LENGTHOF(kAbbrTagUChar)), kAbbrTagUChar, 0, UPRV_LENGTHOF(kAbbrTagUChar)) == 0) { + skip = false; + } + } + } + if (skip) { + // Drop the latest key on the path and continue + path.retainBetween(0, pathLength); + continue; + } + } + + // == Handle aliases == + if (arrays.get(path) != nullptr || maps.get(path) != nullptr) { + // Drop the latest key on the path and continue + path.retainBetween(0, pathLength); + continue; + } + + AliasType aliasType = processAliasFromValue(path, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + if (aliasType == SAME_CALENDAR) { + // Store the alias path and the current path on aliasPathPairs + LocalPointer aliasRelativePathCopy(aliasRelativePath.clone(), errorCode); + aliasPathPairs.adoptElement(aliasRelativePathCopy.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + LocalPointer pathCopy(path.clone(), errorCode); + aliasPathPairs.adoptElement(pathCopy.orphan(), errorCode); + if (U_FAILURE(errorCode)) { return; } + + // Drop the latest key on the path and continue + path.retainBetween(0, pathLength); + continue; + } + U_ASSERT(aliasType == NONE); + + // == Handle data == + if (value.getType() == URES_ARRAY) { + // We are on a leaf, store the array + ResourceArray rDataArray = value.getArray(errorCode); + int32_t dataArraySize = rDataArray.getSize(); + LocalArray dataArray(new UnicodeString[dataArraySize], errorCode); + value.getStringArray(dataArray.getAlias(), dataArraySize, errorCode); + arrays.put(path, dataArray.orphan(), errorCode); + arraySizes.puti(path, dataArraySize, errorCode); + if (U_FAILURE(errorCode)) { return; } + } else if (value.getType() == URES_TABLE) { + // We are not on a leaf, recursively process the subtable. + processResource(path, key, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + } + + // Drop the latest key on the path + path.retainBetween(0, pathLength); + } + } + + // Populates an AliasIdentifier with the alias information contained on the UResource.Value. + AliasType processAliasFromValue(UnicodeString ¤tRelativePath, ResourceValue &value, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return NONE; } + + if (value.getType() == URES_ALIAS) { + int32_t aliasPathSize; + const char16_t* aliasPathUChar = value.getAliasString(aliasPathSize, errorCode); + if (U_FAILURE(errorCode)) { return NONE; } + UnicodeString aliasPath(aliasPathUChar, aliasPathSize); + const int32_t aliasPrefixLength = UPRV_LENGTHOF(kCalendarAliasPrefixUChar); + if (aliasPath.startsWith(kCalendarAliasPrefixUChar, aliasPrefixLength) + && aliasPath.length() > aliasPrefixLength) { + int32_t typeLimit = aliasPath.indexOf(SOLIDUS, aliasPrefixLength); + if (typeLimit > aliasPrefixLength) { + const UnicodeString aliasCalendarType = + aliasPath.tempSubStringBetween(aliasPrefixLength, typeLimit); + aliasRelativePath.setTo(aliasPath, typeLimit + 1, aliasPath.length()); + + if (currentCalendarType == aliasCalendarType + && currentRelativePath != aliasRelativePath) { + // If we have an alias to the same calendar, the path to the resource must be different + return SAME_CALENDAR; + + } else if (currentCalendarType != aliasCalendarType + && currentRelativePath == aliasRelativePath) { + // If we have an alias to a different calendar, the path to the resource must be the same + if (aliasCalendarType.compare(kGregorianTagUChar, UPRV_LENGTHOF(kGregorianTagUChar)) == 0) { + return GREGORIAN; + } else if (nextCalendarType.isBogus()) { + nextCalendarType = aliasCalendarType; + return DIFFERENT_CALENDAR; + } else if (nextCalendarType == aliasCalendarType) { + return DIFFERENT_CALENDAR; + } + } + } + } + errorCode = U_INTERNAL_PROGRAM_ERROR; + return NONE; + } + return NONE; + } + + // Deleter function to be used by 'arrays' + static void U_CALLCONV deleteUnicodeStringArray(void *uArray) { + delete[] static_cast(uArray); + } +}; +// Virtual destructors have to be defined out of line +CalendarDataSink::~CalendarDataSink() { + arrays.setValueDeleter(deleteUnicodeStringArray); +} +} + +//------------------------------------------------------ + +static void +initField(UnicodeString **field, int32_t& length, const char16_t *data, LastResortSize numStr, LastResortSize strLen, UErrorCode &status) { + if (U_SUCCESS(status)) { + length = numStr; + *field = newUnicodeStringArray((size_t)numStr); + if (*field) { + for(int32_t i = 0; isetTo(true, data+(i*((int32_t)strLen)), -1); + } + } + else { + length = 0; + status = U_MEMORY_ALLOCATION_ERROR; + } + } +} + +static void +initField(UnicodeString **field, int32_t& length, CalendarDataSink &sink, CharString &key, UErrorCode &status) { + if (U_SUCCESS(status)) { + UnicodeString keyUString(key.data(), -1, US_INV); + UnicodeString* array = static_cast(sink.arrays.get(keyUString)); + + if (array != nullptr) { + length = sink.arraySizes.geti(keyUString); + *field = array; + // DateFormatSymbols takes ownership of the array: + sink.arrays.remove(keyUString); + } else { + length = 0; + status = U_MISSING_RESOURCE_ERROR; + } + } +} + +static void +initField(UnicodeString **field, int32_t& length, CalendarDataSink &sink, CharString &key, int32_t arrayOffset, UErrorCode &status) { + if (U_SUCCESS(status)) { + UnicodeString keyUString(key.data(), -1, US_INV); + UnicodeString* array = static_cast(sink.arrays.get(keyUString)); + + if (array != nullptr) { + int32_t arrayLength = sink.arraySizes.geti(keyUString); + length = arrayLength + arrayOffset; + *field = new UnicodeString[length]; + if (*field == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_arrayCopy(array, 0, *field, arrayOffset, arrayLength); + } else { + length = 0; + status = U_MISSING_RESOURCE_ERROR; + } + } +} + +static void +initLeapMonthPattern(UnicodeString *field, int32_t index, CalendarDataSink &sink, CharString &path, UErrorCode &status) { + field[index].remove(); + if (U_SUCCESS(status)) { + UnicodeString pathUString(path.data(), -1, US_INV); + Hashtable *leapMonthTable = static_cast(sink.maps.get(pathUString)); + if (leapMonthTable != nullptr) { + UnicodeString leapLabel(false, kLeapTagUChar, UPRV_LENGTHOF(kLeapTagUChar)); + UnicodeString *leapMonthPattern = static_cast(leapMonthTable->get(leapLabel)); + if (leapMonthPattern != nullptr) { + field[index].fastCopyFrom(*leapMonthPattern); + } else { + field[index].setToBogus(); + } + return; + } + status = U_MISSING_RESOURCE_ERROR; + } +} + +static CharString +&buildResourcePath(CharString &path, const char* segment1, UErrorCode &errorCode) { + return path.clear().append(segment1, -1, errorCode); +} + +static CharString +&buildResourcePath(CharString &path, const char* segment1, const char* segment2, + UErrorCode &errorCode) { + return buildResourcePath(path, segment1, errorCode).append('/', errorCode) + .append(segment2, -1, errorCode); +} + +static CharString +&buildResourcePath(CharString &path, const char* segment1, const char* segment2, + const char* segment3, UErrorCode &errorCode) { + return buildResourcePath(path, segment1, segment2, errorCode).append('/', errorCode) + .append(segment3, -1, errorCode); +} + +static CharString +&buildResourcePath(CharString &path, const char* segment1, const char* segment2, + const char* segment3, const char* segment4, UErrorCode &errorCode) { + return buildResourcePath(path, segment1, segment2, segment3, errorCode).append('/', errorCode) + .append(segment4, -1, errorCode); +} + +typedef struct { + const char * usageTypeName; + DateFormatSymbols::ECapitalizationContextUsageType usageTypeEnumValue; +} ContextUsageTypeNameToEnumValue; + +static const ContextUsageTypeNameToEnumValue contextUsageTypeMap[] = { + // Entries must be sorted by usageTypeName; entry with nullptr name terminates list. + { "day-format-except-narrow", DateFormatSymbols::kCapContextUsageDayFormat }, + { "day-narrow", DateFormatSymbols::kCapContextUsageDayNarrow }, + { "day-standalone-except-narrow", DateFormatSymbols::kCapContextUsageDayStandalone }, + { "era-abbr", DateFormatSymbols::kCapContextUsageEraAbbrev }, + { "era-name", DateFormatSymbols::kCapContextUsageEraWide }, + { "era-narrow", DateFormatSymbols::kCapContextUsageEraNarrow }, + { "metazone-long", DateFormatSymbols::kCapContextUsageMetazoneLong }, + { "metazone-short", DateFormatSymbols::kCapContextUsageMetazoneShort }, + { "month-format-except-narrow", DateFormatSymbols::kCapContextUsageMonthFormat }, + { "month-narrow", DateFormatSymbols::kCapContextUsageMonthNarrow }, + { "month-standalone-except-narrow", DateFormatSymbols::kCapContextUsageMonthStandalone }, + { "zone-long", DateFormatSymbols::kCapContextUsageZoneLong }, + { "zone-short", DateFormatSymbols::kCapContextUsageZoneShort }, + { nullptr, (DateFormatSymbols::ECapitalizationContextUsageType)0 }, +}; + +// Resource keys to look up localized strings for day periods. +// The first one must be midnight and the second must be noon, so that their indices coincide +// with the am/pm field. Formatting and parsing code for day periods relies on this coincidence. +static const char *dayPeriodKeys[] = {"midnight", "noon", + "morning1", "afternoon1", "evening1", "night1", + "morning2", "afternoon2", "evening2", "night2"}; + +UnicodeString* loadDayPeriodStrings(CalendarDataSink &sink, CharString &path, + int32_t &stringCount, UErrorCode &status) { + if (U_FAILURE(status)) { return nullptr; } + + UnicodeString pathUString(path.data(), -1, US_INV); + Hashtable* map = static_cast(sink.maps.get(pathUString)); + + stringCount = UPRV_LENGTHOF(dayPeriodKeys); + UnicodeString *strings = new UnicodeString[stringCount]; + if (strings == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + + if (map != nullptr) { + for (int32_t i = 0; i < stringCount; ++i) { + UnicodeString dayPeriodKey(dayPeriodKeys[i], -1, US_INV); + UnicodeString *dayPeriod = static_cast(map->get(dayPeriodKey)); + if (dayPeriod != nullptr) { + strings[i].fastCopyFrom(*dayPeriod); + } else { + strings[i].setToBogus(); + } + } + } else { + for (int32_t i = 0; i < stringCount; i++) { + strings[i].setToBogus(); + } + } + return strings; +} + + +void +DateFormatSymbols::initializeData(const Locale& locale, const char *type, UErrorCode& status, UBool useLastResortData) +{ + int32_t len = 0; + /* In case something goes wrong, initialize all of the data to nullptr. */ + fEras = nullptr; + fErasCount = 0; + fEraNames = nullptr; + fEraNamesCount = 0; + fNarrowEras = nullptr; + fNarrowErasCount = 0; + fMonths = nullptr; + fMonthsCount=0; + fShortMonths = nullptr; + fShortMonthsCount=0; + fNarrowMonths = nullptr; + fNarrowMonthsCount=0; + fStandaloneMonths = nullptr; + fStandaloneMonthsCount=0; + fStandaloneShortMonths = nullptr; + fStandaloneShortMonthsCount=0; + fStandaloneNarrowMonths = nullptr; + fStandaloneNarrowMonthsCount=0; + fWeekdays = nullptr; + fWeekdaysCount=0; + fShortWeekdays = nullptr; + fShortWeekdaysCount=0; + fShorterWeekdays = nullptr; + fShorterWeekdaysCount=0; + fNarrowWeekdays = nullptr; + fNarrowWeekdaysCount=0; + fStandaloneWeekdays = nullptr; + fStandaloneWeekdaysCount=0; + fStandaloneShortWeekdays = nullptr; + fStandaloneShortWeekdaysCount=0; + fStandaloneShorterWeekdays = nullptr; + fStandaloneShorterWeekdaysCount=0; + fStandaloneNarrowWeekdays = nullptr; + fStandaloneNarrowWeekdaysCount=0; + fAmPms = nullptr; + fAmPmsCount=0; + fNarrowAmPms = nullptr; + fNarrowAmPmsCount=0; + fTimeSeparator.setToBogus(); + fQuarters = nullptr; + fQuartersCount = 0; + fShortQuarters = nullptr; + fShortQuartersCount = 0; + fNarrowQuarters = nullptr; + fNarrowQuartersCount = 0; + fStandaloneQuarters = nullptr; + fStandaloneQuartersCount = 0; + fStandaloneShortQuarters = nullptr; + fStandaloneShortQuartersCount = 0; + fStandaloneNarrowQuarters = nullptr; + fStandaloneNarrowQuartersCount = 0; + fLeapMonthPatterns = nullptr; + fLeapMonthPatternsCount = 0; + fShortYearNames = nullptr; + fShortYearNamesCount = 0; + fShortZodiacNames = nullptr; + fShortZodiacNamesCount = 0; + fZoneStringsRowCount = 0; + fZoneStringsColCount = 0; + fZoneStrings = nullptr; + fLocaleZoneStrings = nullptr; + fAbbreviatedDayPeriods = nullptr; + fAbbreviatedDayPeriodsCount = 0; + fWideDayPeriods = nullptr; + fWideDayPeriodsCount = 0; + fNarrowDayPeriods = nullptr; + fNarrowDayPeriodsCount = 0; + fStandaloneAbbreviatedDayPeriods = nullptr; + fStandaloneAbbreviatedDayPeriodsCount = 0; + fStandaloneWideDayPeriods = nullptr; + fStandaloneWideDayPeriodsCount = 0; + fStandaloneNarrowDayPeriods = nullptr; + fStandaloneNarrowDayPeriodsCount = 0; + uprv_memset(fCapitalization, 0, sizeof(fCapitalization)); + + // We need to preserve the requested locale for + // lazy ZoneStringFormat instantiation. ZoneStringFormat + // is region sensitive, thus, bundle locale bundle's locale + // is not sufficient. + fZSFLocale = locale; + + if (U_FAILURE(status)) return; + + // Create a CalendarDataSink to process this data and the resource bundles + CalendarDataSink calendarSink(status); + UResourceBundle *rb = ures_open(nullptr, locale.getBaseName(), &status); + UResourceBundle *cb = ures_getByKey(rb, gCalendarTag, nullptr, &status); + + if (U_FAILURE(status)) return; + + // Iterate over the resource bundle data following the fallbacks through different calendar types + UnicodeString calendarType((type != nullptr && *type != '\0')? type : gGregorianTag, -1, US_INV); + while (!calendarType.isBogus()) { + CharString calendarTypeBuffer; + calendarTypeBuffer.appendInvariantChars(calendarType, status); + if (U_FAILURE(status)) { return; } + const char *calendarTypeCArray = calendarTypeBuffer.data(); + + // Enumerate this calendar type. If the calendar is not found fallback to gregorian + UErrorCode oldStatus = status; + UResourceBundle *ctb = ures_getByKeyWithFallback(cb, calendarTypeCArray, nullptr, &status); + if (status == U_MISSING_RESOURCE_ERROR) { + ures_close(ctb); + if (uprv_strcmp(calendarTypeCArray, gGregorianTag) != 0) { + calendarType.setTo(false, kGregorianTagUChar, UPRV_LENGTHOF(kGregorianTagUChar)); + calendarSink.visitAllResources(); + status = oldStatus; + continue; + } + return; + } + + calendarSink.preEnumerate(calendarType); + ures_getAllItemsWithFallback(ctb, "", calendarSink, status); + ures_close(ctb); + if (U_FAILURE(status)) break; + + // Stop loading when gregorian was loaded + if (uprv_strcmp(calendarTypeCArray, gGregorianTag) == 0) { + break; + } + + // Get the next calendar type to process from the sink + calendarType = calendarSink.nextCalendarType; + + // Gregorian is always the last fallback + if (calendarType.isBogus()) { + calendarType.setTo(false, kGregorianTagUChar, UPRV_LENGTHOF(kGregorianTagUChar)); + calendarSink.visitAllResources(); + } + } + + // CharString object to build paths + CharString path; + + // Load Leap Month Patterns + UErrorCode tempStatus = status; + fLeapMonthPatterns = newUnicodeStringArray(kMonthPatternsCount); + if (fLeapMonthPatterns) { + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternFormatWide, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesFormatTag, gNamesWideTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternFormatAbbrev, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesFormatTag, gNamesAbbrTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternFormatNarrow, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesFormatTag, gNamesNarrowTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternStandaloneWide, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesStandaloneTag, gNamesWideTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternStandaloneAbbrev, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesStandaloneTag, gNamesAbbrTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternStandaloneNarrow, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesStandaloneTag, gNamesNarrowTag, tempStatus), tempStatus); + initLeapMonthPattern(fLeapMonthPatterns, kLeapMonthPatternNumeric, calendarSink, + buildResourcePath(path, gMonthPatternsTag, gNamesNumericTag, gNamesAllTag, tempStatus), tempStatus); + if (U_SUCCESS(tempStatus)) { + // Hack to fix bad C inheritance for dangi monthPatterns (OK in J); this should be handled by aliases in root, but isn't. + // The ordering of the following statements is important. + if (fLeapMonthPatterns[kLeapMonthPatternFormatAbbrev].isEmpty()) { + fLeapMonthPatterns[kLeapMonthPatternFormatAbbrev].setTo(fLeapMonthPatterns[kLeapMonthPatternFormatWide]); + } + if (fLeapMonthPatterns[kLeapMonthPatternFormatNarrow].isEmpty()) { + fLeapMonthPatterns[kLeapMonthPatternFormatNarrow].setTo(fLeapMonthPatterns[kLeapMonthPatternStandaloneNarrow]); + } + if (fLeapMonthPatterns[kLeapMonthPatternStandaloneWide].isEmpty()) { + fLeapMonthPatterns[kLeapMonthPatternStandaloneWide].setTo(fLeapMonthPatterns[kLeapMonthPatternFormatWide]); + } + if (fLeapMonthPatterns[kLeapMonthPatternStandaloneAbbrev].isEmpty()) { + fLeapMonthPatterns[kLeapMonthPatternStandaloneAbbrev].setTo(fLeapMonthPatterns[kLeapMonthPatternFormatAbbrev]); + } + // end of hack + fLeapMonthPatternsCount = kMonthPatternsCount; + } else { + delete[] fLeapMonthPatterns; + fLeapMonthPatterns = nullptr; + } + } + + // Load cyclic names sets + tempStatus = status; + initField(&fShortYearNames, fShortYearNamesCount, calendarSink, + buildResourcePath(path, gCyclicNameSetsTag, gNameSetYearsTag, gNamesFormatTag, gNamesAbbrTag, tempStatus), tempStatus); + initField(&fShortZodiacNames, fShortZodiacNamesCount, calendarSink, + buildResourcePath(path, gCyclicNameSetsTag, gNameSetZodiacsTag, gNamesFormatTag, gNamesAbbrTag, tempStatus), tempStatus); + + // Load context transforms and capitalization + tempStatus = U_ZERO_ERROR; + UResourceBundle *localeBundle = ures_open(nullptr, locale.getName(), &tempStatus); + if (U_SUCCESS(tempStatus)) { + UResourceBundle *contextTransforms = ures_getByKeyWithFallback(localeBundle, gContextTransformsTag, nullptr, &tempStatus); + if (U_SUCCESS(tempStatus)) { + UResourceBundle *contextTransformUsage; + while ( (contextTransformUsage = ures_getNextResource(contextTransforms, nullptr, &tempStatus)) != nullptr ) { + const int32_t * intVector = ures_getIntVector(contextTransformUsage, &len, &status); + if (U_SUCCESS(tempStatus) && intVector != nullptr && len >= 2) { + const char* usageType = ures_getKey(contextTransformUsage); + if (usageType != nullptr) { + const ContextUsageTypeNameToEnumValue * typeMapPtr = contextUsageTypeMap; + int32_t compResult = 0; + // linear search; list is short and we cannot be sure that bsearch is available + while ( typeMapPtr->usageTypeName != nullptr && (compResult = uprv_strcmp(usageType, typeMapPtr->usageTypeName)) > 0 ) { + ++typeMapPtr; + } + if (typeMapPtr->usageTypeName != nullptr && compResult == 0) { + fCapitalization[typeMapPtr->usageTypeEnumValue][0] = static_cast(intVector[0]); + fCapitalization[typeMapPtr->usageTypeEnumValue][1] = static_cast(intVector[1]); + } + } + } + tempStatus = U_ZERO_ERROR; + ures_close(contextTransformUsage); + } + ures_close(contextTransforms); + } + + tempStatus = U_ZERO_ERROR; + const LocalPointer numberingSystem( + NumberingSystem::createInstance(locale, tempStatus), tempStatus); + if (U_SUCCESS(tempStatus)) { + // These functions all fail gracefully if passed nullptr pointers and + // do nothing unless U_SUCCESS(tempStatus), so it's only necessary + // to check for errors once after all calls are made. + const LocalUResourceBundlePointer numberElementsData(ures_getByKeyWithFallback( + localeBundle, gNumberElementsTag, nullptr, &tempStatus)); + const LocalUResourceBundlePointer nsNameData(ures_getByKeyWithFallback( + numberElementsData.getAlias(), numberingSystem->getName(), nullptr, &tempStatus)); + const LocalUResourceBundlePointer symbolsData(ures_getByKeyWithFallback( + nsNameData.getAlias(), gSymbolsTag, nullptr, &tempStatus)); + fTimeSeparator = ures_getUnicodeStringByKey( + symbolsData.getAlias(), gTimeSeparatorTag, &tempStatus); + if (U_FAILURE(tempStatus)) { + fTimeSeparator.setToBogus(); + } + } + + ures_close(localeBundle); + } + + if (fTimeSeparator.isBogus()) { + fTimeSeparator.setTo(DateFormatSymbols::DEFAULT_TIME_SEPARATOR); + } + + // Load day periods + fAbbreviatedDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesFormatTag, gNamesAbbrTag, status), + fAbbreviatedDayPeriodsCount, status); + + fWideDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesFormatTag, gNamesWideTag, status), + fWideDayPeriodsCount, status); + fNarrowDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesFormatTag, gNamesNarrowTag, status), + fNarrowDayPeriodsCount, status); + + fStandaloneAbbreviatedDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesStandaloneTag, gNamesAbbrTag, status), + fStandaloneAbbreviatedDayPeriodsCount, status); + + fStandaloneWideDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesStandaloneTag, gNamesWideTag, status), + fStandaloneWideDayPeriodsCount, status); + fStandaloneNarrowDayPeriods = loadDayPeriodStrings(calendarSink, + buildResourcePath(path, gDayPeriodTag, gNamesStandaloneTag, gNamesNarrowTag, status), + fStandaloneNarrowDayPeriodsCount, status); + + // Fill in for missing/bogus items (dayPeriods are a map so single items might be missing) + if (U_SUCCESS(status)) { + for (int32_t dpidx = 0; dpidx < fAbbreviatedDayPeriodsCount; ++dpidx) { + if (dpidx < fWideDayPeriodsCount && fWideDayPeriods != nullptr && fWideDayPeriods[dpidx].isBogus()) { + fWideDayPeriods[dpidx].fastCopyFrom(fAbbreviatedDayPeriods[dpidx]); + } + if (dpidx < fNarrowDayPeriodsCount && fNarrowDayPeriods != nullptr && fNarrowDayPeriods[dpidx].isBogus()) { + fNarrowDayPeriods[dpidx].fastCopyFrom(fAbbreviatedDayPeriods[dpidx]); + } + if (dpidx < fStandaloneAbbreviatedDayPeriodsCount && fStandaloneAbbreviatedDayPeriods != nullptr && fStandaloneAbbreviatedDayPeriods[dpidx].isBogus()) { + fStandaloneAbbreviatedDayPeriods[dpidx].fastCopyFrom(fAbbreviatedDayPeriods[dpidx]); + } + if (dpidx < fStandaloneWideDayPeriodsCount && fStandaloneWideDayPeriods != nullptr && fStandaloneWideDayPeriods[dpidx].isBogus()) { + fStandaloneWideDayPeriods[dpidx].fastCopyFrom(fStandaloneAbbreviatedDayPeriods[dpidx]); + } + if (dpidx < fStandaloneNarrowDayPeriodsCount && fStandaloneNarrowDayPeriods != nullptr && fStandaloneNarrowDayPeriods[dpidx].isBogus()) { + fStandaloneNarrowDayPeriods[dpidx].fastCopyFrom(fStandaloneAbbreviatedDayPeriods[dpidx]); + } + } + } + + U_LOCALE_BASED(locBased, *this); + // if we make it to here, the resource data is cool, and we can get everything out + // of it that we need except for the time-zone and localized-pattern data, which + // are stored in a separate file + locBased.setLocaleIDs(ures_getLocaleByType(cb, ULOC_VALID_LOCALE, &status), + ures_getLocaleByType(cb, ULOC_ACTUAL_LOCALE, &status)); + + // Load eras + initField(&fEras, fErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesAbbrTag, status), status); + UErrorCode oldStatus = status; + initField(&fEraNames, fEraNamesCount, calendarSink, buildResourcePath(path, gErasTag, gNamesWideTag, status), status); + if (status == U_MISSING_RESOURCE_ERROR) { // Workaround because eras/wide was omitted from CLDR 1.3 + status = oldStatus; + assignArray(fEraNames, fEraNamesCount, fEras, fErasCount); + } + // current ICU4J falls back to abbreviated if narrow eras are missing, so we will too + oldStatus = status; + initField(&fNarrowEras, fNarrowErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesNarrowTag, status), status); + if (status == U_MISSING_RESOURCE_ERROR) { // Workaround because eras/wide was omitted from CLDR 1.3 + status = oldStatus; + assignArray(fNarrowEras, fNarrowErasCount, fEras, fErasCount); + } + + // Load month names + initField(&fMonths, fMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesFormatTag, gNamesWideTag, status), status); + initField(&fShortMonths, fShortMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesFormatTag, gNamesAbbrTag, status), status); + initField(&fStandaloneMonths, fStandaloneMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesStandaloneTag, gNamesWideTag, status), status); + if (status == U_MISSING_RESOURCE_ERROR) { /* If standalone/wide not available, use format/wide */ + status = U_ZERO_ERROR; + assignArray(fStandaloneMonths, fStandaloneMonthsCount, fMonths, fMonthsCount); + } + initField(&fStandaloneShortMonths, fStandaloneShortMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesStandaloneTag, gNamesAbbrTag, status), status); + if (status == U_MISSING_RESOURCE_ERROR) { /* If standalone/abbreviated not available, use format/abbreviated */ + status = U_ZERO_ERROR; + assignArray(fStandaloneShortMonths, fStandaloneShortMonthsCount, fShortMonths, fShortMonthsCount); + } + + UErrorCode narrowMonthsEC = status; + UErrorCode standaloneNarrowMonthsEC = status; + initField(&fNarrowMonths, fNarrowMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesFormatTag, gNamesNarrowTag, narrowMonthsEC), narrowMonthsEC); + initField(&fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount, calendarSink, + buildResourcePath(path, gMonthNamesTag, gNamesStandaloneTag, gNamesNarrowTag, narrowMonthsEC), standaloneNarrowMonthsEC); + if (narrowMonthsEC == U_MISSING_RESOURCE_ERROR && standaloneNarrowMonthsEC != U_MISSING_RESOURCE_ERROR) { + // If format/narrow not available, use standalone/narrow + assignArray(fNarrowMonths, fNarrowMonthsCount, fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount); + } else if (narrowMonthsEC != U_MISSING_RESOURCE_ERROR && standaloneNarrowMonthsEC == U_MISSING_RESOURCE_ERROR) { + // If standalone/narrow not available, use format/narrow + assignArray(fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount, fNarrowMonths, fNarrowMonthsCount); + } else if (narrowMonthsEC == U_MISSING_RESOURCE_ERROR && standaloneNarrowMonthsEC == U_MISSING_RESOURCE_ERROR) { + // If neither is available, use format/abbreviated + assignArray(fNarrowMonths, fNarrowMonthsCount, fShortMonths, fShortMonthsCount); + assignArray(fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount, fShortMonths, fShortMonthsCount); + } + + // Load AM/PM markers; if wide or narrow not available, use short + UErrorCode ampmStatus = U_ZERO_ERROR; + initField(&fAmPms, fAmPmsCount, calendarSink, + buildResourcePath(path, gAmPmMarkersTag, ampmStatus), ampmStatus); + if (U_FAILURE(ampmStatus)) { + initField(&fAmPms, fAmPmsCount, calendarSink, + buildResourcePath(path, gAmPmMarkersAbbrTag, status), status); + } + ampmStatus = U_ZERO_ERROR; + initField(&fNarrowAmPms, fNarrowAmPmsCount, calendarSink, + buildResourcePath(path, gAmPmMarkersNarrowTag, ampmStatus), ampmStatus); + if (U_FAILURE(ampmStatus)) { + initField(&fNarrowAmPms, fNarrowAmPmsCount, calendarSink, + buildResourcePath(path, gAmPmMarkersAbbrTag, status), status); + } + if(status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fNarrowAmPms, fNarrowAmPmsCount, fAmPms, fAmPmsCount); + } + + // Load quarters + initField(&fQuarters, fQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesFormatTag, gNamesWideTag, status), status); + initField(&fShortQuarters, fShortQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesFormatTag, gNamesAbbrTag, status), status); + if(status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fShortQuarters, fShortQuartersCount, fQuarters, fQuartersCount); + } + + initField(&fStandaloneQuarters, fStandaloneQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesStandaloneTag, gNamesWideTag, status), status); + if(status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fStandaloneQuarters, fStandaloneQuartersCount, fQuarters, fQuartersCount); + } + initField(&fStandaloneShortQuarters, fStandaloneShortQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesStandaloneTag, gNamesAbbrTag, status), status); + if(status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fStandaloneShortQuarters, fStandaloneShortQuartersCount, fShortQuarters, fShortQuartersCount); + } + + // unlike the fields above, narrow format quarters fall back on narrow standalone quarters + initField(&fStandaloneNarrowQuarters, fStandaloneNarrowQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesStandaloneTag, gNamesNarrowTag, status), status); + initField(&fNarrowQuarters, fNarrowQuartersCount, calendarSink, + buildResourcePath(path, gQuartersTag, gNamesFormatTag, gNamesNarrowTag, status), status); + if(status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fNarrowQuarters, fNarrowQuartersCount, fStandaloneNarrowQuarters, fStandaloneNarrowQuartersCount); + } + + // ICU 3.8 or later version no longer uses localized date-time pattern characters by default (ticket#5597) + /* + // fastCopyFrom()/setTo() - see assignArray comments + resStr = ures_getStringByKey(fResourceBundle, gLocalPatternCharsTag, &len, &status); + fLocalPatternChars.setTo(true, resStr, len); + // If the locale data does not include new pattern chars, use the defaults + // TODO: Consider making this an error, since this may add conflicting characters. + if (len < PATTERN_CHARS_LEN) { + fLocalPatternChars.append(UnicodeString(true, &gPatternChars[len], PATTERN_CHARS_LEN-len)); + } + */ + fLocalPatternChars.setTo(true, gPatternChars, PATTERN_CHARS_LEN); + + // Format wide weekdays -> fWeekdays + // {sfb} fixed to handle 1-based weekdays + initField(&fWeekdays, fWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesFormatTag, gNamesWideTag, status), 1, status); + + // Format abbreviated weekdays -> fShortWeekdays + initField(&fShortWeekdays, fShortWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesFormatTag, gNamesAbbrTag, status), 1, status); + + // Format short weekdays -> fShorterWeekdays (fall back to abbreviated) + initField(&fShorterWeekdays, fShorterWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesFormatTag, gNamesShortTag, status), 1, status); + if (status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + assignArray(fShorterWeekdays, fShorterWeekdaysCount, fShortWeekdays, fShortWeekdaysCount); + } + + // Stand-alone wide weekdays -> fStandaloneWeekdays + initField(&fStandaloneWeekdays, fStandaloneWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesStandaloneTag, gNamesWideTag, status), 1, status); + if (status == U_MISSING_RESOURCE_ERROR) { /* If standalone/wide is not available, use format/wide */ + status = U_ZERO_ERROR; + assignArray(fStandaloneWeekdays, fStandaloneWeekdaysCount, fWeekdays, fWeekdaysCount); + } + + // Stand-alone abbreviated weekdays -> fStandaloneShortWeekdays + initField(&fStandaloneShortWeekdays, fStandaloneShortWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesStandaloneTag, gNamesAbbrTag, status), 1, status); + if (status == U_MISSING_RESOURCE_ERROR) { /* If standalone/abbreviated is not available, use format/abbreviated */ + status = U_ZERO_ERROR; + assignArray(fStandaloneShortWeekdays, fStandaloneShortWeekdaysCount, fShortWeekdays, fShortWeekdaysCount); + } + + // Stand-alone short weekdays -> fStandaloneShorterWeekdays (fall back to format abbreviated) + initField(&fStandaloneShorterWeekdays, fStandaloneShorterWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesStandaloneTag, gNamesShortTag, status), 1, status); + if (status == U_MISSING_RESOURCE_ERROR) { /* If standalone/short is not available, use format/short */ + status = U_ZERO_ERROR; + assignArray(fStandaloneShorterWeekdays, fStandaloneShorterWeekdaysCount, fShorterWeekdays, fShorterWeekdaysCount); + } + + // Format narrow weekdays -> fNarrowWeekdays + UErrorCode narrowWeeksEC = status; + initField(&fNarrowWeekdays, fNarrowWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesFormatTag, gNamesNarrowTag, status), 1, narrowWeeksEC); + // Stand-alone narrow weekdays -> fStandaloneNarrowWeekdays + UErrorCode standaloneNarrowWeeksEC = status; + initField(&fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount, calendarSink, + buildResourcePath(path, gDayNamesTag, gNamesStandaloneTag, gNamesNarrowTag, status), 1, standaloneNarrowWeeksEC); + + if (narrowWeeksEC == U_MISSING_RESOURCE_ERROR && standaloneNarrowWeeksEC != U_MISSING_RESOURCE_ERROR) { + // If format/narrow not available, use standalone/narrow + assignArray(fNarrowWeekdays, fNarrowWeekdaysCount, fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount); + } else if (narrowWeeksEC != U_MISSING_RESOURCE_ERROR && standaloneNarrowWeeksEC == U_MISSING_RESOURCE_ERROR) { + // If standalone/narrow not available, use format/narrow + assignArray(fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount, fNarrowWeekdays, fNarrowWeekdaysCount); + } else if (narrowWeeksEC == U_MISSING_RESOURCE_ERROR && standaloneNarrowWeeksEC == U_MISSING_RESOURCE_ERROR ) { + // If neither is available, use format/abbreviated + assignArray(fNarrowWeekdays, fNarrowWeekdaysCount, fShortWeekdays, fShortWeekdaysCount); + assignArray(fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount, fShortWeekdays, fShortWeekdaysCount); + } + + // Last resort fallback in case previous data wasn't loaded + if (U_FAILURE(status)) + { + if (useLastResortData) + { + // Handle the case in which there is no resource data present. + // We don't have to generate usable patterns in this situation; + // we just need to produce something that will be semi-intelligible + // in most locales. + + status = U_USING_FALLBACK_WARNING; + //TODO(fabalbon): make sure we are storing las resort data for all fields in here. + initField(&fEras, fErasCount, (const char16_t *)gLastResortEras, kEraNum, kEraLen, status); + initField(&fEraNames, fEraNamesCount, (const char16_t *)gLastResortEras, kEraNum, kEraLen, status); + initField(&fNarrowEras, fNarrowErasCount, (const char16_t *)gLastResortEras, kEraNum, kEraLen, status); + initField(&fMonths, fMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fShortMonths, fShortMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fNarrowMonths, fNarrowMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fStandaloneMonths, fStandaloneMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fStandaloneShortMonths, fStandaloneShortMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fStandaloneNarrowMonths, fStandaloneNarrowMonthsCount, (const char16_t *)gLastResortMonthNames, kMonthNum, kMonthLen, status); + initField(&fWeekdays, fWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fShortWeekdays, fShortWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fShorterWeekdays, fShorterWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fNarrowWeekdays, fNarrowWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fStandaloneWeekdays, fStandaloneWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fStandaloneShortWeekdays, fStandaloneShortWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fStandaloneShorterWeekdays, fStandaloneShorterWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fStandaloneNarrowWeekdays, fStandaloneNarrowWeekdaysCount, (const char16_t *)gLastResortDayNames, kDayNum, kDayLen, status); + initField(&fAmPms, fAmPmsCount, (const char16_t *)gLastResortAmPmMarkers, kAmPmNum, kAmPmLen, status); + initField(&fNarrowAmPms, fNarrowAmPmsCount, (const char16_t *)gLastResortAmPmMarkers, kAmPmNum, kAmPmLen, status); + initField(&fQuarters, fQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + initField(&fShortQuarters, fShortQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + initField(&fNarrowQuarters, fNarrowQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + initField(&fStandaloneQuarters, fStandaloneQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + initField(&fStandaloneShortQuarters, fStandaloneShortQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + initField(&fStandaloneNarrowQuarters, fStandaloneNarrowQuartersCount, (const char16_t *)gLastResortQuarters, kQuarterNum, kQuarterLen, status); + fLocalPatternChars.setTo(true, gPatternChars, PATTERN_CHARS_LEN); + } + } + + // Close resources + ures_close(cb); + ures_close(rb); +} + +Locale +DateFormatSymbols::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocale(type, status); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/dtitv_impl.h b/intl/icu/source/i18n/dtitv_impl.h new file mode 100644 index 0000000000..2cc7764817 --- /dev/null +++ b/intl/icu/source/i18n/dtitv_impl.h @@ -0,0 +1,99 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DTITV_IMPL.H +* +******************************************************************************* +*/ + + +#ifndef DTITV_IMPL_H__ +#define DTITV_IMPL_H__ + +/** + * \file + * \brief C++ API: Defines macros for interval format implementation + */ + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/unistr.h" + + +#define QUOTE ((char16_t)0x0027) +#define LOW_LINE ((char16_t)0x005F) +#define COLON ((char16_t)0x003A) +#define LEFT_CURLY_BRACKET ((char16_t)0x007B) +#define RIGHT_CURLY_BRACKET ((char16_t)0x007D) +#define SPACE ((char16_t)0x0020) +#define EN_DASH ((char16_t)0x2013) +#define SOLIDUS ((char16_t)0x002F) + +#define DIGIT_ZERO ((char16_t)0x0030) +#define DIGIT_ONE ((char16_t)0x0031) + +#define LOW_A ((char16_t)0x0061) +#define LOW_B ((char16_t)0x0062) +#define LOW_C ((char16_t)0x0063) +#define LOW_D ((char16_t)0x0064) +#define LOW_E ((char16_t)0x0065) +#define LOW_F ((char16_t)0x0066) +#define LOW_G ((char16_t)0x0067) +#define LOW_H ((char16_t)0x0068) +#define LOW_I ((char16_t)0x0069) +#define LOW_J ((char16_t)0x006a) +#define LOW_K ((char16_t)0x006B) +#define LOW_L ((char16_t)0x006C) +#define LOW_M ((char16_t)0x006D) +#define LOW_N ((char16_t)0x006E) +#define LOW_O ((char16_t)0x006F) +#define LOW_P ((char16_t)0x0070) +#define LOW_Q ((char16_t)0x0071) +#define LOW_R ((char16_t)0x0072) +#define LOW_S ((char16_t)0x0073) +#define LOW_T ((char16_t)0x0074) +#define LOW_U ((char16_t)0x0075) +#define LOW_V ((char16_t)0x0076) +#define LOW_W ((char16_t)0x0077) +#define LOW_Y ((char16_t)0x0079) +#define LOW_Z ((char16_t)0x007A) + +#define CAP_A ((char16_t)0x0041) +#define CAP_B ((char16_t)0x0042) +#define CAP_C ((char16_t)0x0043) +#define CAP_D ((char16_t)0x0044) +#define CAP_E ((char16_t)0x0045) +#define CAP_F ((char16_t)0x0046) +#define CAP_G ((char16_t)0x0047) +#define CAP_J ((char16_t)0x004A) +#define CAP_H ((char16_t)0x0048) +#define CAP_K ((char16_t)0x004B) +#define CAP_L ((char16_t)0x004C) +#define CAP_M ((char16_t)0x004D) +#define CAP_O ((char16_t)0x004F) +#define CAP_Q ((char16_t)0x0051) +#define CAP_S ((char16_t)0x0053) +#define CAP_T ((char16_t)0x0054) +#define CAP_U ((char16_t)0x0055) +#define CAP_V ((char16_t)0x0056) +#define CAP_W ((char16_t)0x0057) +#define CAP_Y ((char16_t)0x0059) +#define CAP_Z ((char16_t)0x005A) + +//#define MINIMUM_SUPPORTED_CALENDAR_FIELD UCAL_MINUTE + +#define MAX_E_COUNT 5 +#define MAX_M_COUNT 5 +//#define MAX_INTERVAL_INDEX 4 +#define MAX_POSITIVE_INT 56632 + + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif +//eof diff --git a/intl/icu/source/i18n/dtitvfmt.cpp b/intl/icu/source/i18n/dtitvfmt.cpp new file mode 100644 index 0000000000..6087014668 --- /dev/null +++ b/intl/icu/source/i18n/dtitvfmt.cpp @@ -0,0 +1,1909 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/******************************************************************************* +* Copyright (C) 2008-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DTITVFMT.CPP +* +******************************************************************************* +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/dtitvfmt.h" + +#if !UCONFIG_NO_FORMATTING + +//TODO: put in compilation +//#define DTITVFMT_DEBUG 1 + +#include "unicode/calendar.h" +#include "unicode/dtptngen.h" +#include "unicode/dtitvinf.h" +#include "unicode/simpleformatter.h" +#include "unicode/udisplaycontext.h" +#include "cmemory.h" +#include "cstring.h" +#include "dtitv_impl.h" +#include "mutex.h" +#include "uresimp.h" +#include "formattedval_impl.h" + +#ifdef DTITVFMT_DEBUG +#include +#endif + +U_NAMESPACE_BEGIN + + + +#ifdef DTITVFMT_DEBUG +#define PRINTMESG(msg) { std::cout << "(" << __FILE__ << ":" << __LINE__ << ") " << msg << "\n"; } +#endif + + +static const char16_t gDateFormatSkeleton[][11] = { +//yMMMMEEEEd +{LOW_Y, CAP_M, CAP_M, CAP_M, CAP_M, CAP_E, CAP_E, CAP_E, CAP_E, LOW_D, 0}, +//yMMMMd +{LOW_Y, CAP_M, CAP_M, CAP_M, CAP_M, LOW_D, 0}, +//yMMMd +{LOW_Y, CAP_M, CAP_M, CAP_M, LOW_D, 0}, +//yMd +{LOW_Y, CAP_M, LOW_D, 0} }; + + +static const char gCalendarTag[] = "calendar"; +static const char gGregorianTag[] = "gregorian"; +static const char gDateTimePatternsTag[] = "DateTimePatterns"; + + +// latestFirst: +static const char16_t gLaterFirstPrefix[] = {LOW_L, LOW_A, LOW_T, LOW_E, LOW_S,LOW_T, CAP_F, LOW_I, LOW_R, LOW_S, LOW_T, COLON}; + +// earliestFirst: +static const char16_t gEarlierFirstPrefix[] = {LOW_E, LOW_A, LOW_R, LOW_L, LOW_I, LOW_E, LOW_S, LOW_T, CAP_F, LOW_I, LOW_R, LOW_S, LOW_T, COLON}; + + +class FormattedDateIntervalData : public FormattedValueFieldPositionIteratorImpl { +public: + FormattedDateIntervalData(UErrorCode& status) : FormattedValueFieldPositionIteratorImpl(5, status) {} + virtual ~FormattedDateIntervalData(); +}; + +FormattedDateIntervalData::~FormattedDateIntervalData() = default; + +UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(FormattedDateInterval) + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DateIntervalFormat) + +// Mutex, protects access to fDateFormat, fFromCalendar and fToCalendar. +// Needed because these data members are modified by const methods of DateIntervalFormat. + +static UMutex gFormatterMutex; + +DateIntervalFormat* U_EXPORT2 +DateIntervalFormat::createInstance(const UnicodeString& skeleton, + UErrorCode& status) { + return createInstance(skeleton, Locale::getDefault(), status); +} + + +DateIntervalFormat* U_EXPORT2 +DateIntervalFormat::createInstance(const UnicodeString& skeleton, + const Locale& locale, + UErrorCode& status) { +#ifdef DTITVFMT_DEBUG + char result[1000]; + char result_1[1000]; + char mesg[2000]; + skeleton.extract(0, skeleton.length(), result, "UTF-8"); + UnicodeString pat; + ((SimpleDateFormat*)dtfmt)->toPattern(pat); + pat.extract(0, pat.length(), result_1, "UTF-8"); + snprintf(mesg, sizeof(mesg), "skeleton: %s; pattern: %s\n", result, result_1); + PRINTMESG(mesg) +#endif + + DateIntervalInfo* dtitvinf = new DateIntervalInfo(locale, status); + if (dtitvinf == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return create(locale, dtitvinf, &skeleton, status); +} + + + +DateIntervalFormat* U_EXPORT2 +DateIntervalFormat::createInstance(const UnicodeString& skeleton, + const DateIntervalInfo& dtitvinf, + UErrorCode& status) { + return createInstance(skeleton, Locale::getDefault(), dtitvinf, status); +} + + +DateIntervalFormat* U_EXPORT2 +DateIntervalFormat::createInstance(const UnicodeString& skeleton, + const Locale& locale, + const DateIntervalInfo& dtitvinf, + UErrorCode& status) { + DateIntervalInfo* ptn = dtitvinf.clone(); + return create(locale, ptn, &skeleton, status); +} + + +DateIntervalFormat::DateIntervalFormat() +: fInfo(nullptr), + fDateFormat(nullptr), + fFromCalendar(nullptr), + fToCalendar(nullptr), + fLocale(Locale::getRoot()), + fDatePattern(nullptr), + fTimePattern(nullptr), + fDateTimeFormat(nullptr), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) +{} + + +DateIntervalFormat::DateIntervalFormat(const DateIntervalFormat& itvfmt) +: Format(itvfmt), + fInfo(nullptr), + fDateFormat(nullptr), + fFromCalendar(nullptr), + fToCalendar(nullptr), + fLocale(itvfmt.fLocale), + fDatePattern(nullptr), + fTimePattern(nullptr), + fDateTimeFormat(nullptr), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) { + *this = itvfmt; +} + + +DateIntervalFormat& +DateIntervalFormat::operator=(const DateIntervalFormat& itvfmt) { + if ( this != &itvfmt ) { + delete fDateFormat; + delete fInfo; + delete fFromCalendar; + delete fToCalendar; + delete fDatePattern; + delete fTimePattern; + delete fDateTimeFormat; + { + Mutex lock(&gFormatterMutex); + if ( itvfmt.fDateFormat ) { + fDateFormat = itvfmt.fDateFormat->clone(); + } else { + fDateFormat = nullptr; + } + if ( itvfmt.fFromCalendar ) { + fFromCalendar = itvfmt.fFromCalendar->clone(); + } else { + fFromCalendar = nullptr; + } + if ( itvfmt.fToCalendar ) { + fToCalendar = itvfmt.fToCalendar->clone(); + } else { + fToCalendar = nullptr; + } + } + if ( itvfmt.fInfo ) { + fInfo = itvfmt.fInfo->clone(); + } else { + fInfo = nullptr; + } + fSkeleton = itvfmt.fSkeleton; + int8_t i; + for ( i = 0; i< DateIntervalInfo::kIPI_MAX_INDEX; ++i ) { + fIntervalPatterns[i] = itvfmt.fIntervalPatterns[i]; + } + fLocale = itvfmt.fLocale; + fDatePattern = (itvfmt.fDatePattern)? itvfmt.fDatePattern->clone(): nullptr; + fTimePattern = (itvfmt.fTimePattern)? itvfmt.fTimePattern->clone(): nullptr; + fDateTimeFormat = (itvfmt.fDateTimeFormat)? itvfmt.fDateTimeFormat->clone(): nullptr; + fCapitalizationContext = itvfmt.fCapitalizationContext; + } + return *this; +} + + +DateIntervalFormat::~DateIntervalFormat() { + delete fInfo; + delete fDateFormat; + delete fFromCalendar; + delete fToCalendar; + delete fDatePattern; + delete fTimePattern; + delete fDateTimeFormat; +} + + +DateIntervalFormat* +DateIntervalFormat::clone() const { + return new DateIntervalFormat(*this); +} + + +bool +DateIntervalFormat::operator==(const Format& other) const { + if (typeid(*this) != typeid(other)) {return false;} + const DateIntervalFormat* fmt = (DateIntervalFormat*)&other; + if (this == fmt) {return true;} + if (!Format::operator==(other)) {return false;} + if ((fInfo != fmt->fInfo) && (fInfo == nullptr || fmt->fInfo == nullptr)) {return false;} + if (fInfo && fmt->fInfo && (*fInfo != *fmt->fInfo )) {return false;} + { + Mutex lock(&gFormatterMutex); + if (fDateFormat != fmt->fDateFormat && (fDateFormat == nullptr || fmt->fDateFormat == nullptr)) {return false;} + if (fDateFormat && fmt->fDateFormat && (*fDateFormat != *fmt->fDateFormat)) {return false;} + } + // note: fFromCalendar and fToCalendar hold no persistent state, and therefore do not participate in operator ==. + // fDateFormat has the primary calendar for the DateIntervalFormat. + if (fSkeleton != fmt->fSkeleton) {return false;} + if (fDatePattern != fmt->fDatePattern && (fDatePattern == nullptr || fmt->fDatePattern == nullptr)) {return false;} + if (fDatePattern && fmt->fDatePattern && (*fDatePattern != *fmt->fDatePattern)) {return false;} + if (fTimePattern != fmt->fTimePattern && (fTimePattern == nullptr || fmt->fTimePattern == nullptr)) {return false;} + if (fTimePattern && fmt->fTimePattern && (*fTimePattern != *fmt->fTimePattern)) {return false;} + if (fDateTimeFormat != fmt->fDateTimeFormat && (fDateTimeFormat == nullptr || fmt->fDateTimeFormat == nullptr)) {return false;} + if (fDateTimeFormat && fmt->fDateTimeFormat && (*fDateTimeFormat != *fmt->fDateTimeFormat)) {return false;} + if (fLocale != fmt->fLocale) {return false;} + + for (int32_t i = 0; i< DateIntervalInfo::kIPI_MAX_INDEX; ++i ) { + if (fIntervalPatterns[i].firstPart != fmt->fIntervalPatterns[i].firstPart) {return false;} + if (fIntervalPatterns[i].secondPart != fmt->fIntervalPatterns[i].secondPart ) {return false;} + if (fIntervalPatterns[i].laterDateFirst != fmt->fIntervalPatterns[i].laterDateFirst) {return false;} + } + if (fCapitalizationContext != fmt->fCapitalizationContext) {return false;} + return true; +} + + +UnicodeString& +DateIntervalFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& fieldPosition, + UErrorCode& status) const { + if ( U_FAILURE(status) ) { + return appendTo; + } + + if ( obj.getType() == Formattable::kObject ) { + const UObject* formatObj = obj.getObject(); + const DateInterval* interval = dynamic_cast(formatObj); + if (interval != nullptr) { + return format(interval, appendTo, fieldPosition, status); + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; +} + + +UnicodeString& +DateIntervalFormat::format(const DateInterval* dtInterval, + UnicodeString& appendTo, + FieldPosition& fieldPosition, + UErrorCode& status) const { + if ( U_FAILURE(status) ) { + return appendTo; + } + if (fDateFormat == nullptr || fInfo == nullptr) { + status = U_INVALID_STATE_ERROR; + return appendTo; + } + + FieldPositionOnlyHandler handler(fieldPosition); + handler.setAcceptFirstOnly(true); + int8_t ignore; + + Mutex lock(&gFormatterMutex); + return formatIntervalImpl(*dtInterval, appendTo, ignore, handler, status); +} + + +FormattedDateInterval DateIntervalFormat::formatToValue( + const DateInterval& dtInterval, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + // LocalPointer only sets OOM status if U_SUCCESS is true. + LocalPointer result(new FormattedDateIntervalData(status), status); + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + UnicodeString string; + int8_t firstIndex; + auto handler = result->getHandler(status); + handler.setCategory(UFIELD_CATEGORY_DATE); + { + Mutex lock(&gFormatterMutex); + formatIntervalImpl(dtInterval, string, firstIndex, handler, status); + } + handler.getError(status); + result->appendString(string, status); + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + + // Compute the span fields and sort them into place: + if (firstIndex != -1) { + result->addOverlapSpans(UFIELD_CATEGORY_DATE_INTERVAL_SPAN, firstIndex, status); + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + result->sort(); + } + + return FormattedDateInterval(result.orphan()); +} + + +UnicodeString& +DateIntervalFormat::format(Calendar& fromCalendar, + Calendar& toCalendar, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const { + FieldPositionOnlyHandler handler(pos); + handler.setAcceptFirstOnly(true); + int8_t ignore; + + Mutex lock(&gFormatterMutex); + return formatImpl(fromCalendar, toCalendar, appendTo, ignore, handler, status); +} + + +FormattedDateInterval DateIntervalFormat::formatToValue( + Calendar& fromCalendar, + Calendar& toCalendar, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + // LocalPointer only sets OOM status if U_SUCCESS is true. + LocalPointer result(new FormattedDateIntervalData(status), status); + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + UnicodeString string; + int8_t firstIndex; + auto handler = result->getHandler(status); + handler.setCategory(UFIELD_CATEGORY_DATE); + { + Mutex lock(&gFormatterMutex); + formatImpl(fromCalendar, toCalendar, string, firstIndex, handler, status); + } + handler.getError(status); + result->appendString(string, status); + if (U_FAILURE(status)) { + return FormattedDateInterval(status); + } + + // Compute the span fields and sort them into place: + if (firstIndex != -1) { + result->addOverlapSpans(UFIELD_CATEGORY_DATE_INTERVAL_SPAN, firstIndex, status); + result->sort(); + } + + return FormattedDateInterval(result.orphan()); +} + + +UnicodeString& DateIntervalFormat::formatIntervalImpl( + const DateInterval& dtInterval, + UnicodeString& appendTo, + int8_t& firstIndex, + FieldPositionHandler& fphandler, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; + } + if (fFromCalendar == nullptr || fToCalendar == nullptr) { + status = U_INVALID_STATE_ERROR; + return appendTo; + } + fFromCalendar->setTime(dtInterval.getFromDate(), status); + fToCalendar->setTime(dtInterval.getToDate(), status); + return formatImpl(*fFromCalendar, *fToCalendar, appendTo, firstIndex, fphandler, status); +} + + +// The following is only called from within the gFormatterMutex lock +UnicodeString& +DateIntervalFormat::formatImpl(Calendar& fromCalendar, + Calendar& toCalendar, + UnicodeString& appendTo, + int8_t& firstIndex, + FieldPositionHandler& fphandler, + UErrorCode& status) const { + if ( U_FAILURE(status) ) { + return appendTo; + } + + // Initialize firstIndex to -1 (single date, no range) + firstIndex = -1; + + // not support different calendar types and time zones + //if ( fromCalendar.getType() != toCalendar.getType() ) { + if ( !fromCalendar.isEquivalentTo(toCalendar) ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; + } + + // First, find the largest different calendar field. + UCalendarDateFields field = UCAL_FIELD_COUNT; + + if ( fromCalendar.get(UCAL_ERA,status) != toCalendar.get(UCAL_ERA,status)) { + field = UCAL_ERA; + } else if ( fromCalendar.get(UCAL_YEAR, status) != + toCalendar.get(UCAL_YEAR, status) ) { + field = UCAL_YEAR; + } else if ( fromCalendar.get(UCAL_MONTH, status) != + toCalendar.get(UCAL_MONTH, status) ) { + field = UCAL_MONTH; + } else if ( fromCalendar.get(UCAL_DATE, status) != + toCalendar.get(UCAL_DATE, status) ) { + field = UCAL_DATE; + } else if ( fromCalendar.get(UCAL_AM_PM, status) != + toCalendar.get(UCAL_AM_PM, status) ) { + field = UCAL_AM_PM; + } else if ( fromCalendar.get(UCAL_HOUR, status) != + toCalendar.get(UCAL_HOUR, status) ) { + field = UCAL_HOUR; + } else if ( fromCalendar.get(UCAL_MINUTE, status) != + toCalendar.get(UCAL_MINUTE, status) ) { + field = UCAL_MINUTE; + } else if ( fromCalendar.get(UCAL_SECOND, status) != + toCalendar.get(UCAL_SECOND, status) ) { + field = UCAL_SECOND; + } else if ( fromCalendar.get(UCAL_MILLISECOND, status) != + toCalendar.get(UCAL_MILLISECOND, status) ) { + field = UCAL_MILLISECOND; + } + + if ( U_FAILURE(status) ) { + return appendTo; + } + UErrorCode tempStatus = U_ZERO_ERROR; // for setContext, ignored + // Set up fDateFormat to handle the first or only part of the interval + // (override later for any second part). Inside lock, OK to modify fDateFormat. + fDateFormat->setContext(fCapitalizationContext, tempStatus); + + if ( field == UCAL_FIELD_COUNT ) { + /* ignore the millisecond etc. small fields' difference. + * use single date when all the above are the same. + */ + return fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + } + UBool fromToOnSameDay = (field==UCAL_AM_PM || field==UCAL_HOUR || field==UCAL_MINUTE || field==UCAL_SECOND || field==UCAL_MILLISECOND); + + // following call should not set wrong status, + // all the pass-in fields are valid till here + int32_t itvPtnIndex = DateIntervalInfo::calendarFieldToIntervalIndex(field, + status); + const PatternInfo& intervalPattern = fIntervalPatterns[itvPtnIndex]; + + if ( intervalPattern.firstPart.isEmpty() && + intervalPattern.secondPart.isEmpty() ) { + if ( fDateFormat->isFieldUnitIgnored(field) ) { + /* the largest different calendar field is small than + * the smallest calendar field in pattern, + * return single date format. + */ + return fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + } + return fallbackFormat(fromCalendar, toCalendar, fromToOnSameDay, appendTo, firstIndex, fphandler, status); + } + // If the first part in interval pattern is empty, + // the 2nd part of it saves the full-pattern used in fall-back. + // For a 'real' interval pattern, the first part will never be empty. + if ( intervalPattern.firstPart.isEmpty() ) { + // fall back + UnicodeString originalPattern; + fDateFormat->toPattern(originalPattern); + fDateFormat->applyPattern(intervalPattern.secondPart); + appendTo = fallbackFormat(fromCalendar, toCalendar, fromToOnSameDay, appendTo, firstIndex, fphandler, status); + fDateFormat->applyPattern(originalPattern); + return appendTo; + } + Calendar* firstCal; + Calendar* secondCal; + if ( intervalPattern.laterDateFirst ) { + firstCal = &toCalendar; + secondCal = &fromCalendar; + firstIndex = 1; + } else { + firstCal = &fromCalendar; + secondCal = &toCalendar; + firstIndex = 0; + } + // break the interval pattern into 2 parts, + // first part should not be empty, + UnicodeString originalPattern; + fDateFormat->toPattern(originalPattern); + fDateFormat->applyPattern(intervalPattern.firstPart); + fDateFormat->_format(*firstCal, appendTo, fphandler, status); + + if ( !intervalPattern.secondPart.isEmpty() ) { + fDateFormat->applyPattern(intervalPattern.secondPart); + // No capitalization for second part of interval + tempStatus = U_ZERO_ERROR; + fDateFormat->setContext(UDISPCTX_CAPITALIZATION_NONE, tempStatus); + fDateFormat->_format(*secondCal, appendTo, fphandler, status); + } + fDateFormat->applyPattern(originalPattern); + return appendTo; +} + + + +void +DateIntervalFormat::parseObject(const UnicodeString& /* source */, + Formattable& /* result */, + ParsePosition& /* parse_pos */) const { + // parseObject(const UnicodeString&, Formattable&, UErrorCode&) const + // will set status as U_INVALID_FORMAT_ERROR if + // parse_pos is still 0 +} + + + + +const DateIntervalInfo* +DateIntervalFormat::getDateIntervalInfo() const { + return fInfo; +} + + +void +DateIntervalFormat::setDateIntervalInfo(const DateIntervalInfo& newItvPattern, + UErrorCode& status) { + delete fInfo; + fInfo = new DateIntervalInfo(newItvPattern); + if (fInfo == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + + // Delete patterns that get reset by initializePattern + delete fDatePattern; + fDatePattern = nullptr; + delete fTimePattern; + fTimePattern = nullptr; + delete fDateTimeFormat; + fDateTimeFormat = nullptr; + + if (fDateFormat) { + initializePattern(status); + } +} + + + +const DateFormat* +DateIntervalFormat::getDateFormat() const { + return fDateFormat; +} + + +void +DateIntervalFormat::adoptTimeZone(TimeZone* zone) +{ + if (fDateFormat != nullptr) { + fDateFormat->adoptTimeZone(zone); + } + // The fDateFormat has the primary calendar for the DateIntervalFormat and has + // ownership of any adopted TimeZone; fFromCalendar and fToCalendar are internal + // work clones of that calendar (and should not also be given ownership of the + // adopted TimeZone). + if (fFromCalendar) { + fFromCalendar->setTimeZone(*zone); + } + if (fToCalendar) { + fToCalendar->setTimeZone(*zone); + } +} + +void +DateIntervalFormat::setTimeZone(const TimeZone& zone) +{ + if (fDateFormat != nullptr) { + fDateFormat->setTimeZone(zone); + } + // The fDateFormat has the primary calendar for the DateIntervalFormat; + // fFromCalendar and fToCalendar are internal work clones of that calendar. + if (fFromCalendar) { + fFromCalendar->setTimeZone(zone); + } + if (fToCalendar) { + fToCalendar->setTimeZone(zone); + } +} + +const TimeZone& +DateIntervalFormat::getTimeZone() const +{ + if (fDateFormat != nullptr) { + Mutex lock(&gFormatterMutex); + return fDateFormat->getTimeZone(); + } + // If fDateFormat is nullptr (unexpected), create default timezone. + return *(TimeZone::createDefault()); +} + +void +DateIntervalFormat::setContext(UDisplayContext value, UErrorCode& status) +{ + if (U_FAILURE(status)) + return; + if ( (UDisplayContextType)((uint32_t)value >> 8) == UDISPCTX_TYPE_CAPITALIZATION ) { + fCapitalizationContext = value; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + } +} + +UDisplayContext +DateIntervalFormat::getContext(UDisplayContextType type, UErrorCode& status) const +{ + if (U_FAILURE(status)) + return (UDisplayContext)0; + if (type != UDISPCTX_TYPE_CAPITALIZATION) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return (UDisplayContext)0; + } + return fCapitalizationContext; +} + +DateIntervalFormat::DateIntervalFormat(const Locale& locale, + DateIntervalInfo* dtItvInfo, + const UnicodeString* skeleton, + UErrorCode& status) +: fInfo(nullptr), + fDateFormat(nullptr), + fFromCalendar(nullptr), + fToCalendar(nullptr), + fLocale(locale), + fDatePattern(nullptr), + fTimePattern(nullptr), + fDateTimeFormat(nullptr), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) +{ + LocalPointer info(dtItvInfo, status); + LocalPointer dtfmt(static_cast( + DateFormat::createInstanceForSkeleton(*skeleton, locale, status)), status); + if (U_FAILURE(status)) { + return; + } + + if ( skeleton ) { + fSkeleton = *skeleton; + } + fInfo = info.orphan(); + fDateFormat = dtfmt.orphan(); + if ( fDateFormat->getCalendar() ) { + fFromCalendar = fDateFormat->getCalendar()->clone(); + fToCalendar = fDateFormat->getCalendar()->clone(); + } + initializePattern(status); +} + +DateIntervalFormat* U_EXPORT2 +DateIntervalFormat::create(const Locale& locale, + DateIntervalInfo* dtitvinf, + const UnicodeString* skeleton, + UErrorCode& status) { + DateIntervalFormat* f = new DateIntervalFormat(locale, dtitvinf, + skeleton, status); + if ( f == nullptr ) { + status = U_MEMORY_ALLOCATION_ERROR; + delete dtitvinf; + } else if ( U_FAILURE(status) ) { + // safe to delete f, although nothing actually is saved + delete f; + f = 0; + } + return f; +} + + + +/** + * Initialize interval patterns locale to this formatter + * + * This code is a bit complicated since + * 1. the interval patterns saved in resource bundle files are interval + * patterns based on date or time only. + * It does not have interval patterns based on both date and time. + * Interval patterns on both date and time are algorithm generated. + * + * For example, it has interval patterns on skeleton "dMy" and "hm", + * but it does not have interval patterns on skeleton "dMyhm". + * + * The rule to genearte interval patterns for both date and time skeleton are + * 1) when the year, month, or day differs, concatenate the two original + * expressions with a separator between, + * For example, interval pattern from "Jan 10, 2007 10:10 am" + * to "Jan 11, 2007 10:10am" is + * "Jan 10, 2007 10:10 am - Jan 11, 2007 10:10am" + * + * 2) otherwise, present the date followed by the range expression + * for the time. + * For example, interval pattern from "Jan 10, 2007 10:10 am" + * to "Jan 10, 2007 11:10am" is + * "Jan 10, 2007 10:10 am - 11:10am" + * + * 2. even a pattern does not request a certion calendar field, + * the interval pattern needs to include such field if such fields are + * different between 2 dates. + * For example, a pattern/skeleton is "hm", but the interval pattern + * includes year, month, and date when year, month, and date differs. + * + * @param status output param set to success/failure code on exit + * @stable ICU 4.0 + */ +void +DateIntervalFormat::initializePattern(UErrorCode& status) { + if ( U_FAILURE(status) ) { + return; + } + const Locale& locale = fDateFormat->getSmpFmtLocale(); + if ( fSkeleton.isEmpty() ) { + UnicodeString fullPattern; + fDateFormat->toPattern(fullPattern); +#ifdef DTITVFMT_DEBUG + char result[1000]; + char result_1[1000]; + char mesg[2000]; + fSkeleton.extract(0, fSkeleton.length(), result, "UTF-8"); + snprintf(mesg, sizeof(mesg), "in getBestSkeleton: fSkeleton: %s; \n", result); + PRINTMESG(mesg) +#endif + // fSkeleton is already set by createDateIntervalInstance() + // or by createInstance(UnicodeString skeleton, .... ) + fSkeleton = DateTimePatternGenerator::staticGetSkeleton( + fullPattern, status); + if ( U_FAILURE(status) ) { + return; + } + } + + // initialize the fIntervalPattern ordering + int8_t i; + for ( i = 0; i < DateIntervalInfo::kIPI_MAX_INDEX; ++i ) { + fIntervalPatterns[i].laterDateFirst = fInfo->getDefaultOrder(); + } + + /* Check whether the skeleton is a combination of date and time. + * For the complication reason 1 explained above. + */ + UnicodeString dateSkeleton; + UnicodeString timeSkeleton; + UnicodeString normalizedTimeSkeleton; + UnicodeString normalizedDateSkeleton; + + + /* the difference between time skeleton and normalizedTimeSkeleton are: + * 1. (Formerly, normalized time skeleton folded 'H' to 'h'; no longer true) + * 2. (Formerly, 'a' was omitted in normalized time skeleton; this is now handled elsewhere) + * 3. there is only one appearance for 'h' or 'H', 'm','v', 'z' in normalized + * time skeleton + * + * The difference between date skeleton and normalizedDateSkeleton are: + * 1. both 'y' and 'd' appear only once in normalizeDateSkeleton + * 2. 'E' and 'EE' are normalized into 'EEE' + * 3. 'MM' is normalized into 'M' + */ + UnicodeString convertedSkeleton = normalizeHourMetacharacters(fSkeleton); + getDateTimeSkeleton(convertedSkeleton, dateSkeleton, normalizedDateSkeleton, + timeSkeleton, normalizedTimeSkeleton); + +#ifdef DTITVFMT_DEBUG + char result[1000]; + char result_1[1000]; + char mesg[2000]; + fSkeleton.extract(0, fSkeleton.length(), result, "UTF-8"); + snprintf(mesg, sizeof(mesg), "in getBestSkeleton: fSkeleton: %s; \n", result); + PRINTMESG(mesg) +#endif + + // move this up here since we need it for fallbacks + if ( timeSkeleton.length() > 0 && dateSkeleton.length() > 0 ) { + // Need the Date/Time pattern for concatenation of the date + // with the time interval. + // The date/time pattern ( such as {0} {1} ) is saved in + // calendar, that is why need to get the CalendarData here. + LocalUResourceBundlePointer dateTimePatternsRes(ures_open(nullptr, locale.getBaseName(), &status)); + ures_getByKey(dateTimePatternsRes.getAlias(), gCalendarTag, + dateTimePatternsRes.getAlias(), &status); + ures_getByKeyWithFallback(dateTimePatternsRes.getAlias(), gGregorianTag, + dateTimePatternsRes.getAlias(), &status); + ures_getByKeyWithFallback(dateTimePatternsRes.getAlias(), gDateTimePatternsTag, + dateTimePatternsRes.getAlias(), &status); + + int32_t dateTimeFormatLength; + const char16_t* dateTimeFormat = ures_getStringByIndex( + dateTimePatternsRes.getAlias(), + (int32_t)DateFormat::kDateTime, + &dateTimeFormatLength, &status); + if ( U_SUCCESS(status) && dateTimeFormatLength >= 3 ) { + fDateTimeFormat = new UnicodeString(dateTimeFormat, dateTimeFormatLength); + if (fDateTimeFormat == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + } + + UBool found = setSeparateDateTimePtn(normalizedDateSkeleton, + normalizedTimeSkeleton); + + // for skeletons with seconds, found is false and we enter this block + if ( found == false ) { + // use fallback + // TODO: if user asks "m"(minute), but "d"(day) differ + if ( timeSkeleton.length() != 0 ) { + if ( dateSkeleton.length() == 0 ) { + // prefix with yMd + timeSkeleton.insert(0, gDateFormatSkeleton[DateFormat::kShort], -1); + UnicodeString pattern = DateFormat::getBestPattern( + locale, timeSkeleton, status); + if ( U_FAILURE(status) ) { + return; + } + // for fall back interval patterns, + // the first part of the pattern is empty, + // the second part of the pattern is the full-pattern + // should be used in fall-back. + setPatternInfo(UCAL_DATE, nullptr, &pattern, fInfo->getDefaultOrder()); + setPatternInfo(UCAL_MONTH, nullptr, &pattern, fInfo->getDefaultOrder()); + setPatternInfo(UCAL_YEAR, nullptr, &pattern, fInfo->getDefaultOrder()); + + timeSkeleton.insert(0, CAP_G); + pattern = DateFormat::getBestPattern( + locale, timeSkeleton, status); + if ( U_FAILURE(status) ) { + return; + } + setPatternInfo(UCAL_ERA, nullptr, &pattern, fInfo->getDefaultOrder()); + } else { + // TODO: fall back + } + } else { + // TODO: fall back + } + return; + } // end of skeleton not found + // interval patterns for skeleton are found in resource + if ( timeSkeleton.length() == 0 ) { + // done + } else if ( dateSkeleton.length() == 0 ) { + // prefix with yMd + timeSkeleton.insert(0, gDateFormatSkeleton[DateFormat::kShort], -1); + UnicodeString pattern = DateFormat::getBestPattern( + locale, timeSkeleton, status); + if ( U_FAILURE(status) ) { + return; + } + // for fall back interval patterns, + // the first part of the pattern is empty, + // the second part of the pattern is the full-pattern + // should be used in fall-back. + setPatternInfo(UCAL_DATE, nullptr, &pattern, fInfo->getDefaultOrder()); + setPatternInfo(UCAL_MONTH, nullptr, &pattern, fInfo->getDefaultOrder()); + setPatternInfo(UCAL_YEAR, nullptr, &pattern, fInfo->getDefaultOrder()); + + timeSkeleton.insert(0, CAP_G); + pattern = DateFormat::getBestPattern( + locale, timeSkeleton, status); + if ( U_FAILURE(status) ) { + return; + } + setPatternInfo(UCAL_ERA, nullptr, &pattern, fInfo->getDefaultOrder()); + } else { + /* if both present, + * 1) when the era, year, month, or day differs, + * concatenate the two original expressions with a separator between, + * 2) otherwise, present the date followed by the + * range expression for the time. + */ + /* + * 1) when the era, year, month, or day differs, + * concatenate the two original expressions with a separator between, + */ + // if field exists, use fall back + UnicodeString skeleton = fSkeleton; + if ( !fieldExistsInSkeleton(UCAL_DATE, dateSkeleton) ) { + // prefix skeleton with 'd' + skeleton.insert(0, LOW_D); + setFallbackPattern(UCAL_DATE, skeleton, status); + } + if ( !fieldExistsInSkeleton(UCAL_MONTH, dateSkeleton) ) { + // then prefix skeleton with 'M' + skeleton.insert(0, CAP_M); + setFallbackPattern(UCAL_MONTH, skeleton, status); + } + if ( !fieldExistsInSkeleton(UCAL_YEAR, dateSkeleton) ) { + // then prefix skeleton with 'y' + skeleton.insert(0, LOW_Y); + setFallbackPattern(UCAL_YEAR, skeleton, status); + } + if ( !fieldExistsInSkeleton(UCAL_ERA, dateSkeleton) ) { + // then prefix skeleton with 'G' + skeleton.insert(0, CAP_G); + setFallbackPattern(UCAL_ERA, skeleton, status); + } + + /* + * 2) otherwise, present the date followed by the + * range expression for the time. + */ + + if ( fDateTimeFormat == nullptr ) { + // earlier failure getting dateTimeFormat + return; + } + + UnicodeString datePattern = DateFormat::getBestPattern( + locale, dateSkeleton, status); + + concatSingleDate2TimeInterval(*fDateTimeFormat, datePattern, UCAL_AM_PM, status); + concatSingleDate2TimeInterval(*fDateTimeFormat, datePattern, UCAL_HOUR, status); + concatSingleDate2TimeInterval(*fDateTimeFormat, datePattern, UCAL_MINUTE, status); + } +} + + + +UnicodeString +DateIntervalFormat::normalizeHourMetacharacters(const UnicodeString& skeleton) const { + UnicodeString result = skeleton; + + char16_t hourMetachar = u'\0'; + char16_t dayPeriodChar = u'\0'; + int32_t hourFieldStart = 0; + int32_t hourFieldLength = 0; + int32_t dayPeriodStart = 0; + int32_t dayPeriodLength = 0; + for (int32_t i = 0; i < result.length(); i++) { + char16_t c = result[i]; + if (c == LOW_J || c == CAP_J || c == CAP_C || c == LOW_H || c == CAP_H || c == LOW_K || c == CAP_K) { + if (hourMetachar == u'\0') { + hourMetachar = c; + hourFieldStart = i; + } + ++hourFieldLength; + } else if (c == LOW_A || c == LOW_B || c == CAP_B) { + if (dayPeriodChar == u'\0') { + dayPeriodChar = c; + dayPeriodStart = i; + } + ++dayPeriodLength; + } else { + if (hourMetachar != u'\0' && dayPeriodChar != u'\0') { + break; + } + } + } + + if (hourMetachar != u'\0') { + UErrorCode err = U_ZERO_ERROR; + char16_t hourChar = CAP_H; + UnicodeString convertedPattern = DateFormat::getBestPattern(fLocale, UnicodeString(hourMetachar), err); + + if (U_SUCCESS(err)) { + // strip literal text from the pattern (so literal characters don't get mistaken for pattern + // characters-- such as the 'h' in 'Uhr' in Germam) + int32_t firstQuotePos; + while ((firstQuotePos = convertedPattern.indexOf(u'\'')) != -1) { + int32_t secondQuotePos = convertedPattern.indexOf(u'\'', firstQuotePos + 1); + if (secondQuotePos == -1) { + secondQuotePos = firstQuotePos; + } + convertedPattern.replace(firstQuotePos, (secondQuotePos - firstQuotePos) + 1, UnicodeString()); + } + + if (convertedPattern.indexOf(LOW_H) != -1) { + hourChar = LOW_H; + } else if (convertedPattern.indexOf(CAP_K) != -1) { + hourChar = CAP_K; + } else if (convertedPattern.indexOf(LOW_K) != -1) { + hourChar = LOW_K; + } + + if (convertedPattern.indexOf(LOW_B) != -1) { + dayPeriodChar = LOW_B; + } else if (convertedPattern.indexOf(CAP_B) != -1) { + dayPeriodChar = CAP_B; + } else if (dayPeriodChar == u'\0') { + dayPeriodChar = LOW_A; + } + } + + UnicodeString hourAndDayPeriod(hourChar); + if (hourChar != CAP_H && hourChar != LOW_K) { + int32_t newDayPeriodLength = 0; + if (dayPeriodLength >= 5 || hourFieldLength >= 5) { + newDayPeriodLength = 5; + } else if (dayPeriodLength >= 3 || hourFieldLength >= 3) { + newDayPeriodLength = 3; + } else { + newDayPeriodLength = 1; + } + for (int32_t i = 0; i < newDayPeriodLength; i++) { + hourAndDayPeriod.append(dayPeriodChar); + } + } + result.replace(hourFieldStart, hourFieldLength, hourAndDayPeriod); + if (dayPeriodStart > hourFieldStart) { + // before deleting the original day period field, adjust its position in case + // we just changed the size of the hour field (and new day period field) + dayPeriodStart += hourAndDayPeriod.length() - hourFieldLength; + } + result.remove(dayPeriodStart, dayPeriodLength); + } + return result; +} + + +void U_EXPORT2 +DateIntervalFormat::getDateTimeSkeleton(const UnicodeString& skeleton, + UnicodeString& dateSkeleton, + UnicodeString& normalizedDateSkeleton, + UnicodeString& timeSkeleton, + UnicodeString& normalizedTimeSkeleton) { + // dateSkeleton follows the sequence of y*M*E*d* + // timeSkeleton follows the sequence of hm*[v|z]? + int32_t ECount = 0; + int32_t dCount = 0; + int32_t MCount = 0; + int32_t yCount = 0; + int32_t mCount = 0; + int32_t vCount = 0; + int32_t zCount = 0; + char16_t hourChar = u'\0'; + int32_t i; + + for (i = 0; i < skeleton.length(); ++i) { + char16_t ch = skeleton[i]; + switch ( ch ) { + case CAP_E: + dateSkeleton.append(ch); + ++ECount; + break; + case LOW_D: + dateSkeleton.append(ch); + ++dCount; + break; + case CAP_M: + dateSkeleton.append(ch); + ++MCount; + break; + case LOW_Y: + dateSkeleton.append(ch); + ++yCount; + break; + case CAP_G: + case CAP_Y: + case LOW_U: + case CAP_Q: + case LOW_Q: + case CAP_L: + case LOW_L: + case CAP_W: + case LOW_W: + case CAP_D: + case CAP_F: + case LOW_G: + case LOW_E: + case LOW_C: + case CAP_U: + case LOW_R: + normalizedDateSkeleton.append(ch); + dateSkeleton.append(ch); + break; + case LOW_H: + case CAP_H: + case LOW_K: + case CAP_K: + timeSkeleton.append(ch); + if (hourChar == u'\0') { + hourChar = ch; + } + break; + case LOW_M: + timeSkeleton.append(ch); + ++mCount; + break; + case LOW_Z: + ++zCount; + timeSkeleton.append(ch); + break; + case LOW_V: + ++vCount; + timeSkeleton.append(ch); + break; + case LOW_A: + case CAP_V: + case CAP_Z: + case LOW_J: + case LOW_S: + case CAP_S: + case CAP_A: + case LOW_B: + case CAP_B: + timeSkeleton.append(ch); + normalizedTimeSkeleton.append(ch); + break; + } + } + + /* generate normalized form for date*/ + if ( yCount != 0 ) { + for (i = 0; i < yCount; ++i) { + normalizedDateSkeleton.append(LOW_Y); + } + } + if ( MCount != 0 ) { + if ( MCount < 3 ) { + normalizedDateSkeleton.append(CAP_M); + } else { + for ( int32_t j = 0; j < MCount && j < MAX_M_COUNT; ++j) { + normalizedDateSkeleton.append(CAP_M); + } + } + } + if ( ECount != 0 ) { + if ( ECount <= 3 ) { + normalizedDateSkeleton.append(CAP_E); + } else { + for ( int32_t j = 0; j < ECount && j < MAX_E_COUNT; ++j ) { + normalizedDateSkeleton.append(CAP_E); + } + } + } + if ( dCount != 0 ) { + normalizedDateSkeleton.append(LOW_D); + } + + /* generate normalized form for time */ + if ( hourChar != u'\0' ) { + normalizedTimeSkeleton.append(hourChar); + } + if ( mCount != 0 ) { + normalizedTimeSkeleton.append(LOW_M); + } + if ( zCount != 0 ) { + normalizedTimeSkeleton.append(LOW_Z); + } + if ( vCount != 0 ) { + normalizedTimeSkeleton.append(LOW_V); + } +} + + +/** + * Generate date or time interval pattern from resource, + * and set them into the interval pattern locale to this formatter. + * + * It needs to handle the following: + * 1. need to adjust field width. + * For example, the interval patterns saved in DateIntervalInfo + * includes "dMMMy", but not "dMMMMy". + * Need to get interval patterns for dMMMMy from dMMMy. + * Another example, the interval patterns saved in DateIntervalInfo + * includes "hmv", but not "hmz". + * Need to get interval patterns for "hmz' from 'hmv' + * + * 2. there might be no pattern for 'y' differ for skeleton "Md", + * in order to get interval patterns for 'y' differ, + * need to look for it from skeleton 'yMd' + * + * @param dateSkeleton normalized date skeleton + * @param timeSkeleton normalized time skeleton + * @return whether the resource is found for the skeleton. + * true if interval pattern found for the skeleton, + * false otherwise. + * @stable ICU 4.0 + */ +UBool +DateIntervalFormat::setSeparateDateTimePtn( + const UnicodeString& dateSkeleton, + const UnicodeString& timeSkeleton) { + const UnicodeString* skeleton; + // if both date and time skeleton present, + // the final interval pattern might include time interval patterns + // ( when, am_pm, hour, minute differ ), + // but not date interval patterns ( when year, month, day differ ). + // For year/month/day differ, it falls back to fall-back pattern. + if ( timeSkeleton.length() != 0 ) { + skeleton = &timeSkeleton; + } else { + skeleton = &dateSkeleton; + } + + /* interval patterns for skeleton "dMMMy" (but not "dMMMMy") + * are defined in resource, + * interval patterns for skeleton "dMMMMy" are calculated by + * 1. get the best match skeleton for "dMMMMy", which is "dMMMy" + * 2. get the interval patterns for "dMMMy", + * 3. extend "MMM" to "MMMM" in above interval patterns for "dMMMMy" + * getBestSkeleton() is step 1. + */ + // best skeleton, and the difference information + int8_t differenceInfo = 0; + const UnicodeString* bestSkeleton = fInfo->getBestSkeleton(*skeleton, + differenceInfo); + /* best skeleton could be nullptr. + For example: in "ca" resource file, + interval format is defined as following + intervalFormats{ + fallback{"{0} - {1}"} + } + there is no skeletons/interval patterns defined, + and the best skeleton match could be nullptr + */ + if ( bestSkeleton == nullptr ) { + return false; + } + + // Set patterns for fallback use, need to do this + // before returning if differenceInfo == -1 + UErrorCode status; + if ( dateSkeleton.length() != 0) { + status = U_ZERO_ERROR; + fDatePattern = new UnicodeString(DateFormat::getBestPattern( + fLocale, dateSkeleton, status)); + // no way to report OOM. :( + } + if ( timeSkeleton.length() != 0) { + status = U_ZERO_ERROR; + fTimePattern = new UnicodeString(DateFormat::getBestPattern( + fLocale, timeSkeleton, status)); + // no way to report OOM. :( + } + + // difference: + // 0 means the best matched skeleton is the same as input skeleton + // 1 means the fields are the same, but field width are different + // 2 means the only difference between fields are v/z, + // -1 means there are other fields difference + // (this will happen, for instance, if the supplied skeleton has seconds, + // but no skeletons in the intervalFormats data do) + if ( differenceInfo == -1 ) { + // skeleton has different fields, not only v/z difference + return false; + } + + if ( timeSkeleton.length() == 0 ) { + UnicodeString extendedSkeleton; + UnicodeString extendedBestSkeleton; + // only has date skeleton + setIntervalPattern(UCAL_DATE, skeleton, bestSkeleton, differenceInfo, + &extendedSkeleton, &extendedBestSkeleton); + + UBool extended = setIntervalPattern(UCAL_MONTH, skeleton, bestSkeleton, + differenceInfo, + &extendedSkeleton, &extendedBestSkeleton); + + if ( extended ) { + bestSkeleton = &extendedBestSkeleton; + skeleton = &extendedSkeleton; + } + setIntervalPattern(UCAL_YEAR, skeleton, bestSkeleton, differenceInfo, + &extendedSkeleton, &extendedBestSkeleton); + setIntervalPattern(UCAL_ERA, skeleton, bestSkeleton, differenceInfo, + &extendedSkeleton, &extendedBestSkeleton); + } else { + setIntervalPattern(UCAL_MINUTE, skeleton, bestSkeleton, differenceInfo); + setIntervalPattern(UCAL_HOUR, skeleton, bestSkeleton, differenceInfo); + setIntervalPattern(UCAL_AM_PM, skeleton, bestSkeleton, differenceInfo); + } + return true; +} + + + +void +DateIntervalFormat::setFallbackPattern(UCalendarDateFields field, + const UnicodeString& skeleton, + UErrorCode& status) { + if ( U_FAILURE(status) ) { + return; + } + UnicodeString pattern = DateFormat::getBestPattern( + fLocale, skeleton, status); + if ( U_FAILURE(status) ) { + return; + } + setPatternInfo(field, nullptr, &pattern, fInfo->getDefaultOrder()); +} + + + + +void +DateIntervalFormat::setPatternInfo(UCalendarDateFields field, + const UnicodeString* firstPart, + const UnicodeString* secondPart, + UBool laterDateFirst) { + // for fall back interval patterns, + // the first part of the pattern is empty, + // the second part of the pattern is the full-pattern + // should be used in fall-back. + UErrorCode status = U_ZERO_ERROR; + // following should not set any wrong status. + int32_t itvPtnIndex = DateIntervalInfo::calendarFieldToIntervalIndex(field, + status); + if ( U_FAILURE(status) ) { + return; + } + PatternInfo& ptn = fIntervalPatterns[itvPtnIndex]; + if ( firstPart ) { + ptn.firstPart = *firstPart; + } + if ( secondPart ) { + ptn.secondPart = *secondPart; + } + ptn.laterDateFirst = laterDateFirst; +} + +void +DateIntervalFormat::setIntervalPattern(UCalendarDateFields field, + const UnicodeString& intervalPattern) { + UBool order = fInfo->getDefaultOrder(); + setIntervalPattern(field, intervalPattern, order); +} + + +void +DateIntervalFormat::setIntervalPattern(UCalendarDateFields field, + const UnicodeString& intervalPattern, + UBool laterDateFirst) { + const UnicodeString* pattern = &intervalPattern; + UBool order = laterDateFirst; + // check for "latestFirst:" or "earliestFirst:" prefix + int8_t prefixLength = UPRV_LENGTHOF(gLaterFirstPrefix); + int8_t earliestFirstLength = UPRV_LENGTHOF(gEarlierFirstPrefix); + UnicodeString realPattern; + if ( intervalPattern.startsWith(gLaterFirstPrefix, prefixLength) ) { + order = true; + intervalPattern.extract(prefixLength, + intervalPattern.length() - prefixLength, + realPattern); + pattern = &realPattern; + } else if ( intervalPattern.startsWith(gEarlierFirstPrefix, + earliestFirstLength) ) { + order = false; + intervalPattern.extract(earliestFirstLength, + intervalPattern.length() - earliestFirstLength, + realPattern); + pattern = &realPattern; + } + + int32_t splitPoint = splitPatternInto2Part(*pattern); + + UnicodeString firstPart; + UnicodeString secondPart; + pattern->extract(0, splitPoint, firstPart); + if ( splitPoint < pattern->length() ) { + pattern->extract(splitPoint, pattern->length()-splitPoint, secondPart); + } + setPatternInfo(field, &firstPart, &secondPart, order); +} + + + + +/** + * Generate interval pattern from existing resource + * + * It not only save the interval patterns, + * but also return the extended skeleton and its best match skeleton. + * + * @param field largest different calendar field + * @param skeleton skeleton + * @param bestSkeleton the best match skeleton which has interval pattern + * defined in resource + * @param differenceInfo the difference between skeleton and best skeleton + * 0 means the best matched skeleton is the same as input skeleton + * 1 means the fields are the same, but field width are different + * 2 means the only difference between fields are v/z, + * -1 means there are other fields difference + * + * @param extendedSkeleton extended skeleton + * @param extendedBestSkeleton extended best match skeleton + * @return whether the interval pattern is found + * through extending skeleton or not. + * true if interval pattern is found by + * extending skeleton, false otherwise. + * @stable ICU 4.0 + */ +UBool +DateIntervalFormat::setIntervalPattern(UCalendarDateFields field, + const UnicodeString* skeleton, + const UnicodeString* bestSkeleton, + int8_t differenceInfo, + UnicodeString* extendedSkeleton, + UnicodeString* extendedBestSkeleton) { + UErrorCode status = U_ZERO_ERROR; + // following getIntervalPattern() should not generate error status + UnicodeString pattern; + fInfo->getIntervalPattern(*bestSkeleton, field, pattern, status); + if ( pattern.isEmpty() ) { + // single date + if ( SimpleDateFormat::isFieldUnitIgnored(*bestSkeleton, field) ) { + // do nothing, format will handle it + return false; + } + + // for 24 hour system, interval patterns in resource file + // might not include pattern when am_pm differ, + // which should be the same as hour differ. + // add it here for simplicity + if ( field == UCAL_AM_PM ) { + fInfo->getIntervalPattern(*bestSkeleton, UCAL_HOUR, pattern,status); + if ( !pattern.isEmpty() ) { + UBool suppressDayPeriodField = fSkeleton.indexOf(CAP_J) != -1; + UnicodeString adjustIntervalPattern; + adjustFieldWidth(*skeleton, *bestSkeleton, pattern, differenceInfo, + suppressDayPeriodField, adjustIntervalPattern); + setIntervalPattern(field, adjustIntervalPattern); + } + return false; + } + // else, looking for pattern when 'y' differ for 'dMMMM' skeleton, + // first, get best match pattern "MMMd", + // since there is no pattern for 'y' differs for skeleton 'MMMd', + // need to look for it from skeleton 'yMMMd', + // if found, adjust field width in interval pattern from + // "MMM" to "MMMM". + char16_t fieldLetter = fgCalendarFieldToPatternLetter[field]; + if ( extendedSkeleton ) { + *extendedSkeleton = *skeleton; + *extendedBestSkeleton = *bestSkeleton; + extendedSkeleton->insert(0, fieldLetter); + extendedBestSkeleton->insert(0, fieldLetter); + // for example, looking for patterns when 'y' differ for + // skeleton "MMMM". + fInfo->getIntervalPattern(*extendedBestSkeleton,field,pattern,status); + if ( pattern.isEmpty() && differenceInfo == 0 ) { + // if there is no skeleton "yMMMM" defined, + // look for the best match skeleton, for example: "yMMM" + const UnicodeString* tmpBest = fInfo->getBestSkeleton( + *extendedBestSkeleton, differenceInfo); + if ( tmpBest != 0 && differenceInfo != -1 ) { + fInfo->getIntervalPattern(*tmpBest, field, pattern, status); + bestSkeleton = tmpBest; + } + } + } + } + if ( !pattern.isEmpty() ) { + UBool suppressDayPeriodField = fSkeleton.indexOf(CAP_J) != -1; + if ( differenceInfo != 0 || suppressDayPeriodField) { + UnicodeString adjustIntervalPattern; + adjustFieldWidth(*skeleton, *bestSkeleton, pattern, differenceInfo, + suppressDayPeriodField, adjustIntervalPattern); + setIntervalPattern(field, adjustIntervalPattern); + } else { + setIntervalPattern(field, pattern); + } + if ( extendedSkeleton && !extendedSkeleton->isEmpty() ) { + return true; + } + } + return false; +} + + + +int32_t U_EXPORT2 +DateIntervalFormat::splitPatternInto2Part(const UnicodeString& intervalPattern) { + UBool inQuote = false; + char16_t prevCh = 0; + int32_t count = 0; + + /* repeatedPattern used to record whether a pattern has already seen. + It is a pattern applies to first calendar if it is first time seen, + otherwise, it is a pattern applies to the second calendar + */ + UBool patternRepeated[] = + { + // A B C D E F G H I J K L M N O + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // P Q R S T U V W X Y Z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // a b c d e f g h i j k l m n o + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // p q r s t u v w x y z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + int8_t PATTERN_CHAR_BASE = 0x41; + + /* loop through the pattern string character by character looking for + * the first repeated pattern letter, which breaks the interval pattern + * into 2 parts. + */ + int32_t i; + UBool foundRepetition = false; + for (i = 0; i < intervalPattern.length(); ++i) { + char16_t ch = intervalPattern.charAt(i); + + if (ch != prevCh && count > 0) { + // check the repeativeness of pattern letter + UBool repeated = patternRepeated[(int)(prevCh - PATTERN_CHAR_BASE)]; + if ( repeated == false ) { + patternRepeated[prevCh - PATTERN_CHAR_BASE] = true; + } else { + foundRepetition = true; + break; + } + count = 0; + } + if (ch == 0x0027 /*'*/) { + // Consecutive single quotes are a single quote literal, + // either outside of quotes or between quotes + if ((i+1) < intervalPattern.length() && + intervalPattern.charAt(i+1) == 0x0027 /*'*/) { + ++i; + } else { + inQuote = ! inQuote; + } + } + else if (!inQuote && ((ch >= 0x0061 /*'a'*/ && ch <= 0x007A /*'z'*/) + || (ch >= 0x0041 /*'A'*/ && ch <= 0x005A /*'Z'*/))) { + // ch is a date-time pattern character + prevCh = ch; + ++count; + } + } + // check last pattern char, distinguish + // "dd MM" ( no repetition ), + // "d-d"(last char repeated ), and + // "d-d MM" ( repetition found ) + if ( count > 0 && foundRepetition == false ) { + if ( patternRepeated[(int)(prevCh - PATTERN_CHAR_BASE)] == false ) { + count = 0; + } + } + return (i - count); +} + +// The following is only called from fallbackFormat, i.e. within the gFormatterMutex lock +void DateIntervalFormat::fallbackFormatRange( + Calendar& fromCalendar, + Calendar& toCalendar, + UnicodeString& appendTo, + int8_t& firstIndex, + FieldPositionHandler& fphandler, + UErrorCode& status) const { + UnicodeString fallbackPattern; + fInfo->getFallbackIntervalPattern(fallbackPattern); + SimpleFormatter sf(fallbackPattern, 2, 2, status); + if (U_FAILURE(status)) { + return; + } + int32_t offsets[2]; + UnicodeString patternBody = sf.getTextWithNoArguments(offsets, 2); + + UErrorCode tempStatus = U_ZERO_ERROR; // for setContext, ignored + // TODO(ICU-20406): Use SimpleFormatter Iterator interface when available. + if (offsets[0] < offsets[1]) { + firstIndex = 0; + appendTo.append(patternBody.tempSubStringBetween(0, offsets[0])); + fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[0], offsets[1])); + // No capitalization for second part of interval + fDateFormat->setContext(UDISPCTX_CAPITALIZATION_NONE, tempStatus); + fDateFormat->_format(toCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[1])); + } else { + firstIndex = 1; + appendTo.append(patternBody.tempSubStringBetween(0, offsets[1])); + fDateFormat->_format(toCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[1], offsets[0])); + // No capitalization for second part of interval + fDateFormat->setContext(UDISPCTX_CAPITALIZATION_NONE, tempStatus); + fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[0])); + } +} + +// The following is only called from formatImpl, i.e. within the gFormatterMutex lock +UnicodeString& +DateIntervalFormat::fallbackFormat(Calendar& fromCalendar, + Calendar& toCalendar, + UBool fromToOnSameDay, // new + UnicodeString& appendTo, + int8_t& firstIndex, + FieldPositionHandler& fphandler, + UErrorCode& status) const { + if ( U_FAILURE(status) ) { + return appendTo; + } + + UBool formatDatePlusTimeRange = (fromToOnSameDay && fDatePattern && fTimePattern); + if (formatDatePlusTimeRange) { + SimpleFormatter sf(*fDateTimeFormat, 2, 2, status); + if (U_FAILURE(status)) { + return appendTo; + } + int32_t offsets[2]; + UnicodeString patternBody = sf.getTextWithNoArguments(offsets, 2); + + UnicodeString fullPattern; // for saving the pattern in fDateFormat + fDateFormat->toPattern(fullPattern); // save current pattern, restore later + + UErrorCode tempStatus = U_ZERO_ERROR; // for setContext, ignored + // {0} is time range + // {1} is single date portion + // TODO(ICU-20406): Use SimpleFormatter Iterator interface when available. + if (offsets[0] < offsets[1]) { + appendTo.append(patternBody.tempSubStringBetween(0, offsets[0])); + fDateFormat->applyPattern(*fTimePattern); + fallbackFormatRange(fromCalendar, toCalendar, appendTo, firstIndex, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[0], offsets[1])); + fDateFormat->applyPattern(*fDatePattern); + // No capitalization for second portion + fDateFormat->setContext(UDISPCTX_CAPITALIZATION_NONE, tempStatus); + fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[1])); + } else { + appendTo.append(patternBody.tempSubStringBetween(0, offsets[1])); + fDateFormat->applyPattern(*fDatePattern); + fDateFormat->_format(fromCalendar, appendTo, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[1], offsets[0])); + fDateFormat->applyPattern(*fTimePattern); + // No capitalization for second portion + fDateFormat->setContext(UDISPCTX_CAPITALIZATION_NONE, tempStatus); + fallbackFormatRange(fromCalendar, toCalendar, appendTo, firstIndex, fphandler, status); + appendTo.append(patternBody.tempSubStringBetween(offsets[0])); + } + + // restore full pattern + fDateFormat->applyPattern(fullPattern); + } else { + fallbackFormatRange(fromCalendar, toCalendar, appendTo, firstIndex, fphandler, status); + } + return appendTo; +} + + + + +UBool U_EXPORT2 +DateIntervalFormat::fieldExistsInSkeleton(UCalendarDateFields field, + const UnicodeString& skeleton) +{ + const char16_t fieldChar = fgCalendarFieldToPatternLetter[field]; + return ( (skeleton.indexOf(fieldChar) == -1)?false:true ) ; +} + + + +void U_EXPORT2 +DateIntervalFormat::adjustFieldWidth(const UnicodeString& inputSkeleton, + const UnicodeString& bestMatchSkeleton, + const UnicodeString& bestIntervalPattern, + int8_t differenceInfo, + UBool suppressDayPeriodField, + UnicodeString& adjustedPtn) { + adjustedPtn = bestIntervalPattern; + int32_t inputSkeletonFieldWidth[] = + { + // A B C D E F G H I J K L M N O + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // P Q R S T U V W X Y Z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // a b c d e f g h i j k l m n o + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // p q r s t u v w x y z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + int32_t bestMatchSkeletonFieldWidth[] = + { + // A B C D E F G H I J K L M N O + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // P Q R S T U V W X Y Z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // a b c d e f g h i j k l m n o + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // p q r s t u v w x y z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + const int8_t PATTERN_CHAR_BASE = 0x41; + + DateIntervalInfo::parseSkeleton(inputSkeleton, inputSkeletonFieldWidth); + DateIntervalInfo::parseSkeleton(bestMatchSkeleton, bestMatchSkeletonFieldWidth); + if (suppressDayPeriodField) { + // remove the 'a' and any NBSP/NNBSP on one side of it + findReplaceInPattern(adjustedPtn, UnicodeString(u"\u00A0a",-1), UnicodeString()); + findReplaceInPattern(adjustedPtn, UnicodeString(u"\u202Fa",-1), UnicodeString()); + findReplaceInPattern(adjustedPtn, UnicodeString(u"a\u00A0",-1), UnicodeString()); + findReplaceInPattern(adjustedPtn, UnicodeString(u"a\u202F",-1), UnicodeString()); + findReplaceInPattern(adjustedPtn, UnicodeString(LOW_A), UnicodeString()); + // adjust interior double spaces, remove exterior whitespace + findReplaceInPattern(adjustedPtn, UnicodeString(" "), UnicodeString(" ")); + adjustedPtn.trim(); + } + if ( differenceInfo == 2 ) { + if (inputSkeleton.indexOf(LOW_Z) != -1) { + findReplaceInPattern(adjustedPtn, UnicodeString(LOW_V), UnicodeString(LOW_Z)); + } + if (inputSkeleton.indexOf(CAP_K) != -1) { + findReplaceInPattern(adjustedPtn, UnicodeString(LOW_H), UnicodeString(CAP_K)); + } + if (inputSkeleton.indexOf(LOW_K) != -1) { + findReplaceInPattern(adjustedPtn, UnicodeString(CAP_H), UnicodeString(LOW_K)); + } + if (inputSkeleton.indexOf(LOW_B) != -1) { + findReplaceInPattern(adjustedPtn, UnicodeString(LOW_A), UnicodeString(LOW_B)); + } + } + if (adjustedPtn.indexOf(LOW_A) != -1 && bestMatchSkeletonFieldWidth[LOW_A - PATTERN_CHAR_BASE] == 0) { + bestMatchSkeletonFieldWidth[LOW_A - PATTERN_CHAR_BASE] = 1; + } + if (adjustedPtn.indexOf(LOW_B) != -1 && bestMatchSkeletonFieldWidth[LOW_B - PATTERN_CHAR_BASE] == 0) { + bestMatchSkeletonFieldWidth[LOW_B - PATTERN_CHAR_BASE] = 1; + } + + UBool inQuote = false; + char16_t prevCh = 0; + int32_t count = 0; + + // loop through the pattern string character by character + int32_t adjustedPtnLength = adjustedPtn.length(); + int32_t i; + for (i = 0; i < adjustedPtnLength; ++i) { + char16_t ch = adjustedPtn.charAt(i); + if (ch != prevCh && count > 0) { + // check the repeativeness of pattern letter + char16_t skeletonChar = prevCh; + if ( skeletonChar == CAP_L ) { + // there is no "L" (always be "M") in skeleton, + // but there is "L" in pattern. + // for skeleton "M+", the pattern might be "...L..." + skeletonChar = CAP_M; + } + int32_t fieldCount = bestMatchSkeletonFieldWidth[(int)(skeletonChar - PATTERN_CHAR_BASE)]; + int32_t inputFieldCount = inputSkeletonFieldWidth[(int)(skeletonChar - PATTERN_CHAR_BASE)]; + if ( fieldCount == count && inputFieldCount > fieldCount ) { + count = inputFieldCount - fieldCount; + int32_t j; + for ( j = 0; j < count; ++j ) { + adjustedPtn.insert(i, prevCh); + } + i += count; + adjustedPtnLength += count; + } + count = 0; + } + if (ch == 0x0027 /*'*/) { + // Consecutive single quotes are a single quote literal, + // either outside of quotes or between quotes + if ((i+1) < adjustedPtn.length() && adjustedPtn.charAt(i+1) == 0x0027 /* ' */) { + ++i; + } else { + inQuote = ! inQuote; + } + } + else if ( ! inQuote && ((ch >= 0x0061 /*'a'*/ && ch <= 0x007A /*'z'*/) + || (ch >= 0x0041 /*'A'*/ && ch <= 0x005A /*'Z'*/))) { + // ch is a date-time pattern character + prevCh = ch; + ++count; + } + } + if ( count > 0 ) { + // last item + // check the repeativeness of pattern letter + char16_t skeletonChar = prevCh; + if ( skeletonChar == CAP_L ) { + // there is no "L" (always be "M") in skeleton, + // but there is "L" in pattern. + // for skeleton "M+", the pattern might be "...L..." + skeletonChar = CAP_M; + } + int32_t fieldCount = bestMatchSkeletonFieldWidth[(int)(skeletonChar - PATTERN_CHAR_BASE)]; + int32_t inputFieldCount = inputSkeletonFieldWidth[(int)(skeletonChar - PATTERN_CHAR_BASE)]; + if ( fieldCount == count && inputFieldCount > fieldCount ) { + count = inputFieldCount - fieldCount; + int32_t j; + for ( j = 0; j < count; ++j ) { + adjustedPtn.append(prevCh); + } + } + } +} + +void +DateIntervalFormat::findReplaceInPattern(UnicodeString& targetString, + const UnicodeString& strToReplace, + const UnicodeString& strToReplaceWith) { + int32_t firstQuoteIndex = targetString.indexOf(u'\''); + if (firstQuoteIndex == -1) { + targetString.findAndReplace(strToReplace, strToReplaceWith); + } else { + UnicodeString result; + UnicodeString source = targetString; + + while (firstQuoteIndex >= 0) { + int32_t secondQuoteIndex = source.indexOf(u'\'', firstQuoteIndex + 1); + if (secondQuoteIndex == -1) { + secondQuoteIndex = source.length() - 1; + } + + UnicodeString unquotedText(source, 0, firstQuoteIndex); + UnicodeString quotedText(source, firstQuoteIndex, secondQuoteIndex - firstQuoteIndex + 1); + + unquotedText.findAndReplace(strToReplace, strToReplaceWith); + result += unquotedText; + result += quotedText; + + source.remove(0, secondQuoteIndex + 1); + firstQuoteIndex = source.indexOf(u'\''); + } + source.findAndReplace(strToReplace, strToReplaceWith); + result += source; + targetString = result; + } +} + + + +void +DateIntervalFormat::concatSingleDate2TimeInterval(UnicodeString& format, + const UnicodeString& datePattern, + UCalendarDateFields field, + UErrorCode& status) { + // following should not set wrong status + int32_t itvPtnIndex = DateIntervalInfo::calendarFieldToIntervalIndex(field, + status); + if ( U_FAILURE(status) ) { + return; + } + PatternInfo& timeItvPtnInfo = fIntervalPatterns[itvPtnIndex]; + if ( !timeItvPtnInfo.firstPart.isEmpty() ) { + UnicodeString timeIntervalPattern(timeItvPtnInfo.firstPart); + timeIntervalPattern.append(timeItvPtnInfo.secondPart); + UnicodeString combinedPattern; + SimpleFormatter(format, 2, 2, status). + format(timeIntervalPattern, datePattern, combinedPattern, status); + if ( U_FAILURE(status) ) { + return; + } + setIntervalPattern(field, combinedPattern, timeItvPtnInfo.laterDateFirst); + } + // else: fall back + // it should not happen if the interval format defined is valid +} + + + +const char16_t +DateIntervalFormat::fgCalendarFieldToPatternLetter[] = +{ + /*GyM*/ CAP_G, LOW_Y, CAP_M, + /*wWd*/ LOW_W, CAP_W, LOW_D, + /*DEF*/ CAP_D, CAP_E, CAP_F, + /*ahH*/ LOW_A, LOW_H, CAP_H, + /*msS*/ LOW_M, LOW_S, CAP_S, // MINUTE, SECOND, MILLISECOND + /*z.Y*/ LOW_Z, SPACE, CAP_Y, // ZONE_OFFSET, DST_OFFSET, YEAR_WOY, + /*eug*/ LOW_E, LOW_U, LOW_G, // DOW_LOCAL, EXTENDED_YEAR, JULIAN_DAY, + /*A..*/ CAP_A, SPACE, SPACE, // MILLISECONDS_IN_DAY, IS_LEAP_MONTH, FIELD_COUNT +}; + + + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/dtitvinf.cpp b/intl/icu/source/i18n/dtitvinf.cpp new file mode 100644 index 0000000000..3733d04518 --- /dev/null +++ b/intl/icu/source/i18n/dtitvinf.cpp @@ -0,0 +1,814 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/******************************************************************************* +* Copyright (C) 2008-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DTITVINF.CPP +* +******************************************************************************* +*/ + +#include "unicode/dtitvinf.h" + + +#if !UCONFIG_NO_FORMATTING + +//TODO: define it in compiler time +//#define DTITVINF_DEBUG 1 + + +#ifdef DTITVINF_DEBUG +#include +#endif + +#include "cmemory.h" +#include "cstring.h" +#include "unicode/msgfmt.h" +#include "unicode/uloc.h" +#include "unicode/ures.h" +#include "dtitv_impl.h" +#include "charstr.h" +#include "hash.h" +#include "gregoimp.h" +#include "uresimp.h" +#include "hash.h" +#include "gregoimp.h" +#include "uresimp.h" + + +U_NAMESPACE_BEGIN + + +#ifdef DTITVINF_DEBUG +#define PRINTMESG(msg) UPRV_BLOCK_MACRO_BEGIN { \ + std::cout << "(" << __FILE__ << ":" << __LINE__ << ") " << msg << "\n"; \ +} UPRV_BLOCK_MACRO_END +#endif + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DateIntervalInfo) + +static const char gCalendarTag[]="calendar"; +static const char gGregorianTag[]="gregorian"; +static const char gIntervalDateTimePatternTag[]="intervalFormats"; +static const char gFallbackPatternTag[]="fallback"; + +// {0} +static const char16_t gFirstPattern[] = {LEFT_CURLY_BRACKET, DIGIT_ZERO, RIGHT_CURLY_BRACKET}; +// {1} +static const char16_t gSecondPattern[] = {LEFT_CURLY_BRACKET, DIGIT_ONE, RIGHT_CURLY_BRACKET}; + +// default fall-back +static const char16_t gDefaultFallbackPattern[] = {LEFT_CURLY_BRACKET, DIGIT_ZERO, RIGHT_CURLY_BRACKET, SPACE, EN_DASH, SPACE, LEFT_CURLY_BRACKET, DIGIT_ONE, RIGHT_CURLY_BRACKET, 0}; + +DateIntervalInfo::DateIntervalInfo(UErrorCode& status) +: fFallbackIntervalPattern(gDefaultFallbackPattern), + fFirstDateInPtnIsLaterDate(false), + fIntervalPatterns(nullptr) +{ + fIntervalPatterns = initHash(status); +} + + + +DateIntervalInfo::DateIntervalInfo(const Locale& locale, UErrorCode& status) +: fFallbackIntervalPattern(gDefaultFallbackPattern), + fFirstDateInPtnIsLaterDate(false), + fIntervalPatterns(nullptr) +{ + initializeData(locale, status); +} + + + +void +DateIntervalInfo::setIntervalPattern(const UnicodeString& skeleton, + UCalendarDateFields lrgDiffCalUnit, + const UnicodeString& intervalPattern, + UErrorCode& status) { + + if ( lrgDiffCalUnit == UCAL_HOUR_OF_DAY ) { + setIntervalPatternInternally(skeleton, UCAL_AM_PM, intervalPattern, status); + setIntervalPatternInternally(skeleton, UCAL_HOUR, intervalPattern, status); + } else if ( lrgDiffCalUnit == UCAL_DAY_OF_MONTH || + lrgDiffCalUnit == UCAL_DAY_OF_WEEK ) { + setIntervalPatternInternally(skeleton, UCAL_DATE, intervalPattern, status); + } else { + setIntervalPatternInternally(skeleton, lrgDiffCalUnit, intervalPattern, status); + } +} + + +void +DateIntervalInfo::setFallbackIntervalPattern( + const UnicodeString& fallbackPattern, + UErrorCode& status) { + if ( U_FAILURE(status) ) { + return; + } + int32_t firstPatternIndex = fallbackPattern.indexOf(gFirstPattern, + UPRV_LENGTHOF(gFirstPattern), 0); + int32_t secondPatternIndex = fallbackPattern.indexOf(gSecondPattern, + UPRV_LENGTHOF(gSecondPattern), 0); + if ( firstPatternIndex == -1 || secondPatternIndex == -1 ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if ( firstPatternIndex > secondPatternIndex ) { + fFirstDateInPtnIsLaterDate = true; + } + fFallbackIntervalPattern = fallbackPattern; +} + + + +DateIntervalInfo::DateIntervalInfo(const DateIntervalInfo& dtitvinf) +: UObject(dtitvinf), + fIntervalPatterns(nullptr) +{ + *this = dtitvinf; +} + + + +DateIntervalInfo& +DateIntervalInfo::operator=(const DateIntervalInfo& dtitvinf) { + if ( this == &dtitvinf ) { + return *this; + } + + UErrorCode status = U_ZERO_ERROR; + deleteHash(fIntervalPatterns); + fIntervalPatterns = initHash(status); + copyHash(dtitvinf.fIntervalPatterns, fIntervalPatterns, status); + if ( U_FAILURE(status) ) { + return *this; + } + + fFallbackIntervalPattern = dtitvinf.fFallbackIntervalPattern; + fFirstDateInPtnIsLaterDate = dtitvinf.fFirstDateInPtnIsLaterDate; + return *this; +} + + +DateIntervalInfo* +DateIntervalInfo::clone() const { + return new DateIntervalInfo(*this); +} + + +DateIntervalInfo::~DateIntervalInfo() { + deleteHash(fIntervalPatterns); + fIntervalPatterns = nullptr; +} + + +bool +DateIntervalInfo::operator==(const DateIntervalInfo& other) const { + bool equal = ( + fFallbackIntervalPattern == other.fFallbackIntervalPattern && + fFirstDateInPtnIsLaterDate == other.fFirstDateInPtnIsLaterDate ); + + if ( equal ) { + equal = fIntervalPatterns->equals(*(other.fIntervalPatterns)); + } + + return equal; +} + + +UnicodeString& +DateIntervalInfo::getIntervalPattern(const UnicodeString& skeleton, + UCalendarDateFields field, + UnicodeString& result, + UErrorCode& status) const { + if ( U_FAILURE(status) ) { + return result; + } + + const UnicodeString* patternsOfOneSkeleton = (UnicodeString*) fIntervalPatterns->get(skeleton); + if ( patternsOfOneSkeleton != nullptr ) { + IntervalPatternIndex index = calendarFieldToIntervalIndex(field, status); + if ( U_FAILURE(status) ) { + return result; + } + const UnicodeString& intervalPattern = patternsOfOneSkeleton[index]; + if ( !intervalPattern.isEmpty() ) { + result = intervalPattern; + } + } + return result; +} + + +UBool +DateIntervalInfo::getDefaultOrder() const { + return fFirstDateInPtnIsLaterDate; +} + + +UnicodeString& +DateIntervalInfo::getFallbackIntervalPattern(UnicodeString& result) const { + result = fFallbackIntervalPattern; + return result; +} + +#define ULOC_LOCALE_IDENTIFIER_CAPACITY (ULOC_FULLNAME_CAPACITY + 1 + ULOC_KEYWORD_AND_VALUES_CAPACITY) + + +static const int32_t PATH_PREFIX_LENGTH = 17; +static const char16_t PATH_PREFIX[] = {SOLIDUS, CAP_L, CAP_O, CAP_C, CAP_A, CAP_L, CAP_E, SOLIDUS, + LOW_C, LOW_A, LOW_L, LOW_E, LOW_N, LOW_D, LOW_A, LOW_R, SOLIDUS}; +static const int32_t PATH_SUFFIX_LENGTH = 16; +static const char16_t PATH_SUFFIX[] = {SOLIDUS, LOW_I, LOW_N, LOW_T, LOW_E, LOW_R, LOW_V, LOW_A, + LOW_L, CAP_F, LOW_O, LOW_R, LOW_M, LOW_A, LOW_T, LOW_S}; + +/** + * Sink for enumerating all of the date interval skeletons. + */ +struct DateIntervalInfo::DateIntervalSink : public ResourceSink { + + // Output data + DateIntervalInfo &dateIntervalInfo; + + // Next calendar type + UnicodeString nextCalendarType; + + DateIntervalSink(DateIntervalInfo &diInfo, const char *currentCalendarType) + : dateIntervalInfo(diInfo), nextCalendarType(currentCalendarType, -1, US_INV) { } + virtual ~DateIntervalSink(); + + virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/, UErrorCode &errorCode) override { + if (U_FAILURE(errorCode)) { return; } + + // Iterate over all the calendar entries and only pick the 'intervalFormats' table. + ResourceTable dateIntervalData = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + for (int32_t i = 0; dateIntervalData.getKeyAndValue(i, key, value); i++) { + if (uprv_strcmp(key, gIntervalDateTimePatternTag) != 0) { + continue; + } + + // Handle aliases and tables. Ignore the rest. + if (value.getType() == URES_ALIAS) { + // Get the calendar type for the alias path. + const UnicodeString &aliasPath = value.getAliasUnicodeString(errorCode); + if (U_FAILURE(errorCode)) { return; } + + nextCalendarType.remove(); + getCalendarTypeFromPath(aliasPath, nextCalendarType, errorCode); + + if (U_FAILURE(errorCode)) { + resetNextCalendarType(); + } + break; + + } else if (value.getType() == URES_TABLE) { + // Iterate over all the skeletons in the 'intervalFormat' table. + ResourceTable skeletonData = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + for (int32_t j = 0; skeletonData.getKeyAndValue(j, key, value); j++) { + if (value.getType() == URES_TABLE) { + // Process the skeleton + processSkeletonTable(key, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } + break; + } + } + } + + /** + * Processes the patterns for a skeleton table + */ + void processSkeletonTable(const char *key, ResourceValue &value, UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return; } + + // Iterate over all the patterns in the current skeleton table + const char *currentSkeleton = key; + ResourceTable patternData = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + for (int32_t k = 0; patternData.getKeyAndValue(k, key, value); k++) { + if (value.getType() == URES_STRING) { + // Process the key + UCalendarDateFields calendarField = validateAndProcessPatternLetter(key); + + // If the calendar field has a valid value + if (calendarField < UCAL_FIELD_COUNT) { + // Set the interval pattern + setIntervalPatternIfAbsent(currentSkeleton, calendarField, value, errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } + } + } + + /** + * Extracts the calendar type from the path. + */ + static void getCalendarTypeFromPath(const UnicodeString &path, UnicodeString &calendarType, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { return; } + + if (!path.startsWith(PATH_PREFIX, PATH_PREFIX_LENGTH) || !path.endsWith(PATH_SUFFIX, PATH_SUFFIX_LENGTH)) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + + path.extractBetween(PATH_PREFIX_LENGTH, path.length() - PATH_SUFFIX_LENGTH, calendarType); + } + + /** + * Validates and processes the pattern letter + */ + UCalendarDateFields validateAndProcessPatternLetter(const char *patternLetter) { + // Check that patternLetter is just one letter + char c0; + if ((c0 = patternLetter[0]) != 0 && patternLetter[1] == 0) { + // Check that the pattern letter is accepted + if (c0 == 'G') { + return UCAL_ERA; + } else if (c0 == 'y') { + return UCAL_YEAR; + } else if (c0 == 'M') { + return UCAL_MONTH; + } else if (c0 == 'd') { + return UCAL_DATE; + } else if (c0 == 'a') { + return UCAL_AM_PM; + } else if (c0 == 'B') { + // TODO: Using AM/PM as a proxy for flexible day period isn't really correct, but it's close + return UCAL_AM_PM; + } else if (c0 == 'h' || c0 == 'H') { + return UCAL_HOUR; + } else if (c0 == 'm') { + return UCAL_MINUTE; + }// TODO(ticket:12190): Why icu4c doesn't accept the calendar field "s" but icu4j does? + } + return UCAL_FIELD_COUNT; + } + + /** + * Stores the interval pattern for the current skeleton in the internal data structure + * if it's not present. + */ + void setIntervalPatternIfAbsent(const char *currentSkeleton, UCalendarDateFields lrgDiffCalUnit, + const ResourceValue &value, UErrorCode &errorCode) { + // Check if the pattern has already been stored on the data structure + IntervalPatternIndex index = + dateIntervalInfo.calendarFieldToIntervalIndex(lrgDiffCalUnit, errorCode); + if (U_FAILURE(errorCode)) { return; } + + UnicodeString skeleton(currentSkeleton, -1, US_INV); + UnicodeString* patternsOfOneSkeleton = + (UnicodeString*)(dateIntervalInfo.fIntervalPatterns->get(skeleton)); + + if (patternsOfOneSkeleton == nullptr || patternsOfOneSkeleton[index].isEmpty()) { + UnicodeString pattern = value.getUnicodeString(errorCode); + dateIntervalInfo.setIntervalPatternInternally(skeleton, lrgDiffCalUnit, + pattern, errorCode); + } + } + + const UnicodeString &getNextCalendarType() { + return nextCalendarType; + } + + void resetNextCalendarType() { + nextCalendarType.setToBogus(); + } +}; + +// Virtual destructors must be defined out of line. +DateIntervalInfo::DateIntervalSink::~DateIntervalSink() {} + + + +void +DateIntervalInfo::initializeData(const Locale& locale, UErrorCode& status) +{ + fIntervalPatterns = initHash(status); + if (U_FAILURE(status)) { + return; + } + const char *locName = locale.getName(); + + // Get the correct calendar type + const char * calendarTypeToUse = gGregorianTag; // initial default + char calendarType[ULOC_KEYWORDS_CAPACITY]; // to be filled in with the type to use, if all goes well + char localeWithCalendarKey[ULOC_LOCALE_IDENTIFIER_CAPACITY]; + // obtain a locale that always has the calendar key value that should be used + (void)ures_getFunctionalEquivalent(localeWithCalendarKey, ULOC_LOCALE_IDENTIFIER_CAPACITY, nullptr, + "calendar", "calendar", locName, nullptr, false, &status); + localeWithCalendarKey[ULOC_LOCALE_IDENTIFIER_CAPACITY-1] = 0; // ensure null termination + // now get the calendar key value from that locale + int32_t calendarTypeLen = uloc_getKeywordValue(localeWithCalendarKey, "calendar", calendarType, + ULOC_KEYWORDS_CAPACITY, &status); + if (U_SUCCESS(status) && calendarTypeLen < ULOC_KEYWORDS_CAPACITY) { + calendarTypeToUse = calendarType; + } + status = U_ZERO_ERROR; + + // Instantiate the resource bundles + UResourceBundle *rb, *calBundle; + rb = ures_open(nullptr, locName, &status); + if (U_FAILURE(status)) { + return; + } + calBundle = ures_getByKeyWithFallback(rb, gCalendarTag, nullptr, &status); + + + if (U_SUCCESS(status)) { + UResourceBundle *calTypeBundle, *itvDtPtnResource; + + // Get the fallback pattern + const char16_t* resStr = nullptr; + int32_t resStrLen = 0; + calTypeBundle = ures_getByKeyWithFallback(calBundle, calendarTypeToUse, nullptr, &status); + itvDtPtnResource = ures_getByKeyWithFallback(calTypeBundle, + gIntervalDateTimePatternTag, nullptr, &status); + // TODO(ICU-20400): After the fixing, we should find the "fallback" from + // the rb directly by the path "calendar/${calendar}/intervalFormats/fallback". + if ( U_SUCCESS(status) ) { + resStr = ures_getStringByKeyWithFallback(itvDtPtnResource, gFallbackPatternTag, + &resStrLen, &status); + } + + if ( U_SUCCESS(status) && (resStr != nullptr)) { + UnicodeString pattern = UnicodeString(true, resStr, resStrLen); + setFallbackIntervalPattern(pattern, status); + } + ures_close(itvDtPtnResource); + ures_close(calTypeBundle); + + + // Instantiate the sink + DateIntervalSink sink(*this, calendarTypeToUse); + const UnicodeString &calendarTypeToUseUString = sink.getNextCalendarType(); + + // Already loaded calendar types + Hashtable loadedCalendarTypes(false, status); + + if (U_SUCCESS(status)) { + while (!calendarTypeToUseUString.isBogus()) { + // Set an error when a loop is detected + if (loadedCalendarTypes.geti(calendarTypeToUseUString) == 1) { + status = U_INVALID_FORMAT_ERROR; + break; + } + + // Register the calendar type to avoid loops + loadedCalendarTypes.puti(calendarTypeToUseUString, 1, status); + if (U_FAILURE(status)) { break; } + + // Get the calendar string + CharString calTypeBuffer; + calTypeBuffer.appendInvariantChars(calendarTypeToUseUString, status); + if (U_FAILURE(status)) { break; } + const char *calType = calTypeBuffer.data(); + + // Reset the next calendar type to load. + sink.resetNextCalendarType(); + + // Get all resources for this calendar type + ures_getAllItemsWithFallback(calBundle, calType, sink, status); + } + } + } + + // Close the opened resource bundles + ures_close(calBundle); + ures_close(rb); +} + +void +DateIntervalInfo::setIntervalPatternInternally(const UnicodeString& skeleton, + UCalendarDateFields lrgDiffCalUnit, + const UnicodeString& intervalPattern, + UErrorCode& status) { + IntervalPatternIndex index = calendarFieldToIntervalIndex(lrgDiffCalUnit,status); + if ( U_FAILURE(status) ) { + return; + } + UnicodeString* patternsOfOneSkeleton = (UnicodeString*)(fIntervalPatterns->get(skeleton)); + UBool emptyHash = false; + if ( patternsOfOneSkeleton == nullptr ) { + patternsOfOneSkeleton = new UnicodeString[kIPI_MAX_INDEX]; + if (patternsOfOneSkeleton == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + emptyHash = true; + } + + patternsOfOneSkeleton[index] = intervalPattern; + if ( emptyHash ) { + fIntervalPatterns->put(skeleton, patternsOfOneSkeleton, status); + } +} + + + +void +DateIntervalInfo::parseSkeleton(const UnicodeString& skeleton, + int32_t* skeletonFieldWidth) { + const int8_t PATTERN_CHAR_BASE = 0x41; + int32_t i; + for ( i = 0; i < skeleton.length(); ++i ) { + // it is an ASCII char in skeleton + int8_t ch = (int8_t)skeleton.charAt(i); + ++skeletonFieldWidth[ch - PATTERN_CHAR_BASE]; + } +} + + + +UBool +DateIntervalInfo::stringNumeric(int32_t fieldWidth, int32_t anotherFieldWidth, + char patternLetter) { + if ( patternLetter == 'M' ) { + if ( (fieldWidth <= 2 && anotherFieldWidth > 2) || + (fieldWidth > 2 && anotherFieldWidth <= 2 )) { + return true; + } + } + return false; +} + + + +const UnicodeString* +DateIntervalInfo::getBestSkeleton(const UnicodeString& skeleton, + int8_t& bestMatchDistanceInfo) const { +#ifdef DTITVINF_DEBUG + char result[1000]; + char result_1[1000]; + char mesg[2000]; + skeleton.extract(0, skeleton.length(), result, "UTF-8"); + snprintf(mesg, sizeof(mesg), "in getBestSkeleton: skeleton: %s; \n", result); + PRINTMESG(mesg) +#endif + + + int32_t inputSkeletonFieldWidth[] = + { + // A B C D E F G H I J K L M N O + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // P Q R S T U V W X Y Z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // a b c d e f g h i j k l m n o + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // p q r s t u v w x y z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + int32_t skeletonFieldWidth[] = + { + // A B C D E F G H I J K L M N O + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // P Q R S T U V W X Y Z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // a b c d e f g h i j k l m n o + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // p q r s t u v w x y z + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + const int32_t DIFFERENT_FIELD = 0x1000; + const int32_t STRING_NUMERIC_DIFFERENCE = 0x100; + const int32_t BASE = 0x41; + + // hack for certain alternate characters + // resource bundles only have time skeletons containing 'v', 'h', and 'H' + // but not time skeletons containing 'z', 'K', or 'k' + // the skeleton may also include 'a' or 'b', which never occur in the resource bundles, so strip them out too + UBool replacedAlternateChars = false; + const UnicodeString* inputSkeleton = &skeleton; + UnicodeString copySkeleton; + if ( skeleton.indexOf(LOW_Z) != -1 || skeleton.indexOf(LOW_K) != -1 || skeleton.indexOf(CAP_K) != -1 || skeleton.indexOf(LOW_A) != -1 || skeleton.indexOf(LOW_B) != -1 ) { + copySkeleton = skeleton; + copySkeleton.findAndReplace(UnicodeString(LOW_Z), UnicodeString(LOW_V)); + copySkeleton.findAndReplace(UnicodeString(LOW_K), UnicodeString(CAP_H)); + copySkeleton.findAndReplace(UnicodeString(CAP_K), UnicodeString(LOW_H)); + copySkeleton.findAndReplace(UnicodeString(LOW_A), UnicodeString()); + copySkeleton.findAndReplace(UnicodeString(LOW_B), UnicodeString()); + inputSkeleton = ©Skeleton; + replacedAlternateChars = true; + } + + parseSkeleton(*inputSkeleton, inputSkeletonFieldWidth); + int32_t bestDistance = MAX_POSITIVE_INT; + const UnicodeString* bestSkeleton = nullptr; + + // 0 means exact the same skeletons; + // 1 means having the same field, but with different length, + // 2 means only z/v, h/K, or H/k differs + // -1 means having different field. + bestMatchDistanceInfo = 0; + int8_t fieldLength = UPRV_LENGTHOF(skeletonFieldWidth); + + int32_t pos = UHASH_FIRST; + const UHashElement* elem = nullptr; + while ( (elem = fIntervalPatterns->nextElement(pos)) != nullptr ) { + const UHashTok keyTok = elem->key; + UnicodeString* newSkeleton = (UnicodeString*)keyTok.pointer; +#ifdef DTITVINF_DEBUG + skeleton->extract(0, skeleton->length(), result, "UTF-8"); + snprintf(mesg, sizeof(mesg), "available skeletons: skeleton: %s; \n", result); + PRINTMESG(mesg) +#endif + + // clear skeleton field width + int8_t i; + for ( i = 0; i < fieldLength; ++i ) { + skeletonFieldWidth[i] = 0; + } + parseSkeleton(*newSkeleton, skeletonFieldWidth); + // calculate distance + int32_t distance = 0; + int8_t fieldDifference = 1; + for ( i = 0; i < fieldLength; ++i ) { + int32_t inputFieldWidth = inputSkeletonFieldWidth[i]; + int32_t fieldWidth = skeletonFieldWidth[i]; + if ( inputFieldWidth == fieldWidth ) { + continue; + } + if ( inputFieldWidth == 0 ) { + fieldDifference = -1; + distance += DIFFERENT_FIELD; + } else if ( fieldWidth == 0 ) { + fieldDifference = -1; + distance += DIFFERENT_FIELD; + } else if (stringNumeric(inputFieldWidth, fieldWidth, + (char)(i+BASE) ) ) { + distance += STRING_NUMERIC_DIFFERENCE; + } else { + distance += (inputFieldWidth > fieldWidth) ? + (inputFieldWidth - fieldWidth) : + (fieldWidth - inputFieldWidth); + } + } + if ( distance < bestDistance ) { + bestSkeleton = newSkeleton; + bestDistance = distance; + bestMatchDistanceInfo = fieldDifference; + } + if ( distance == 0 ) { + bestMatchDistanceInfo = 0; + break; + } + } + if ( replacedAlternateChars && bestMatchDistanceInfo != -1 ) { + bestMatchDistanceInfo = 2; + } + return bestSkeleton; +} + + + +DateIntervalInfo::IntervalPatternIndex +DateIntervalInfo::calendarFieldToIntervalIndex(UCalendarDateFields field, + UErrorCode& status) { + if ( U_FAILURE(status) ) { + return kIPI_MAX_INDEX; + } + IntervalPatternIndex index = kIPI_MAX_INDEX; + switch ( field ) { + case UCAL_ERA: + index = kIPI_ERA; + break; + case UCAL_YEAR: + index = kIPI_YEAR; + break; + case UCAL_MONTH: + index = kIPI_MONTH; + break; + case UCAL_DATE: + case UCAL_DAY_OF_WEEK: + //case UCAL_DAY_OF_MONTH: + index = kIPI_DATE; + break; + case UCAL_AM_PM: + index = kIPI_AM_PM; + break; + case UCAL_HOUR: + case UCAL_HOUR_OF_DAY: + index = kIPI_HOUR; + break; + case UCAL_MINUTE: + index = kIPI_MINUTE; + break; + case UCAL_SECOND: + index = kIPI_SECOND; + break; + case UCAL_MILLISECOND: + index = kIPI_MILLISECOND; + break; + default: + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return index; +} + + + +void +DateIntervalInfo::deleteHash(Hashtable* hTable) +{ + if ( hTable == nullptr ) { + return; + } + int32_t pos = UHASH_FIRST; + const UHashElement* element = nullptr; + while ( (element = hTable->nextElement(pos)) != nullptr ) { + const UHashTok valueTok = element->value; + const UnicodeString* value = (UnicodeString*)valueTok.pointer; + delete[] value; + } + delete fIntervalPatterns; +} + + +U_CDECL_BEGIN + +/** + * set hash table value comparator + * + * @param val1 one value in comparison + * @param val2 the other value in comparison + * @return true if 2 values are the same, false otherwise + */ +static UBool U_CALLCONV dtitvinfHashTableValueComparator(UHashTok val1, UHashTok val2); + +static UBool +U_CALLCONV dtitvinfHashTableValueComparator(UHashTok val1, UHashTok val2) { + const UnicodeString* pattern1 = (UnicodeString*)val1.pointer; + const UnicodeString* pattern2 = (UnicodeString*)val2.pointer; + UBool ret = true; + int8_t i; + for ( i = 0; i < DateIntervalInfo::kMaxIntervalPatternIndex && ret ; ++i ) { + ret = (pattern1[i] == pattern2[i]); + } + return ret; +} + +U_CDECL_END + + +Hashtable* +DateIntervalInfo::initHash(UErrorCode& status) { + if ( U_FAILURE(status) ) { + return nullptr; + } + Hashtable* hTable; + if ( (hTable = new Hashtable(false, status)) == nullptr ) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if ( U_FAILURE(status) ) { + delete hTable; + return nullptr; + } + hTable->setValueComparator(dtitvinfHashTableValueComparator); + return hTable; +} + + +void +DateIntervalInfo::copyHash(const Hashtable* source, + Hashtable* target, + UErrorCode& status) { + if ( U_FAILURE(status) ) { + return; + } + int32_t pos = UHASH_FIRST; + const UHashElement* element = nullptr; + if ( source ) { + while ( (element = source->nextElement(pos)) != nullptr ) { + const UHashTok keyTok = element->key; + const UnicodeString* key = (UnicodeString*)keyTok.pointer; + const UHashTok valueTok = element->value; + const UnicodeString* value = (UnicodeString*)valueTok.pointer; + UnicodeString* copy = new UnicodeString[kIPI_MAX_INDEX]; + if (copy == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + int8_t i; + for ( i = 0; i < kIPI_MAX_INDEX; ++i ) { + copy[i] = value[i]; + } + target->put(UnicodeString(*key), copy, status); + if ( U_FAILURE(status) ) { + return; + } + } + } +} + + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/dtptngen.cpp b/intl/icu/source/i18n/dtptngen.cpp new file mode 100644 index 0000000000..3425046763 --- /dev/null +++ b/intl/icu/source/i18n/dtptngen.cpp @@ -0,0 +1,3020 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DTPTNGEN.CPP +* +******************************************************************************* +*/ + +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include "unicode/datefmt.h" +#include "unicode/decimfmt.h" +#include "unicode/dtfmtsym.h" +#include "unicode/dtptngen.h" +#include "unicode/localpointer.h" +#include "unicode/simpleformatter.h" +#include "unicode/smpdtfmt.h" +#include "unicode/udat.h" +#include "unicode/udatpg.h" +#include "unicode/uniset.h" +#include "unicode/uloc.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "unicode/rep.h" +#include "unicode/region.h" +#include "cpputils.h" +#include "mutex.h" +#include "umutex.h" +#include "cmemory.h" +#include "cstring.h" +#include "locbased.h" +#include "hash.h" +#include "uhash.h" +#include "uresimp.h" +#include "dtptngen_impl.h" +#include "ucln_in.h" +#include "charstr.h" +#include "uassert.h" + +#if U_CHARSET_FAMILY==U_EBCDIC_FAMILY +/** + * If we are on EBCDIC, use an iterator which will + * traverse the bundles in ASCII order. + */ +#define U_USE_ASCII_BUNDLE_ITERATOR +#define U_SORT_ASCII_BUNDLE_ITERATOR +#endif + +#if defined(U_USE_ASCII_BUNDLE_ITERATOR) + +#include "unicode/ustring.h" +#include "uarrsort.h" + +struct UResAEntry { + char16_t *key; + UResourceBundle *item; +}; + +struct UResourceBundleAIterator { + UResourceBundle *bund; + UResAEntry *entries; + int32_t num; + int32_t cursor; +}; + +/* Must be C linkage to pass function pointer to the sort function */ + +U_CDECL_BEGIN + +static int32_t U_CALLCONV +ures_a_codepointSort(const void *context, const void *left, const void *right) { + //CompareContext *cmp=(CompareContext *)context; + return u_strcmp(((const UResAEntry *)left)->key, + ((const UResAEntry *)right)->key); +} + +U_CDECL_END + +static void ures_a_open(UResourceBundleAIterator *aiter, UResourceBundle *bund, UErrorCode *status) { + if(U_FAILURE(*status)) { + return; + } + aiter->bund = bund; + aiter->num = ures_getSize(aiter->bund); + aiter->cursor = 0; +#if !defined(U_SORT_ASCII_BUNDLE_ITERATOR) + aiter->entries = nullptr; +#else + aiter->entries = (UResAEntry*)uprv_malloc(sizeof(UResAEntry)*aiter->num); + for(int i=0;inum;i++) { + aiter->entries[i].item = ures_getByIndex(aiter->bund, i, nullptr, status); + const char *akey = ures_getKey(aiter->entries[i].item); + int32_t len = uprv_strlen(akey)+1; + aiter->entries[i].key = (char16_t*)uprv_malloc(len*sizeof(char16_t)); + u_charsToUChars(akey, aiter->entries[i].key, len); + } + uprv_sortArray(aiter->entries, aiter->num, sizeof(UResAEntry), ures_a_codepointSort, nullptr, true, status); +#endif +} + +static void ures_a_close(UResourceBundleAIterator *aiter) { +#if defined(U_SORT_ASCII_BUNDLE_ITERATOR) + for(int i=0;inum;i++) { + uprv_free(aiter->entries[i].key); + ures_close(aiter->entries[i].item); + } +#endif +} + +static const char16_t *ures_a_getNextString(UResourceBundleAIterator *aiter, int32_t *len, const char **key, UErrorCode *err) { +#if !defined(U_SORT_ASCII_BUNDLE_ITERATOR) + return ures_getNextString(aiter->bund, len, key, err); +#else + if(U_FAILURE(*err)) return nullptr; + UResourceBundle *item = aiter->entries[aiter->cursor].item; + const char16_t* ret = ures_getString(item, len, err); + *key = ures_getKey(item); + aiter->cursor++; + return ret; +#endif +} + + +#endif + + +U_NAMESPACE_BEGIN + +// ***************************************************************************** +// class DateTimePatternGenerator +// ***************************************************************************** +static const char16_t Canonical_Items[] = { + // GyQMwWEDFdaHmsSv + CAP_G, LOW_Y, CAP_Q, CAP_M, LOW_W, CAP_W, CAP_E, + CAP_D, CAP_F, LOW_D, LOW_A, // The UDATPG_x_FIELD constants and these fields have a different order than in ICU4J + CAP_H, LOW_M, LOW_S, CAP_S, LOW_V, 0 +}; + +static const dtTypeElem dtTypes[] = { + // patternChar, field, type, minLen, weight + {CAP_G, UDATPG_ERA_FIELD, DT_SHORT, 1, 3,}, + {CAP_G, UDATPG_ERA_FIELD, DT_LONG, 4, 0}, + {CAP_G, UDATPG_ERA_FIELD, DT_NARROW, 5, 0}, + + {LOW_Y, UDATPG_YEAR_FIELD, DT_NUMERIC, 1, 20}, + {CAP_Y, UDATPG_YEAR_FIELD, DT_NUMERIC + DT_DELTA, 1, 20}, + {LOW_U, UDATPG_YEAR_FIELD, DT_NUMERIC + 2*DT_DELTA, 1, 20}, + {LOW_R, UDATPG_YEAR_FIELD, DT_NUMERIC + 3*DT_DELTA, 1, 20}, + {CAP_U, UDATPG_YEAR_FIELD, DT_SHORT, 1, 3}, + {CAP_U, UDATPG_YEAR_FIELD, DT_LONG, 4, 0}, + {CAP_U, UDATPG_YEAR_FIELD, DT_NARROW, 5, 0}, + + {CAP_Q, UDATPG_QUARTER_FIELD, DT_NUMERIC, 1, 2}, + {CAP_Q, UDATPG_QUARTER_FIELD, DT_SHORT, 3, 0}, + {CAP_Q, UDATPG_QUARTER_FIELD, DT_LONG, 4, 0}, + {CAP_Q, UDATPG_QUARTER_FIELD, DT_NARROW, 5, 0}, + {LOW_Q, UDATPG_QUARTER_FIELD, DT_NUMERIC + DT_DELTA, 1, 2}, + {LOW_Q, UDATPG_QUARTER_FIELD, DT_SHORT - DT_DELTA, 3, 0}, + {LOW_Q, UDATPG_QUARTER_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {LOW_Q, UDATPG_QUARTER_FIELD, DT_NARROW - DT_DELTA, 5, 0}, + + {CAP_M, UDATPG_MONTH_FIELD, DT_NUMERIC, 1, 2}, + {CAP_M, UDATPG_MONTH_FIELD, DT_SHORT, 3, 0}, + {CAP_M, UDATPG_MONTH_FIELD, DT_LONG, 4, 0}, + {CAP_M, UDATPG_MONTH_FIELD, DT_NARROW, 5, 0}, + {CAP_L, UDATPG_MONTH_FIELD, DT_NUMERIC + DT_DELTA, 1, 2}, + {CAP_L, UDATPG_MONTH_FIELD, DT_SHORT - DT_DELTA, 3, 0}, + {CAP_L, UDATPG_MONTH_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {CAP_L, UDATPG_MONTH_FIELD, DT_NARROW - DT_DELTA, 5, 0}, + {LOW_L, UDATPG_MONTH_FIELD, DT_NUMERIC + DT_DELTA, 1, 1}, + + {LOW_W, UDATPG_WEEK_OF_YEAR_FIELD, DT_NUMERIC, 1, 2}, + + {CAP_W, UDATPG_WEEK_OF_MONTH_FIELD, DT_NUMERIC, 1, 0}, + + {CAP_E, UDATPG_WEEKDAY_FIELD, DT_SHORT, 1, 3}, + {CAP_E, UDATPG_WEEKDAY_FIELD, DT_LONG, 4, 0}, + {CAP_E, UDATPG_WEEKDAY_FIELD, DT_NARROW, 5, 0}, + {CAP_E, UDATPG_WEEKDAY_FIELD, DT_SHORTER, 6, 0}, + {LOW_C, UDATPG_WEEKDAY_FIELD, DT_NUMERIC + 2*DT_DELTA, 1, 2}, + {LOW_C, UDATPG_WEEKDAY_FIELD, DT_SHORT - 2*DT_DELTA, 3, 0}, + {LOW_C, UDATPG_WEEKDAY_FIELD, DT_LONG - 2*DT_DELTA, 4, 0}, + {LOW_C, UDATPG_WEEKDAY_FIELD, DT_NARROW - 2*DT_DELTA, 5, 0}, + {LOW_C, UDATPG_WEEKDAY_FIELD, DT_SHORTER - 2*DT_DELTA, 6, 0}, + {LOW_E, UDATPG_WEEKDAY_FIELD, DT_NUMERIC + DT_DELTA, 1, 2}, // LOW_E is currently not used in CLDR data, should not be canonical + {LOW_E, UDATPG_WEEKDAY_FIELD, DT_SHORT - DT_DELTA, 3, 0}, + {LOW_E, UDATPG_WEEKDAY_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {LOW_E, UDATPG_WEEKDAY_FIELD, DT_NARROW - DT_DELTA, 5, 0}, + {LOW_E, UDATPG_WEEKDAY_FIELD, DT_SHORTER - DT_DELTA, 6, 0}, + + {LOW_D, UDATPG_DAY_FIELD, DT_NUMERIC, 1, 2}, + {LOW_G, UDATPG_DAY_FIELD, DT_NUMERIC + DT_DELTA, 1, 20}, // really internal use, so we don't care + + {CAP_D, UDATPG_DAY_OF_YEAR_FIELD, DT_NUMERIC, 1, 3}, + + {CAP_F, UDATPG_DAY_OF_WEEK_IN_MONTH_FIELD, DT_NUMERIC, 1, 0}, + + {LOW_A, UDATPG_DAYPERIOD_FIELD, DT_SHORT, 1, 3}, + {LOW_A, UDATPG_DAYPERIOD_FIELD, DT_LONG, 4, 0}, + {LOW_A, UDATPG_DAYPERIOD_FIELD, DT_NARROW, 5, 0}, + {LOW_B, UDATPG_DAYPERIOD_FIELD, DT_SHORT - DT_DELTA, 1, 3}, + {LOW_B, UDATPG_DAYPERIOD_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {LOW_B, UDATPG_DAYPERIOD_FIELD, DT_NARROW - DT_DELTA, 5, 0}, + // b needs to be closer to a than to B, so we make this 3*DT_DELTA + {CAP_B, UDATPG_DAYPERIOD_FIELD, DT_SHORT - 3*DT_DELTA, 1, 3}, + {CAP_B, UDATPG_DAYPERIOD_FIELD, DT_LONG - 3*DT_DELTA, 4, 0}, + {CAP_B, UDATPG_DAYPERIOD_FIELD, DT_NARROW - 3*DT_DELTA, 5, 0}, + + {CAP_H, UDATPG_HOUR_FIELD, DT_NUMERIC + 10*DT_DELTA, 1, 2}, // 24 hour + {LOW_K, UDATPG_HOUR_FIELD, DT_NUMERIC + 11*DT_DELTA, 1, 2}, // 24 hour + {LOW_H, UDATPG_HOUR_FIELD, DT_NUMERIC, 1, 2}, // 12 hour + {CAP_K, UDATPG_HOUR_FIELD, DT_NUMERIC + DT_DELTA, 1, 2}, // 12 hour + // The C code has had versions of the following 3, keep & update. Should not need these, but... + // Without these, certain tests using e.g. staticGetSkeleton fail because j/J in patterns + // get skipped instead of mapped to the right hour chars, for example in + // DateFormatTest::TestPatternFromSkeleton + // IntlTestDateTimePatternGeneratorAPI:: testStaticGetSkeleton + // DateIntervalFormatTest::testTicket11985 + // Need to investigate better handling of jJC replacement e.g. in staticGetSkeleton. + {CAP_J, UDATPG_HOUR_FIELD, DT_NUMERIC + 5*DT_DELTA, 1, 2}, // 12/24 hour no AM/PM + {LOW_J, UDATPG_HOUR_FIELD, DT_NUMERIC + 6*DT_DELTA, 1, 6}, // 12/24 hour + {CAP_C, UDATPG_HOUR_FIELD, DT_NUMERIC + 7*DT_DELTA, 1, 6}, // 12/24 hour with preferred dayPeriods for 12 + + {LOW_M, UDATPG_MINUTE_FIELD, DT_NUMERIC, 1, 2}, + + {LOW_S, UDATPG_SECOND_FIELD, DT_NUMERIC, 1, 2}, + {CAP_A, UDATPG_SECOND_FIELD, DT_NUMERIC + DT_DELTA, 1, 1000}, + + {CAP_S, UDATPG_FRACTIONAL_SECOND_FIELD, DT_NUMERIC, 1, 1000}, + + {LOW_V, UDATPG_ZONE_FIELD, DT_SHORT - 2*DT_DELTA, 1, 0}, + {LOW_V, UDATPG_ZONE_FIELD, DT_LONG - 2*DT_DELTA, 4, 0}, + {LOW_Z, UDATPG_ZONE_FIELD, DT_SHORT, 1, 3}, + {LOW_Z, UDATPG_ZONE_FIELD, DT_LONG, 4, 0}, + {CAP_Z, UDATPG_ZONE_FIELD, DT_NARROW - DT_DELTA, 1, 3}, + {CAP_Z, UDATPG_ZONE_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {CAP_Z, UDATPG_ZONE_FIELD, DT_SHORT - DT_DELTA, 5, 0}, + {CAP_O, UDATPG_ZONE_FIELD, DT_SHORT - DT_DELTA, 1, 0}, + {CAP_O, UDATPG_ZONE_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {CAP_V, UDATPG_ZONE_FIELD, DT_SHORT - DT_DELTA, 1, 0}, + {CAP_V, UDATPG_ZONE_FIELD, DT_LONG - DT_DELTA, 2, 0}, + {CAP_V, UDATPG_ZONE_FIELD, DT_LONG-1 - DT_DELTA, 3, 0}, + {CAP_V, UDATPG_ZONE_FIELD, DT_LONG-2 - DT_DELTA, 4, 0}, + {CAP_X, UDATPG_ZONE_FIELD, DT_NARROW - DT_DELTA, 1, 0}, + {CAP_X, UDATPG_ZONE_FIELD, DT_SHORT - DT_DELTA, 2, 0}, + {CAP_X, UDATPG_ZONE_FIELD, DT_LONG - DT_DELTA, 4, 0}, + {LOW_X, UDATPG_ZONE_FIELD, DT_NARROW - DT_DELTA, 1, 0}, + {LOW_X, UDATPG_ZONE_FIELD, DT_SHORT - DT_DELTA, 2, 0}, + {LOW_X, UDATPG_ZONE_FIELD, DT_LONG - DT_DELTA, 4, 0}, + + {0, UDATPG_FIELD_COUNT, 0, 0, 0} , // last row of dtTypes[] + }; + +static const char* const CLDR_FIELD_APPEND[] = { + "Era", "Year", "Quarter", "Month", "Week", "*", "Day-Of-Week", + "*", "*", "Day", "DayPeriod", // The UDATPG_x_FIELD constants and these fields have a different order than in ICU4J + "Hour", "Minute", "Second", "FractionalSecond", "Timezone" +}; + +static const char* const CLDR_FIELD_NAME[UDATPG_FIELD_COUNT] = { + "era", "year", "quarter", "month", "week", "weekOfMonth", "weekday", + "dayOfYear", "weekdayOfMonth", "day", "dayperiod", // The UDATPG_x_FIELD constants and these fields have a different order than in ICU4J + "hour", "minute", "second", "fractionalSecond", "zone" +}; + +static const char* const CLDR_FIELD_WIDTH[] = { // [UDATPG_WIDTH_COUNT] + "", "-short", "-narrow" +}; + +static constexpr UDateTimePGDisplayWidth UDATPG_WIDTH_APPENDITEM = UDATPG_WIDE; +static constexpr int32_t UDATPG_FIELD_KEY_MAX = 24; // max length of CLDR field tag (type + width) + +// For appendItems +static const char16_t UDATPG_ItemFormat[]= {0x7B, 0x30, 0x7D, 0x20, 0x251C, 0x7B, 0x32, 0x7D, 0x3A, + 0x20, 0x7B, 0x31, 0x7D, 0x2524, 0}; // {0} \u251C{2}: {1}\u2524 + +//static const char16_t repeatedPatterns[6]={CAP_G, CAP_E, LOW_Z, LOW_V, CAP_Q, 0}; // "GEzvQ" + +static const char DT_DateTimePatternsTag[]="DateTimePatterns"; +static const char DT_DateAtTimePatternsTag[]="DateTimePatterns%atTime"; +static const char DT_DateTimeCalendarTag[]="calendar"; +static const char DT_DateTimeGregorianTag[]="gregorian"; +static const char DT_DateTimeAppendItemsTag[]="appendItems"; +static const char DT_DateTimeFieldsTag[]="fields"; +static const char DT_DateTimeAvailableFormatsTag[]="availableFormats"; +//static const UnicodeString repeatedPattern=UnicodeString(repeatedPatterns); + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DateTimePatternGenerator) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DTSkeletonEnumeration) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DTRedundantEnumeration) + +DateTimePatternGenerator* U_EXPORT2 +DateTimePatternGenerator::createInstance(UErrorCode& status) { + return createInstance(Locale::getDefault(), status); +} + +DateTimePatternGenerator* U_EXPORT2 +DateTimePatternGenerator::createInstance(const Locale& locale, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result( + new DateTimePatternGenerator(locale, status), status); + return U_SUCCESS(status) ? result.orphan() : nullptr; +} + +DateTimePatternGenerator* U_EXPORT2 +DateTimePatternGenerator::createInstanceNoStdPat(const Locale& locale, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result( + new DateTimePatternGenerator(locale, status, true), status); + return U_SUCCESS(status) ? result.orphan() : nullptr; +} + +DateTimePatternGenerator* U_EXPORT2 +DateTimePatternGenerator::createEmptyInstance(UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result( + new DateTimePatternGenerator(status), status); + return U_SUCCESS(status) ? result.orphan() : nullptr; +} + +DateTimePatternGenerator::DateTimePatternGenerator(UErrorCode &status) : + skipMatcher(nullptr), + fAvailableFormatKeyHash(nullptr), + fDefaultHourFormatChar(0), + internalErrorCode(U_ZERO_ERROR) +{ + fp = new FormatParser(); + dtMatcher = new DateTimeMatcher(); + distanceInfo = new DistanceInfo(); + patternMap = new PatternMap(); + if (fp == nullptr || dtMatcher == nullptr || distanceInfo == nullptr || patternMap == nullptr) { + internalErrorCode = status = U_MEMORY_ALLOCATION_ERROR; + } +} + +DateTimePatternGenerator::DateTimePatternGenerator(const Locale& locale, UErrorCode &status, UBool skipStdPatterns) : + skipMatcher(nullptr), + fAvailableFormatKeyHash(nullptr), + fDefaultHourFormatChar(0), + internalErrorCode(U_ZERO_ERROR) +{ + fp = new FormatParser(); + dtMatcher = new DateTimeMatcher(); + distanceInfo = new DistanceInfo(); + patternMap = new PatternMap(); + if (fp == nullptr || dtMatcher == nullptr || distanceInfo == nullptr || patternMap == nullptr) { + internalErrorCode = status = U_MEMORY_ALLOCATION_ERROR; + } + else { + initData(locale, status, skipStdPatterns); + } +} + +DateTimePatternGenerator::DateTimePatternGenerator(const DateTimePatternGenerator& other) : + UObject(), + skipMatcher(nullptr), + fAvailableFormatKeyHash(nullptr), + fDefaultHourFormatChar(0), + internalErrorCode(U_ZERO_ERROR) +{ + fp = new FormatParser(); + dtMatcher = new DateTimeMatcher(); + distanceInfo = new DistanceInfo(); + patternMap = new PatternMap(); + if (fp == nullptr || dtMatcher == nullptr || distanceInfo == nullptr || patternMap == nullptr) { + internalErrorCode = U_MEMORY_ALLOCATION_ERROR; + } + *this=other; +} + +DateTimePatternGenerator& +DateTimePatternGenerator::operator=(const DateTimePatternGenerator& other) { + // reflexive case + if (&other == this) { + return *this; + } + internalErrorCode = other.internalErrorCode; + pLocale = other.pLocale; + fDefaultHourFormatChar = other.fDefaultHourFormatChar; + *fp = *(other.fp); + dtMatcher->copyFrom(other.dtMatcher->skeleton); + *distanceInfo = *(other.distanceInfo); + for (int32_t style = UDAT_FULL; style <= UDAT_SHORT; style++) { + dateTimeFormat[style] = other.dateTimeFormat[style]; + } + decimal = other.decimal; + for (int32_t style = UDAT_FULL; style <= UDAT_SHORT; style++) { + dateTimeFormat[style].getTerminatedBuffer(); // NUL-terminate for the C API. + } + decimal.getTerminatedBuffer(); + delete skipMatcher; + if ( other.skipMatcher == nullptr ) { + skipMatcher = nullptr; + } + else { + skipMatcher = new DateTimeMatcher(*other.skipMatcher); + if (skipMatcher == nullptr) + { + internalErrorCode = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + } + for (int32_t i=0; i< UDATPG_FIELD_COUNT; ++i ) { + appendItemFormats[i] = other.appendItemFormats[i]; + appendItemFormats[i].getTerminatedBuffer(); // NUL-terminate for the C API. + for (int32_t j=0; j< UDATPG_WIDTH_COUNT; ++j ) { + fieldDisplayNames[i][j] = other.fieldDisplayNames[i][j]; + fieldDisplayNames[i][j].getTerminatedBuffer(); // NUL-terminate for the C API. + } + } + patternMap->copyFrom(*other.patternMap, internalErrorCode); + copyHashtable(other.fAvailableFormatKeyHash, internalErrorCode); + return *this; +} + + +bool +DateTimePatternGenerator::operator==(const DateTimePatternGenerator& other) const { + if (this == &other) { + return true; + } + if ((pLocale==other.pLocale) && (patternMap->equals(*other.patternMap)) && + (decimal==other.decimal)) { + for (int32_t style = UDAT_FULL; style <= UDAT_SHORT; style++) { + if (dateTimeFormat[style] != other.dateTimeFormat[style]) { + return false; + } + } + for ( int32_t i=0 ; i list; + int32_t length = 0; + int32_t preferredFormat = ALLOWED_HOUR_FORMAT_UNKNOWN; + for (int32_t j = 0; formatList.getKeyAndValue(j, key, value); ++j) { + if (uprv_strcmp(key, "allowed") == 0) { + if (value.getType() == URES_STRING) { + length = 2; // 1 preferred to add later, 1 allowed to add now + if (list.allocateInsteadAndReset(length + 1) == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + list[1] = getHourFormatFromUnicodeString(value.getUnicodeString(errorCode)); + } + else { + ResourceArray allowedFormats = value.getArray(errorCode); + length = allowedFormats.getSize() + 1; // 1 preferred, getSize allowed + if (list.allocateInsteadAndReset(length + 1) == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (int32_t k = 1; k < length; ++k) { + allowedFormats.getValue(k-1, value); + list[k] = getHourFormatFromUnicodeString(value.getUnicodeString(errorCode)); + } + } + } else if (uprv_strcmp(key, "preferred") == 0) { + preferredFormat = getHourFormatFromUnicodeString(value.getUnicodeString(errorCode)); + } + } + if (length > 1) { + list[0] = (preferredFormat!=ALLOWED_HOUR_FORMAT_UNKNOWN)? preferredFormat: list[1]; + } else { + // fallback handling for missing data + length = 2; // 1 preferred, 1 allowed + if (list.allocateInsteadAndReset(length + 1) == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + list[0] = (preferredFormat!=ALLOWED_HOUR_FORMAT_UNKNOWN)? preferredFormat: ALLOWED_HOUR_FORMAT_H; + list[1] = list[0]; + } + list[length] = ALLOWED_HOUR_FORMAT_UNKNOWN; + // At this point list[] will have at least two non-ALLOWED_HOUR_FORMAT_UNKNOWN entries, + // followed by ALLOWED_HOUR_FORMAT_UNKNOWN. + uhash_put(localeToAllowedHourFormatsMap, const_cast(regionOrLocale), list.orphan(), &errorCode); + if (U_FAILURE(errorCode)) { return; } + } + } + + AllowedHourFormat getHourFormatFromUnicodeString(const UnicodeString &s) { + if (s.length() == 1) { + if (s[0] == LOW_H) { return ALLOWED_HOUR_FORMAT_h; } + if (s[0] == CAP_H) { return ALLOWED_HOUR_FORMAT_H; } + if (s[0] == CAP_K) { return ALLOWED_HOUR_FORMAT_K; } + if (s[0] == LOW_K) { return ALLOWED_HOUR_FORMAT_k; } + } else if (s.length() == 2) { + if (s[0] == LOW_H && s[1] == LOW_B) { return ALLOWED_HOUR_FORMAT_hb; } + if (s[0] == LOW_H && s[1] == CAP_B) { return ALLOWED_HOUR_FORMAT_hB; } + if (s[0] == CAP_K && s[1] == LOW_B) { return ALLOWED_HOUR_FORMAT_Kb; } + if (s[0] == CAP_K && s[1] == CAP_B) { return ALLOWED_HOUR_FORMAT_KB; } + if (s[0] == CAP_H && s[1] == LOW_B) { return ALLOWED_HOUR_FORMAT_Hb; } + if (s[0] == CAP_H && s[1] == CAP_B) { return ALLOWED_HOUR_FORMAT_HB; } + } + + return ALLOWED_HOUR_FORMAT_UNKNOWN; + } +}; + +} // namespace + +AllowedHourFormatsSink::~AllowedHourFormatsSink() {} + +U_CFUNC void U_CALLCONV DateTimePatternGenerator::loadAllowedHourFormatsData(UErrorCode &status) { + if (U_FAILURE(status)) { return; } + localeToAllowedHourFormatsMap = uhash_open( + uhash_hashChars, uhash_compareChars, nullptr, &status); + if (U_FAILURE(status)) { return; } + + uhash_setValueDeleter(localeToAllowedHourFormatsMap, deleteAllowedHourFormats); + ucln_i18n_registerCleanup(UCLN_I18N_ALLOWED_HOUR_FORMATS, allowedHourFormatsCleanup); + + LocalUResourceBundlePointer rb(ures_openDirect(nullptr, "supplementalData", &status)); + if (U_FAILURE(status)) { return; } + + AllowedHourFormatsSink sink; + // TODO: Currently in the enumeration each table allocates a new array. + // Try to reduce the number of memory allocations. Consider storing a + // UVector32 with the concatenation of all of the sub-arrays, put the start index + // into the hashmap, store 6 single-value sub-arrays right at the beginning of the + // vector (at index enum*2) for easy data sharing, copy sub-arrays into runtime + // object. Remember to clean up the vector, too. + ures_getAllItemsWithFallback(rb.getAlias(), "timeData", sink, status); +} + +static int32_t* getAllowedHourFormatsLangCountry(const char* language, const char* country, UErrorCode& status) { + CharString langCountry; + langCountry.append(language, status); + langCountry.append('_', status); + langCountry.append(country, status); + + int32_t* allowedFormats; + allowedFormats = (int32_t *)uhash_get(localeToAllowedHourFormatsMap, langCountry.data()); + if (allowedFormats == nullptr) { + allowedFormats = (int32_t *)uhash_get(localeToAllowedHourFormatsMap, const_cast(country)); + } + + return allowedFormats; +} + +void DateTimePatternGenerator::getAllowedHourFormats(const Locale &locale, UErrorCode &status) { + if (U_FAILURE(status)) { return; } + + const char *language = locale.getLanguage(); + const char *country = locale.getCountry(); + + char regionOverride[8]; + int32_t regionOverrideLength = locale.getKeywordValue("rg", regionOverride, sizeof(regionOverride), status); + if (U_SUCCESS(status) && regionOverrideLength > 0) { + country = regionOverride; + if (regionOverrideLength > 2) { + // chop off any subdivision codes that may have been included + regionOverride[2] = '\0'; + } + } + + Locale maxLocale; // must be here for correct lifetime + if (*language == '\0' || *country == '\0') { + maxLocale = locale; + UErrorCode localStatus = U_ZERO_ERROR; + maxLocale.addLikelySubtags(localStatus); + if (U_SUCCESS(localStatus)) { + language = maxLocale.getLanguage(); + country = maxLocale.getCountry(); + } + } + if (*language == '\0') { + // Unexpected, but fail gracefully + language = "und"; + } + if (*country == '\0') { + country = "001"; + } + + int32_t* allowedFormats = getAllowedHourFormatsLangCountry(language, country, status); + + // We need to check if there is an hour cycle on locale + char buffer[8]; + int32_t count = locale.getKeywordValue("hours", buffer, sizeof(buffer), status); + + fDefaultHourFormatChar = 0; + if (U_SUCCESS(status) && count > 0) { + if(uprv_strcmp(buffer, "h24") == 0) { + fDefaultHourFormatChar = LOW_K; + } else if(uprv_strcmp(buffer, "h23") == 0) { + fDefaultHourFormatChar = CAP_H; + } else if(uprv_strcmp(buffer, "h12") == 0) { + fDefaultHourFormatChar = LOW_H; + } else if(uprv_strcmp(buffer, "h11") == 0) { + fDefaultHourFormatChar = CAP_K; + } + } + + // Check if the region has an alias + if (allowedFormats == nullptr) { + UErrorCode localStatus = U_ZERO_ERROR; + const Region* region = Region::getInstance(country, localStatus); + if (U_SUCCESS(localStatus)) { + country = region->getRegionCode(); // the real region code + allowedFormats = getAllowedHourFormatsLangCountry(language, country, status); + } + } + + if (allowedFormats != nullptr) { // Lookup is successful + // Here allowedFormats points to a list consisting of key for preferredFormat, + // followed by one or more keys for allowedFormats, then followed by ALLOWED_HOUR_FORMAT_UNKNOWN. + if (!fDefaultHourFormatChar) { + switch (allowedFormats[0]) { + case ALLOWED_HOUR_FORMAT_h: fDefaultHourFormatChar = LOW_H; break; + case ALLOWED_HOUR_FORMAT_H: fDefaultHourFormatChar = CAP_H; break; + case ALLOWED_HOUR_FORMAT_K: fDefaultHourFormatChar = CAP_K; break; + case ALLOWED_HOUR_FORMAT_k: fDefaultHourFormatChar = LOW_K; break; + default: fDefaultHourFormatChar = CAP_H; break; + } + } + + for (int32_t i = 0; i < UPRV_LENGTHOF(fAllowedHourFormats); ++i) { + fAllowedHourFormats[i] = allowedFormats[i + 1]; + if (fAllowedHourFormats[i] == ALLOWED_HOUR_FORMAT_UNKNOWN) { + break; + } + } + } else { // Lookup failed, twice + if (!fDefaultHourFormatChar) { + fDefaultHourFormatChar = CAP_H; + } + fAllowedHourFormats[0] = ALLOWED_HOUR_FORMAT_H; + fAllowedHourFormats[1] = ALLOWED_HOUR_FORMAT_UNKNOWN; + } +} + +UDateFormatHourCycle +DateTimePatternGenerator::getDefaultHourCycle(UErrorCode& status) const { + if (U_FAILURE(status)) { + return UDAT_HOUR_CYCLE_23; + } + if (fDefaultHourFormatChar == 0) { + // We need to return something, but the caller should ignore it + // anyways since the returned status is a failure. + status = U_UNSUPPORTED_ERROR; + return UDAT_HOUR_CYCLE_23; + } + switch (fDefaultHourFormatChar) { + case CAP_K: + return UDAT_HOUR_CYCLE_11; + case LOW_H: + return UDAT_HOUR_CYCLE_12; + case CAP_H: + return UDAT_HOUR_CYCLE_23; + case LOW_K: + return UDAT_HOUR_CYCLE_24; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +UnicodeString +DateTimePatternGenerator::getSkeleton(const UnicodeString& pattern, UErrorCode& +/*status*/) { + FormatParser fp2; + DateTimeMatcher matcher; + PtnSkeleton localSkeleton; + matcher.set(pattern, &fp2, localSkeleton); + return localSkeleton.getSkeleton(); +} + +UnicodeString +DateTimePatternGenerator::staticGetSkeleton( + const UnicodeString& pattern, UErrorCode& /*status*/) { + FormatParser fp; + DateTimeMatcher matcher; + PtnSkeleton localSkeleton; + matcher.set(pattern, &fp, localSkeleton); + return localSkeleton.getSkeleton(); +} + +UnicodeString +DateTimePatternGenerator::getBaseSkeleton(const UnicodeString& pattern, UErrorCode& /*status*/) { + FormatParser fp2; + DateTimeMatcher matcher; + PtnSkeleton localSkeleton; + matcher.set(pattern, &fp2, localSkeleton); + return localSkeleton.getBaseSkeleton(); +} + +UnicodeString +DateTimePatternGenerator::staticGetBaseSkeleton( + const UnicodeString& pattern, UErrorCode& /*status*/) { + FormatParser fp; + DateTimeMatcher matcher; + PtnSkeleton localSkeleton; + matcher.set(pattern, &fp, localSkeleton); + return localSkeleton.getBaseSkeleton(); +} + +void +DateTimePatternGenerator::addICUPatterns(const Locale& locale, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + UnicodeString dfPattern; + UnicodeString conflictingString; + DateFormat* df; + + // Load with ICU patterns + for (int32_t i=DateFormat::kFull; i<=DateFormat::kShort; i++) { + DateFormat::EStyle style = (DateFormat::EStyle)i; + df = DateFormat::createDateInstance(style, locale); + SimpleDateFormat* sdf; + if (df != nullptr && (sdf = dynamic_cast(df)) != nullptr) { + sdf->toPattern(dfPattern); + addPattern(dfPattern, false, conflictingString, status); + } + // TODO Maybe we should return an error when the date format isn't simple. + delete df; + if (U_FAILURE(status)) { return; } + + df = DateFormat::createTimeInstance(style, locale); + if (df != nullptr && (sdf = dynamic_cast(df)) != nullptr) { + sdf->toPattern(dfPattern); + addPattern(dfPattern, false, conflictingString, status); + + // TODO: C++ and Java are inconsistent (see #12568). + // C++ uses MEDIUM, but Java uses SHORT. + if ( i==DateFormat::kShort && !dfPattern.isEmpty() ) { + consumeShortTimePattern(dfPattern, status); + } + } + // TODO Maybe we should return an error when the date format isn't simple. + delete df; + if (U_FAILURE(status)) { return; } + } +} + +void +DateTimePatternGenerator::hackTimes(const UnicodeString& hackPattern, UErrorCode& status) { + UnicodeString conflictingString; + + fp->set(hackPattern); + UnicodeString mmss; + UBool gotMm=false; + for (int32_t i=0; iitemNumber; ++i) { + UnicodeString field = fp->items[i]; + if ( fp->isQuoteLiteral(field) ) { + if ( gotMm ) { + UnicodeString quoteLiteral; + fp->getQuoteLiteral(quoteLiteral, &i); + mmss += quoteLiteral; + } + } + else { + if (fp->isPatternSeparator(field) && gotMm) { + mmss+=field; + } + else { + char16_t ch=field.charAt(0); + if (ch==LOW_M) { + gotMm=true; + mmss+=field; + } + else { + if (ch==LOW_S) { + if (!gotMm) { + break; + } + mmss+= field; + addPattern(mmss, false, conflictingString, status); + break; + } + else { + if (gotMm || ch==LOW_Z || ch==CAP_Z || ch==LOW_V || ch==CAP_V) { + break; + } + } + } + } + } + } +} + +#define ULOC_LOCALE_IDENTIFIER_CAPACITY (ULOC_FULLNAME_CAPACITY + 1 + ULOC_KEYWORD_AND_VALUES_CAPACITY) + +void +DateTimePatternGenerator::getCalendarTypeToUse(const Locale& locale, CharString& destination, UErrorCode& err) { + destination.clear().append(DT_DateTimeGregorianTag, -1, err); // initial default + if ( U_SUCCESS(err) ) { + UErrorCode localStatus = U_ZERO_ERROR; + char localeWithCalendarKey[ULOC_LOCALE_IDENTIFIER_CAPACITY]; + // obtain a locale that always has the calendar key value that should be used + ures_getFunctionalEquivalent( + localeWithCalendarKey, + ULOC_LOCALE_IDENTIFIER_CAPACITY, + nullptr, + "calendar", + "calendar", + locale.getName(), + nullptr, + false, + &localStatus); + localeWithCalendarKey[ULOC_LOCALE_IDENTIFIER_CAPACITY-1] = 0; // ensure null termination + // now get the calendar key value from that locale + char calendarType[ULOC_KEYWORDS_CAPACITY]; + int32_t calendarTypeLen = uloc_getKeywordValue( + localeWithCalendarKey, + "calendar", + calendarType, + ULOC_KEYWORDS_CAPACITY, + &localStatus); + // If the input locale was invalid, don't fail with missing resource error, instead + // continue with default of Gregorian. + if (U_FAILURE(localStatus) && localStatus != U_MISSING_RESOURCE_ERROR) { + err = localStatus; + return; + } + if (calendarTypeLen > 0 && calendarTypeLen < ULOC_KEYWORDS_CAPACITY) { + destination.clear().append(calendarType, -1, err); + if (U_FAILURE(err)) { return; } + } + } +} + +void +DateTimePatternGenerator::consumeShortTimePattern(const UnicodeString& shortTimePattern, + UErrorCode& status) { + if (U_FAILURE(status)) { return; } + // ICU-20383 No longer set fDefaultHourFormatChar to the hour format character from + // this pattern; instead it is set from localeToAllowedHourFormatsMap which now + // includes entries for both preferred and allowed formats. + + // HACK for hh:ss + hackTimes(shortTimePattern, status); +} + +struct DateTimePatternGenerator::AppendItemFormatsSink : public ResourceSink { + + // Destination for data, modified via setters. + DateTimePatternGenerator& dtpg; + + AppendItemFormatsSink(DateTimePatternGenerator& _dtpg) : dtpg(_dtpg) {} + virtual ~AppendItemFormatsSink(); + + virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/, + UErrorCode &errorCode) override { + UDateTimePatternField field = dtpg.getAppendFormatNumber(key); + if (field == UDATPG_FIELD_COUNT) { return; } + const UnicodeString& valueStr = value.getUnicodeString(errorCode); + if (dtpg.getAppendItemFormat(field).isEmpty() && !valueStr.isEmpty()) { + dtpg.setAppendItemFormat(field, valueStr); + } + } + + void fillInMissing() { + UnicodeString defaultItemFormat(true, UDATPG_ItemFormat, UPRV_LENGTHOF(UDATPG_ItemFormat)-1); // Read-only alias. + for (int32_t i = 0; i < UDATPG_FIELD_COUNT; i++) { + UDateTimePatternField field = (UDateTimePatternField)i; + if (dtpg.getAppendItemFormat(field).isEmpty()) { + dtpg.setAppendItemFormat(field, defaultItemFormat); + } + } + } +}; + +struct DateTimePatternGenerator::AppendItemNamesSink : public ResourceSink { + + // Destination for data, modified via setters. + DateTimePatternGenerator& dtpg; + + AppendItemNamesSink(DateTimePatternGenerator& _dtpg) : dtpg(_dtpg) {} + virtual ~AppendItemNamesSink(); + + virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/, + UErrorCode &errorCode) override { + UDateTimePGDisplayWidth width; + UDateTimePatternField field = dtpg.getFieldAndWidthIndices(key, &width); + if (field == UDATPG_FIELD_COUNT) { return; } + ResourceTable detailsTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + if (!detailsTable.findValue("dn", value)) { return; } + const UnicodeString& valueStr = value.getUnicodeString(errorCode); + if (U_SUCCESS(errorCode) && dtpg.getFieldDisplayName(field,width).isEmpty() && !valueStr.isEmpty()) { + dtpg.setFieldDisplayName(field,width,valueStr); + } + } + + void fillInMissing() { + for (int32_t i = 0; i < UDATPG_FIELD_COUNT; i++) { + UnicodeString& valueStr = dtpg.getMutableFieldDisplayName((UDateTimePatternField)i, UDATPG_WIDE); + if (valueStr.isEmpty()) { + valueStr = CAP_F; + U_ASSERT(i < 20); + if (i < 10) { + // F0, F1, ..., F9 + valueStr += (char16_t)(i+0x30); + } else { + // F10, F11, ... + valueStr += (char16_t)0x31; + valueStr += (char16_t)(i-10 + 0x30); + } + // NUL-terminate for the C API. + valueStr.getTerminatedBuffer(); + } + for (int32_t j = 1; j < UDATPG_WIDTH_COUNT; j++) { + UnicodeString& valueStr2 = dtpg.getMutableFieldDisplayName((UDateTimePatternField)i, (UDateTimePGDisplayWidth)j); + if (valueStr2.isEmpty()) { + valueStr2 = dtpg.getFieldDisplayName((UDateTimePatternField)i, (UDateTimePGDisplayWidth)(j-1)); + } + } + } + } +}; + +struct DateTimePatternGenerator::AvailableFormatsSink : public ResourceSink { + + // Destination for data, modified via setters. + DateTimePatternGenerator& dtpg; + + // Temporary variable, required for calling addPatternWithSkeleton. + UnicodeString conflictingPattern; + + AvailableFormatsSink(DateTimePatternGenerator& _dtpg) : dtpg(_dtpg) {} + virtual ~AvailableFormatsSink(); + + virtual void put(const char *key, ResourceValue &value, UBool isRoot, + UErrorCode &errorCode) override { + const UnicodeString formatKey(key, -1, US_INV); + if (!dtpg.isAvailableFormatSet(formatKey) ) { + dtpg.setAvailableFormat(formatKey, errorCode); + // Add pattern with its associated skeleton. Override any duplicate + // derived from std patterns, but not a previous availableFormats entry: + const UnicodeString& formatValue = value.getUnicodeString(errorCode); + conflictingPattern.remove(); + dtpg.addPatternWithSkeleton(formatValue, &formatKey, !isRoot, conflictingPattern, errorCode); + } + } +}; + +// Virtual destructors must be defined out of line. +DateTimePatternGenerator::AppendItemFormatsSink::~AppendItemFormatsSink() {} +DateTimePatternGenerator::AppendItemNamesSink::~AppendItemNamesSink() {} +DateTimePatternGenerator::AvailableFormatsSink::~AvailableFormatsSink() {} + +void +DateTimePatternGenerator::addCLDRData(const Locale& locale, UErrorCode& errorCode) { + if (U_FAILURE(errorCode)) { return; } + UnicodeString rbPattern, value, field; + CharString path; + + LocalUResourceBundlePointer rb(ures_open(nullptr, locale.getName(), &errorCode)); + if (U_FAILURE(errorCode)) { return; } + + CharString calendarTypeToUse; // to be filled in with the type to use, if all goes well + getCalendarTypeToUse(locale, calendarTypeToUse, errorCode); + if (U_FAILURE(errorCode)) { return; } + + // Local err to ignore resource not found exceptions + UErrorCode err = U_ZERO_ERROR; + + // Load append item formats. + AppendItemFormatsSink appendItemFormatsSink(*this); + path.clear() + .append(DT_DateTimeCalendarTag, errorCode) + .append('/', errorCode) + .append(calendarTypeToUse, errorCode) + .append('/', errorCode) + .append(DT_DateTimeAppendItemsTag, errorCode); // i.e., calendar/xxx/appendItems + if (U_FAILURE(errorCode)) { return; } + ures_getAllChildrenWithFallback(rb.getAlias(), path.data(), appendItemFormatsSink, err); + appendItemFormatsSink.fillInMissing(); + + // Load CLDR item names. + err = U_ZERO_ERROR; + AppendItemNamesSink appendItemNamesSink(*this); + ures_getAllChildrenWithFallback(rb.getAlias(), DT_DateTimeFieldsTag, appendItemNamesSink, err); + appendItemNamesSink.fillInMissing(); + + // Load the available formats from CLDR. + err = U_ZERO_ERROR; + initHashtable(errorCode); + if (U_FAILURE(errorCode)) { return; } + AvailableFormatsSink availableFormatsSink(*this); + path.clear() + .append(DT_DateTimeCalendarTag, errorCode) + .append('/', errorCode) + .append(calendarTypeToUse, errorCode) + .append('/', errorCode) + .append(DT_DateTimeAvailableFormatsTag, errorCode); // i.e., calendar/xxx/availableFormats + if (U_FAILURE(errorCode)) { return; } + ures_getAllChildrenWithFallback(rb.getAlias(), path.data(), availableFormatsSink, err); +} + +void +DateTimePatternGenerator::initHashtable(UErrorCode& err) { + if (U_FAILURE(err)) { return; } + if (fAvailableFormatKeyHash!=nullptr) { + return; + } + LocalPointer hash(new Hashtable(false, err), err); + if (U_SUCCESS(err)) { + fAvailableFormatKeyHash = hash.orphan(); + } +} + +void +DateTimePatternGenerator::setAppendItemFormat(UDateTimePatternField field, const UnicodeString& value) { + appendItemFormats[field] = value; + // NUL-terminate for the C API. + appendItemFormats[field].getTerminatedBuffer(); +} + +const UnicodeString& +DateTimePatternGenerator::getAppendItemFormat(UDateTimePatternField field) const { + return appendItemFormats[field]; +} + +void +DateTimePatternGenerator::setAppendItemName(UDateTimePatternField field, const UnicodeString& value) { + setFieldDisplayName(field, UDATPG_WIDTH_APPENDITEM, value); +} + +const UnicodeString& +DateTimePatternGenerator::getAppendItemName(UDateTimePatternField field) const { + return fieldDisplayNames[field][UDATPG_WIDTH_APPENDITEM]; +} + +void +DateTimePatternGenerator::setFieldDisplayName(UDateTimePatternField field, UDateTimePGDisplayWidth width, const UnicodeString& value) { + fieldDisplayNames[field][width] = value; + // NUL-terminate for the C API. + fieldDisplayNames[field][width].getTerminatedBuffer(); +} + +UnicodeString +DateTimePatternGenerator::getFieldDisplayName(UDateTimePatternField field, UDateTimePGDisplayWidth width) const { + return fieldDisplayNames[field][width]; +} + +UnicodeString& +DateTimePatternGenerator::getMutableFieldDisplayName(UDateTimePatternField field, UDateTimePGDisplayWidth width) { + return fieldDisplayNames[field][width]; +} + +void +DateTimePatternGenerator::getAppendName(UDateTimePatternField field, UnicodeString& value) { + value = SINGLE_QUOTE; + value += fieldDisplayNames[field][UDATPG_WIDTH_APPENDITEM]; + value += SINGLE_QUOTE; +} + +UnicodeString +DateTimePatternGenerator::getBestPattern(const UnicodeString& patternForm, UErrorCode& status) { + return getBestPattern(patternForm, UDATPG_MATCH_NO_OPTIONS, status); +} + +UnicodeString +DateTimePatternGenerator::getBestPattern(const UnicodeString& patternForm, UDateTimePatternMatchOptions options, UErrorCode& status) { + if (U_FAILURE(status)) { + return UnicodeString(); + } + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return UnicodeString(); + } + const UnicodeString *bestPattern = nullptr; + UnicodeString dtFormat; + UnicodeString resultPattern; + int32_t flags = kDTPGNoFlags; + + int32_t dateMask=(1<set(patternFormMapped, fp); + const PtnSkeleton* specifiedSkeleton = nullptr; + bestPattern=getBestRaw(*dtMatcher, -1, distanceInfo, status, &specifiedSkeleton); + if (U_FAILURE(status)) { + return UnicodeString(); + } + + if ( distanceInfo->missingFieldMask==0 && distanceInfo->extraFieldMask==0 ) { + resultPattern = adjustFieldTypes(*bestPattern, specifiedSkeleton, flags, options); + + return resultPattern; + } + int32_t neededFields = dtMatcher->getFieldMask(); + UnicodeString datePattern=getBestAppending(neededFields & dateMask, flags, status, options); + UnicodeString timePattern=getBestAppending(neededFields & timeMask, flags, status, options); + if (U_FAILURE(status)) { + return UnicodeString(); + } + if (datePattern.length()==0) { + if (timePattern.length()==0) { + resultPattern.remove(); + } + else { + return timePattern; + } + } + if (timePattern.length()==0) { + return datePattern; + } + resultPattern.remove(); + status = U_ZERO_ERROR; + // determine which dateTimeFormat to use + PtnSkeleton* reqSkeleton = dtMatcher->getSkeletonPtr(); + UDateFormatStyle style = UDAT_SHORT; + int32_t monthFieldLen = reqSkeleton->baseOriginal.getFieldLength(UDATPG_MONTH_FIELD); + if (monthFieldLen == 4) { + if (reqSkeleton->baseOriginal.getFieldLength(UDATPG_WEEKDAY_FIELD) > 0) { + style = UDAT_FULL; + } else { + style = UDAT_LONG; + } + } else if (monthFieldLen == 3) { + style = UDAT_MEDIUM; + } + // and now use it to compose date and time + dtFormat=getDateTimeFormat(style, status); + SimpleFormatter(dtFormat, 2, 2, status).format(timePattern, datePattern, resultPattern, status); + return resultPattern; +} + +/* + * Map a skeleton that may have metacharacters jJC to one without, by replacing + * the metacharacters with locale-appropriate fields of h/H/k/K and of a/b/B + * (depends on fDefaultHourFormatChar and fAllowedHourFormats being set, which in + * turn depends on initData having been run). This method also updates the flags + * as necessary. Returns the updated skeleton. + */ +UnicodeString +DateTimePatternGenerator::mapSkeletonMetacharacters(const UnicodeString& patternForm, int32_t* flags, UErrorCode& status) { + UnicodeString patternFormMapped; + patternFormMapped.remove(); + UBool inQuoted = false; + int32_t patPos, patLen = patternForm.length(); + for (patPos = 0; patPos < patLen; patPos++) { + char16_t patChr = patternForm.charAt(patPos); + if (patChr == SINGLE_QUOTE) { + inQuoted = !inQuoted; + } else if (!inQuoted) { + // Handle special mappings for 'j' and 'C' in which fields lengths + // 1,3,5 => hour field length 1 + // 2,4,6 => hour field length 2 + // 1,2 => abbreviated dayPeriod (field length 1..3) + // 3,4 => long dayPeriod (field length 4) + // 5,6 => narrow dayPeriod (field length 5) + if (patChr == LOW_J || patChr == CAP_C) { + int32_t extraLen = 0; // 1 less than total field length + while (patPos+1 < patLen && patternForm.charAt(patPos+1)==patChr) { + extraLen++; + patPos++; + } + int32_t hourLen = 1 + (extraLen & 1); + int32_t dayPeriodLen = (extraLen < 2)? 1: 3 + (extraLen >> 1); + char16_t hourChar = LOW_H; + char16_t dayPeriodChar = LOW_A; + if (patChr == LOW_J) { + hourChar = fDefaultHourFormatChar; + } else { + AllowedHourFormat bestAllowed; + if (fAllowedHourFormats[0] != ALLOWED_HOUR_FORMAT_UNKNOWN) { + bestAllowed = (AllowedHourFormat)fAllowedHourFormats[0]; + } else { + status = U_INVALID_FORMAT_ERROR; + return UnicodeString(); + } + if (bestAllowed == ALLOWED_HOUR_FORMAT_H || bestAllowed == ALLOWED_HOUR_FORMAT_HB || bestAllowed == ALLOWED_HOUR_FORMAT_Hb) { + hourChar = CAP_H; + } else if (bestAllowed == ALLOWED_HOUR_FORMAT_K || bestAllowed == ALLOWED_HOUR_FORMAT_KB || bestAllowed == ALLOWED_HOUR_FORMAT_Kb) { + hourChar = CAP_K; + } else if (bestAllowed == ALLOWED_HOUR_FORMAT_k) { + hourChar = LOW_K; + } + // in #13183 just add b/B to skeleton, no longer need to set special flags + if (bestAllowed == ALLOWED_HOUR_FORMAT_HB || bestAllowed == ALLOWED_HOUR_FORMAT_hB || bestAllowed == ALLOWED_HOUR_FORMAT_KB) { + dayPeriodChar = CAP_B; + } else if (bestAllowed == ALLOWED_HOUR_FORMAT_Hb || bestAllowed == ALLOWED_HOUR_FORMAT_hb || bestAllowed == ALLOWED_HOUR_FORMAT_Kb) { + dayPeriodChar = LOW_B; + } + } + if (hourChar==CAP_H || hourChar==LOW_K) { + dayPeriodLen = 0; + } + while (dayPeriodLen-- > 0) { + patternFormMapped.append(dayPeriodChar); + } + while (hourLen-- > 0) { + patternFormMapped.append(hourChar); + } + } else if (patChr == CAP_J) { + // Get pattern for skeleton with H, then replace H or k + // with fDefaultHourFormatChar (if different) + patternFormMapped.append(CAP_H); + *flags |= kDTPGSkeletonUsesCapJ; + } else { + patternFormMapped.append(patChr); + } + } + } + return patternFormMapped; +} + +UnicodeString +DateTimePatternGenerator::replaceFieldTypes(const UnicodeString& pattern, + const UnicodeString& skeleton, + UErrorCode& status) { + return replaceFieldTypes(pattern, skeleton, UDATPG_MATCH_NO_OPTIONS, status); +} + +UnicodeString +DateTimePatternGenerator::replaceFieldTypes(const UnicodeString& pattern, + const UnicodeString& skeleton, + UDateTimePatternMatchOptions options, + UErrorCode& status) { + if (U_FAILURE(status)) { + return UnicodeString(); + } + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return UnicodeString(); + } + dtMatcher->set(skeleton, fp); + UnicodeString result = adjustFieldTypes(pattern, nullptr, kDTPGNoFlags, options); + return result; +} + +void +DateTimePatternGenerator::setDecimal(const UnicodeString& newDecimal) { + this->decimal = newDecimal; + // NUL-terminate for the C API. + this->decimal.getTerminatedBuffer(); +} + +const UnicodeString& +DateTimePatternGenerator::getDecimal() const { + return decimal; +} + +void +DateTimePatternGenerator::addCanonicalItems(UErrorCode& status) { + if (U_FAILURE(status)) { return; } + UnicodeString conflictingPattern; + + for (int32_t i=0; i 0) { + addPattern(UnicodeString(Canonical_Items[i]), false, conflictingPattern, status); + } + if (U_FAILURE(status)) { return; } + } +} + +void +DateTimePatternGenerator::setDateTimeFormat(const UnicodeString& dtFormat) { + UErrorCode status = U_ZERO_ERROR; + for (int32_t style = UDAT_FULL; style <= UDAT_SHORT; style++) { + setDateTimeFormat((UDateFormatStyle)style, dtFormat, status); + } +} + +const UnicodeString& +DateTimePatternGenerator::getDateTimeFormat() const { + UErrorCode status = U_ZERO_ERROR; + return getDateTimeFormat(UDAT_MEDIUM, status); +} + +void +DateTimePatternGenerator::setDateTimeFormat(UDateFormatStyle style, const UnicodeString& dtFormat, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (style < UDAT_FULL || style > UDAT_SHORT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + dateTimeFormat[style] = dtFormat; + // Note for the following: getTerminatedBuffer() can re-allocate the UnicodeString + // buffer so we do this here before clients request a const ref to the UnicodeString + // or its buffer. + dateTimeFormat[style].getTerminatedBuffer(); // NUL-terminate for the C API. +} + +const UnicodeString& +DateTimePatternGenerator::getDateTimeFormat(UDateFormatStyle style, UErrorCode& status) const { + static const UnicodeString emptyString = UNICODE_STRING_SIMPLE(""); + if (U_FAILURE(status)) { + return emptyString; + } + if (style < UDAT_FULL || style > UDAT_SHORT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return emptyString; + } + return dateTimeFormat[style]; +} + +static const int32_t cTypeBufMax = 32; + +void +DateTimePatternGenerator::setDateTimeFromCalendar(const Locale& locale, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + + const char16_t *resStr; + int32_t resStrLen = 0; + + LocalUResourceBundlePointer calData(ures_open(nullptr, locale.getBaseName(), &status)); + if (U_FAILURE(status)) { return; } + ures_getByKey(calData.getAlias(), DT_DateTimeCalendarTag, calData.getAlias(), &status); + if (U_FAILURE(status)) { return; } + + char cType[cTypeBufMax + 1]; + Calendar::getCalendarTypeFromLocale(locale, cType, cTypeBufMax, status); + cType[cTypeBufMax] = 0; + if (U_FAILURE(status) || cType[0] == 0) { + status = U_ZERO_ERROR; + uprv_strcpy(cType, DT_DateTimeGregorianTag); + } + UBool cTypeIsGregorian = (uprv_strcmp(cType, DT_DateTimeGregorianTag) == 0); + + // Currently, for compatibility with pre-CLDR-42 data, we default to the "atTime" + // combining patterns. Depending on guidance in CLDR 42 spec and on DisplayOptions, + // we may change this. + LocalUResourceBundlePointer specificCalBundle; + LocalUResourceBundlePointer dateTimePatterns; + int32_t dateTimeOffset = 0; // initially for DateTimePatterns%atTime + if (!cTypeIsGregorian) { + specificCalBundle.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), cType, + nullptr, &status)); + dateTimePatterns.adoptInstead(ures_getByKeyWithFallback(specificCalBundle.getAlias(), DT_DateAtTimePatternsTag, // the %atTime variant, 4 entries + nullptr, &status)); + } + if (dateTimePatterns.isNull() || status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + specificCalBundle.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), DT_DateTimeGregorianTag, + nullptr, &status)); + dateTimePatterns.adoptInstead(ures_getByKeyWithFallback(specificCalBundle.getAlias(), DT_DateAtTimePatternsTag, // the %atTime variant, 4 entries + nullptr, &status)); + } + if (U_SUCCESS(status) && (ures_getSize(dateTimePatterns.getAlias()) < 4)) { + status = U_INVALID_FORMAT_ERROR; + } + if (status == U_MISSING_RESOURCE_ERROR) { + // Try again with standard variant + status = U_ZERO_ERROR; + dateTimePatterns.orphan(); + dateTimeOffset = (int32_t)DateFormat::kDateTimeOffset; + if (!cTypeIsGregorian) { + specificCalBundle.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), cType, + nullptr, &status)); + dateTimePatterns.adoptInstead(ures_getByKeyWithFallback(specificCalBundle.getAlias(), DT_DateTimePatternsTag, // the standard variant, 13 entries + nullptr, &status)); + } + if (dateTimePatterns.isNull() || status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + specificCalBundle.adoptInstead(ures_getByKeyWithFallback(calData.getAlias(), DT_DateTimeGregorianTag, + nullptr, &status)); + dateTimePatterns.adoptInstead(ures_getByKeyWithFallback(specificCalBundle.getAlias(), DT_DateTimePatternsTag, // the standard variant, 13 entries + nullptr, &status)); + } + if (U_SUCCESS(status) && (ures_getSize(dateTimePatterns.getAlias()) <= DateFormat::kDateTimeOffset + DateFormat::kShort)) { + status = U_INVALID_FORMAT_ERROR; + } + } + if (U_FAILURE(status)) { return; } + for (int32_t style = UDAT_FULL; style <= UDAT_SHORT; style++) { + resStr = ures_getStringByIndex(dateTimePatterns.getAlias(), dateTimeOffset + style, &resStrLen, &status); + setDateTimeFormat((UDateFormatStyle)style, UnicodeString(true, resStr, resStrLen), status); + } +} + +void +DateTimePatternGenerator::setDecimalSymbols(const Locale& locale, UErrorCode& status) { + DecimalFormatSymbols dfs = DecimalFormatSymbols(locale, status); + if(U_SUCCESS(status)) { + decimal = dfs.getSymbol(DecimalFormatSymbols::kDecimalSeparatorSymbol); + // NUL-terminate for the C API. + decimal.getTerminatedBuffer(); + } +} + +UDateTimePatternConflict +DateTimePatternGenerator::addPattern( + const UnicodeString& pattern, + UBool override, + UnicodeString &conflictingPattern, + UErrorCode& status) +{ + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return UDATPG_NO_CONFLICT; + } + + return addPatternWithSkeleton(pattern, nullptr, override, conflictingPattern, status); +} + +// For DateTimePatternGenerator::addPatternWithSkeleton - +// If skeletonToUse is specified, then an availableFormats entry is being added. In this case: +// 1. We pass that skeleton to matcher.set instead of having it derive a skeleton from the pattern. +// 2. If the new entry's skeleton or basePattern does match an existing entry but that entry also had a skeleton specified +// (i.e. it was also from availableFormats), then the new entry does not override it regardless of the value of the override +// parameter. This prevents later availableFormats entries from a parent locale overriding earlier ones from the actual +// specified locale. However, availableFormats entries *should* override entries with matching skeleton whose skeleton was +// derived (i.e. entries derived from the standard date/time patters for the specified locale). +// 3. When adding the pattern (patternMap->add), we set a new boolean to indicate that the added entry had a +// specified skeleton (which sets a new field in the PtnElem in the PatternMap). +UDateTimePatternConflict +DateTimePatternGenerator::addPatternWithSkeleton( + const UnicodeString& pattern, + const UnicodeString* skeletonToUse, + UBool override, + UnicodeString& conflictingPattern, + UErrorCode& status) +{ + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return UDATPG_NO_CONFLICT; + } + + UnicodeString basePattern; + PtnSkeleton skeleton; + UDateTimePatternConflict conflictingStatus = UDATPG_NO_CONFLICT; + + DateTimeMatcher matcher; + if ( skeletonToUse == nullptr ) { + matcher.set(pattern, fp, skeleton); + matcher.getBasePattern(basePattern); + } else { + matcher.set(*skeletonToUse, fp, skeleton); // no longer trims skeleton fields to max len 3, per #7930 + matcher.getBasePattern(basePattern); // or perhaps instead: basePattern = *skeletonToUse; + } + // We only care about base conflicts - and replacing the pattern associated with a base - if: + // 1. the conflicting previous base pattern did *not* have an explicit skeleton; in that case the previous + // base + pattern combination was derived from either (a) a canonical item, (b) a standard format, or + // (c) a pattern specified programmatically with a previous call to addPattern (which would only happen + // if we are getting here from a subsequent call to addPattern). + // 2. a skeleton is specified for the current pattern, but override=false; in that case we are checking + // availableFormats items from root, which should not override any previous entry with the same base. + UBool entryHadSpecifiedSkeleton; + const UnicodeString *duplicatePattern = patternMap->getPatternFromBasePattern(basePattern, entryHadSpecifiedSkeleton); + if (duplicatePattern != nullptr && (!entryHadSpecifiedSkeleton || (skeletonToUse != nullptr && !override))) { + conflictingStatus = UDATPG_BASE_CONFLICT; + conflictingPattern = *duplicatePattern; + if (!override) { + return conflictingStatus; + } + } + // The only time we get here with override=true and skeletonToUse!=null is when adding availableFormats + // items from CLDR data. In that case, we don't want an item from a parent locale to replace an item with + // same skeleton from the specified locale, so skip the current item if skeletonWasSpecified is true for + // the previously-specified conflicting item. + const PtnSkeleton* entrySpecifiedSkeleton = nullptr; + duplicatePattern = patternMap->getPatternFromSkeleton(skeleton, &entrySpecifiedSkeleton); + if (duplicatePattern != nullptr ) { + conflictingStatus = UDATPG_CONFLICT; + conflictingPattern = *duplicatePattern; + if (!override || (skeletonToUse != nullptr && entrySpecifiedSkeleton != nullptr)) { + return conflictingStatus; + } + } + patternMap->add(basePattern, skeleton, pattern, skeletonToUse != nullptr, status); + if(U_FAILURE(status)) { + return conflictingStatus; + } + + return UDATPG_NO_CONFLICT; +} + + +UDateTimePatternField +DateTimePatternGenerator::getAppendFormatNumber(const char* field) const { + for (int32_t i=0; i0; --i) { + if (uprv_strcmp(CLDR_FIELD_WIDTH[i], hyphenPtr)==0) { + *widthP=(UDateTimePGDisplayWidth)i; + break; + } + } + *hyphenPtr = 0; // now delete width portion of key + } + for (int32_t i=0; igetPatternFromSkeleton(*trial.getSkeletonPtr(), &specifiedSkeleton); + missingFields->setTo(tempInfo); + if (distance==0) { + break; + } + } + } + + // If the best raw match had a specified skeleton and that skeleton was requested by the caller, + // then return it too. This generally happens when the caller needs to pass that skeleton + // through to adjustFieldTypes so the latter can do a better job. + if (bestPattern && specifiedSkeletonPtr) { + *specifiedSkeletonPtr = specifiedSkeleton; + } + return bestPattern; +} + +UnicodeString +DateTimePatternGenerator::adjustFieldTypes(const UnicodeString& pattern, + const PtnSkeleton* specifiedSkeleton, + int32_t flags, + UDateTimePatternMatchOptions options) { + UnicodeString newPattern; + fp->set(pattern); + for (int32_t i=0; i < fp->itemNumber; i++) { + UnicodeString field = fp->items[i]; + if ( fp->isQuoteLiteral(field) ) { + + UnicodeString quoteLiteral; + fp->getQuoteLiteral(quoteLiteral, &i); + newPattern += quoteLiteral; + } + else { + if (fp->isPatternSeparator(field)) { + newPattern+=field; + continue; + } + int32_t canonicalIndex = fp->getCanonicalIndex(field); + if (canonicalIndex < 0) { + newPattern+=field; + continue; // don't adjust + } + const dtTypeElem *row = &dtTypes[canonicalIndex]; + int32_t typeValue = row->field; + + // handle day periods - with #13183, no longer need special handling here, integrated with normal types + + if ((flags & kDTPGFixFractionalSeconds) != 0 && typeValue == UDATPG_SECOND_FIELD) { + field += decimal; + dtMatcher->skeleton.original.appendFieldTo(UDATPG_FRACTIONAL_SECOND_FIELD, field); + } else if (dtMatcher->skeleton.type[typeValue]!=0) { + // Here: + // - "reqField" is the field from the originally requested skeleton after replacement + // of metacharacters 'j', 'C' and 'J', with length "reqFieldLen". + // - "field" is the field from the found pattern. + // + // The adjusted field should consist of characters from the originally requested + // skeleton, except in the case of UDATPG_MONTH_FIELD or + // UDATPG_WEEKDAY_FIELD or UDATPG_YEAR_FIELD, in which case it should consist + // of characters from the found pattern. In some cases of UDATPG_HOUR_FIELD, + // there is adjustment following the "defaultHourFormatChar". There is explanation + // how it is done below. + // + // The length of the adjusted field (adjFieldLen) should match that in the originally + // requested skeleton, except that in the following cases the length of the adjusted field + // should match that in the found pattern (i.e. the length of this pattern field should + // not be adjusted): + // 1. typeValue is UDATPG_HOUR_FIELD/MINUTE/SECOND and the corresponding bit in options is + // not set (ticket #7180). Note, we may want to implement a similar change for other + // numeric fields (MM, dd, etc.) so the default behavior is to get locale preference for + // field length, but options bits can be used to override this. + // 2. There is a specified skeleton for the found pattern and one of the following is true: + // a) The length of the field in the skeleton (skelFieldLen) is equal to reqFieldLen. + // b) The pattern field is numeric and the skeleton field is not, or vice versa. + + char16_t reqFieldChar = dtMatcher->skeleton.original.getFieldChar(typeValue); + int32_t reqFieldLen = dtMatcher->skeleton.original.getFieldLength(typeValue); + if (reqFieldChar == CAP_E && reqFieldLen < 3) + reqFieldLen = 3; // 1-3 for E are equivalent to 3 for c,e + int32_t adjFieldLen = reqFieldLen; + if ( (typeValue==UDATPG_HOUR_FIELD && (options & UDATPG_MATCH_HOUR_FIELD_LENGTH)==0) || + (typeValue==UDATPG_MINUTE_FIELD && (options & UDATPG_MATCH_MINUTE_FIELD_LENGTH)==0) || + (typeValue==UDATPG_SECOND_FIELD && (options & UDATPG_MATCH_SECOND_FIELD_LENGTH)==0) ) { + adjFieldLen = field.length(); + } else if (specifiedSkeleton && reqFieldChar != LOW_C && reqFieldChar != LOW_E) { + // (we skip this section for 'c' and 'e' because unlike the other characters considered in this function, + // they have no minimum field length-- 'E' and 'EE' are equivalent to 'EEE', but 'e' and 'ee' are not + // equivalent to 'eee' -- see the entries for "week day" in + // https://www.unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table for more info) + int32_t skelFieldLen = specifiedSkeleton->original.getFieldLength(typeValue); + UBool patFieldIsNumeric = (row->type > 0); + UBool skelFieldIsNumeric = (specifiedSkeleton->type[typeValue] > 0); + if (skelFieldLen == reqFieldLen || (patFieldIsNumeric && !skelFieldIsNumeric) || (skelFieldIsNumeric && !patFieldIsNumeric)) { + // don't adjust the field length in the found pattern + adjFieldLen = field.length(); + } + } + char16_t c = (typeValue!= UDATPG_HOUR_FIELD + && typeValue!= UDATPG_MONTH_FIELD + && typeValue!= UDATPG_WEEKDAY_FIELD + && (typeValue!= UDATPG_YEAR_FIELD || reqFieldChar==CAP_Y)) + ? reqFieldChar + : field.charAt(0); + if (c == CAP_E && adjFieldLen < 3) { + c = LOW_E; + } + if (typeValue == UDATPG_HOUR_FIELD && fDefaultHourFormatChar != 0) { + // The adjustment here is required to match spec (https://www.unicode.org/reports/tr35/tr35-dates.html#dfst-hour). + // It is necessary to match the hour-cycle preferred by the Locale. + // Given that, we need to do the following adjustments: + // 1. When hour-cycle is h11 it should replace 'h' by 'K'. + // 2. When hour-cycle is h23 it should replace 'H' by 'k'. + // 3. When hour-cycle is h24 it should replace 'k' by 'H'. + // 4. When hour-cycle is h12 it should replace 'K' by 'h'. + + if ((flags & kDTPGSkeletonUsesCapJ) != 0 || reqFieldChar == fDefaultHourFormatChar) { + c = fDefaultHourFormatChar; + } else if (reqFieldChar == LOW_H && fDefaultHourFormatChar == CAP_K) { + c = CAP_K; + } else if (reqFieldChar == CAP_H && fDefaultHourFormatChar == LOW_K) { + c = LOW_K; + } else if (reqFieldChar == LOW_K && fDefaultHourFormatChar == CAP_H) { + c = CAP_H; + } else if (reqFieldChar == CAP_K && fDefaultHourFormatChar == LOW_H) { + c = LOW_H; + } + } + + field.remove(); + for (int32_t j=adjFieldLen; j>0; --j) { + field += c; + } + } + newPattern+=field; + } + } + return newPattern; +} + +UnicodeString +DateTimePatternGenerator::getBestAppending(int32_t missingFields, int32_t flags, UErrorCode &status, UDateTimePatternMatchOptions options) { + if (U_FAILURE(status)) { + return UnicodeString(); + } + UnicodeString resultPattern, tempPattern; + const UnicodeString* tempPatternPtr; + int32_t lastMissingFieldMask=0; + if (missingFields!=0) { + resultPattern=UnicodeString(); + const PtnSkeleton* specifiedSkeleton=nullptr; + tempPatternPtr = getBestRaw(*dtMatcher, missingFields, distanceInfo, status, &specifiedSkeleton); + if (U_FAILURE(status)) { + return UnicodeString(); + } + tempPattern = *tempPatternPtr; + resultPattern = adjustFieldTypes(tempPattern, specifiedSkeleton, flags, options); + if ( distanceInfo->missingFieldMask==0 ) { + return resultPattern; + } + while (distanceInfo->missingFieldMask!=0) { // precondition: EVERY single field must work! + if ( lastMissingFieldMask == distanceInfo->missingFieldMask ) { + break; // cannot find the proper missing field + } + if (((distanceInfo->missingFieldMask & UDATPG_SECOND_AND_FRACTIONAL_MASK)==UDATPG_FRACTIONAL_MASK) && + ((missingFields & UDATPG_SECOND_AND_FRACTIONAL_MASK) == UDATPG_SECOND_AND_FRACTIONAL_MASK)) { + resultPattern = adjustFieldTypes(resultPattern, specifiedSkeleton, flags | kDTPGFixFractionalSeconds, options); + distanceInfo->missingFieldMask &= ~UDATPG_FRACTIONAL_MASK; + continue; + } + int32_t startingMask = distanceInfo->missingFieldMask; + tempPatternPtr = getBestRaw(*dtMatcher, distanceInfo->missingFieldMask, distanceInfo, status, &specifiedSkeleton); + if (U_FAILURE(status)) { + return UnicodeString(); + } + tempPattern = *tempPatternPtr; + tempPattern = adjustFieldTypes(tempPattern, specifiedSkeleton, flags, options); + int32_t foundMask=startingMask& ~distanceInfo->missingFieldMask; + int32_t topField=getTopBitNumber(foundMask); + + if (appendItemFormats[topField].length() != 0) { + UnicodeString appendName; + getAppendName((UDateTimePatternField)topField, appendName); + const UnicodeString *values[3] = { + &resultPattern, + &tempPattern, + &appendName + }; + SimpleFormatter(appendItemFormats[topField], 2, 3, status). + formatAndReplace(values, 3, resultPattern, nullptr, 0, status); + } + lastMissingFieldMask = distanceInfo->missingFieldMask; + } + } + return resultPattern; +} + +int32_t +DateTimePatternGenerator::getTopBitNumber(int32_t foundMask) const { + if ( foundMask==0 ) { + return 0; + } + int32_t i=0; + while (foundMask!=0) { + foundMask >>=1; + ++i; + } + if (i-1 >UDATPG_ZONE_FIELD) { + return UDATPG_ZONE_FIELD; + } + else + return i-1; +} + +void +DateTimePatternGenerator::setAvailableFormat(const UnicodeString &key, UErrorCode& err) +{ + fAvailableFormatKeyHash->puti(key, 1, err); +} + +UBool +DateTimePatternGenerator::isAvailableFormatSet(const UnicodeString &key) const { + return (UBool)(fAvailableFormatKeyHash->geti(key) == 1); +} + +void +DateTimePatternGenerator::copyHashtable(Hashtable *other, UErrorCode &status) { + if (other == nullptr || U_FAILURE(status)) { + return; + } + if (fAvailableFormatKeyHash != nullptr) { + delete fAvailableFormatKeyHash; + fAvailableFormatKeyHash = nullptr; + } + initHashtable(status); + if(U_FAILURE(status)){ + return; + } + int32_t pos = UHASH_FIRST; + const UHashElement* elem = nullptr; + // walk through the hash table and create a deep clone + while((elem = other->nextElement(pos))!= nullptr){ + const UHashTok otherKeyTok = elem->key; + UnicodeString* otherKey = (UnicodeString*)otherKeyTok.pointer; + fAvailableFormatKeyHash->puti(*otherKey, 1, status); + if(U_FAILURE(status)){ + return; + } + } +} + +StringEnumeration* +DateTimePatternGenerator::getSkeletons(UErrorCode& status) const { + if (U_FAILURE(status)) { + return nullptr; + } + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return nullptr; + } + LocalPointer skeletonEnumerator( + new DTSkeletonEnumeration(*patternMap, DT_SKELETON, status), status); + + return U_SUCCESS(status) ? skeletonEnumerator.orphan() : nullptr; +} + +const UnicodeString& +DateTimePatternGenerator::getPatternForSkeleton(const UnicodeString& skeleton) const { + PtnElem *curElem; + + if (skeleton.length() ==0) { + return emptyString; + } + curElem = patternMap->getHeader(skeleton.charAt(0)); + while ( curElem != nullptr ) { + if ( curElem->skeleton->getSkeleton()==skeleton ) { + return curElem->pattern; + } + curElem = curElem->next.getAlias(); + } + return emptyString; +} + +StringEnumeration* +DateTimePatternGenerator::getBaseSkeletons(UErrorCode& status) const { + if (U_FAILURE(status)) { + return nullptr; + } + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return nullptr; + } + LocalPointer baseSkeletonEnumerator( + new DTSkeletonEnumeration(*patternMap, DT_BASESKELETON, status), status); + + return U_SUCCESS(status) ? baseSkeletonEnumerator.orphan() : nullptr; +} + +StringEnumeration* +DateTimePatternGenerator::getRedundants(UErrorCode& status) { + if (U_FAILURE(status)) { return nullptr; } + if (U_FAILURE(internalErrorCode)) { + status = internalErrorCode; + return nullptr; + } + LocalPointer output(new DTRedundantEnumeration(), status); + if (U_FAILURE(status)) { return nullptr; } + const UnicodeString *pattern; + PatternMapIterator it(status); + if (U_FAILURE(status)) { return nullptr; } + + for (it.set(*patternMap); it.hasNext(); ) { + DateTimeMatcher current = it.next(); + pattern = patternMap->getPatternFromSkeleton(*(it.getSkeleton())); + if ( isCanonicalItem(*pattern) ) { + continue; + } + if ( skipMatcher == nullptr ) { + skipMatcher = new DateTimeMatcher(current); + if (skipMatcher == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + } + else { + *skipMatcher = current; + } + UnicodeString trial = getBestPattern(current.getPattern(), status); + if (U_FAILURE(status)) { return nullptr; } + if (trial == *pattern) { + ((DTRedundantEnumeration *)output.getAlias())->add(*pattern, status); + if (U_FAILURE(status)) { return nullptr; } + } + if (current.equals(skipMatcher)) { + continue; + } + } + return output.orphan(); +} + +UBool +DateTimePatternGenerator::isCanonicalItem(const UnicodeString& item) const { + if ( item.length() != 1 ) { + return false; + } + for (int32_t i=0; iisDupAllowed = other.isDupAllowed; + for (int32_t bootIndex = 0; bootIndex < MAX_PATTERN_ENTRIES; ++bootIndex) { + PtnElem *curElem, *otherElem, *prevElem=nullptr; + otherElem = other.boot[bootIndex]; + while (otherElem != nullptr) { + LocalPointer newElem(new PtnElem(otherElem->basePattern, otherElem->pattern), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeleton.adoptInsteadAndCheckErrorCode(new PtnSkeleton(*(otherElem->skeleton)), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeletonWasSpecified = otherElem->skeletonWasSpecified; + + // Release ownership from the LocalPointer of the PtnElem object. + // The PtnElem will now be owned by either the boot (for the first entry in the linked-list) + // or owned by the previous PtnElem object in the linked-list. + curElem = newElem.orphan(); + + if (this->boot[bootIndex] == nullptr) { + this->boot[bootIndex] = curElem; + } else { + if (prevElem != nullptr) { + prevElem->next.adoptInstead(curElem); + } else { + UPRV_UNREACHABLE_EXIT; + } + } + prevElem = curElem; + otherElem = otherElem->next.getAlias(); + } + + } +} + +PtnElem* +PatternMap::getHeader(char16_t baseChar) const { + PtnElem* curElem; + + if ( (baseChar >= CAP_A) && (baseChar <= CAP_Z) ) { + curElem = boot[baseChar-CAP_A]; + } + else { + if ( (baseChar >=LOW_A) && (baseChar <= LOW_Z) ) { + curElem = boot[26+baseChar-LOW_A]; + } + else { + return nullptr; + } + } + return curElem; +} + +PatternMap::~PatternMap() { + for (int32_t i=0; i < MAX_PATTERN_ENTRIES; ++i ) { + if (boot[i] != nullptr ) { + delete boot[i]; + boot[i] = nullptr; + } + } +} // PatternMap destructor + +void +PatternMap::add(const UnicodeString& basePattern, + const PtnSkeleton& skeleton, + const UnicodeString& value,// mapped pattern value + UBool skeletonWasSpecified, + UErrorCode &status) { + char16_t baseChar = basePattern.charAt(0); + PtnElem *curElem, *baseElem; + status = U_ZERO_ERROR; + + // the baseChar must be A-Z or a-z + if ((baseChar >= CAP_A) && (baseChar <= CAP_Z)) { + baseElem = boot[baseChar-CAP_A]; + } + else { + if ((baseChar >=LOW_A) && (baseChar <= LOW_Z)) { + baseElem = boot[26+baseChar-LOW_A]; + } + else { + status = U_ILLEGAL_CHARACTER; + return; + } + } + + if (baseElem == nullptr) { + LocalPointer newElem(new PtnElem(basePattern, value), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeleton.adoptInsteadAndCheckErrorCode(new PtnSkeleton(skeleton), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeletonWasSpecified = skeletonWasSpecified; + if (baseChar >= LOW_A) { + boot[26 + (baseChar - LOW_A)] = newElem.orphan(); // the boot array now owns the PtnElem. + } + else { + boot[baseChar - CAP_A] = newElem.orphan(); // the boot array now owns the PtnElem. + } + } + if ( baseElem != nullptr ) { + curElem = getDuplicateElem(basePattern, skeleton, baseElem); + + if (curElem == nullptr) { + // add new element to the list. + curElem = baseElem; + while( curElem -> next != nullptr ) + { + curElem = curElem->next.getAlias(); + } + + LocalPointer newElem(new PtnElem(basePattern, value), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeleton.adoptInsteadAndCheckErrorCode(new PtnSkeleton(skeleton), status); + if (U_FAILURE(status)) { + return; // out of memory + } + newElem->skeletonWasSpecified = skeletonWasSpecified; + curElem->next.adoptInstead(newElem.orphan()); + curElem = curElem->next.getAlias(); + } + else { + // Pattern exists in the list already. + if ( !isDupAllowed ) { + return; + } + // Overwrite the value. + curElem->pattern = value; + // It was a bug that we were not doing the following previously, + // though that bug hid other problems by making things partly work. + curElem->skeletonWasSpecified = skeletonWasSpecified; + } + } +} // PatternMap::add + +// Find the pattern from the given basePattern string. +const UnicodeString * +PatternMap::getPatternFromBasePattern(const UnicodeString& basePattern, UBool& skeletonWasSpecified) const { // key to search for + PtnElem *curElem; + + if ((curElem=getHeader(basePattern.charAt(0)))==nullptr) { + return nullptr; // no match + } + + do { + if ( basePattern.compare(curElem->basePattern)==0 ) { + skeletonWasSpecified = curElem->skeletonWasSpecified; + return &(curElem->pattern); + } + curElem = curElem->next.getAlias(); + } while (curElem != nullptr); + + return nullptr; +} // PatternMap::getFromBasePattern + + +// Find the pattern from the given skeleton. +// At least when this is called from getBestRaw & addPattern (in which case specifiedSkeletonPtr is non-nullptr), +// the comparison should be based on skeleton.original (which is unique and tied to the distance measurement in bestRaw) +// and not skeleton.baseOriginal (which is not unique); otherwise we may pick a different skeleton than the one with the +// optimum distance value in getBestRaw. When this is called from public getRedundants (specifiedSkeletonPtr is nullptr), +// for now it will continue to compare based on baseOriginal so as not to change the behavior unnecessarily. +const UnicodeString * +PatternMap::getPatternFromSkeleton(const PtnSkeleton& skeleton, const PtnSkeleton** specifiedSkeletonPtr) const { // key to search for + PtnElem *curElem; + + if (specifiedSkeletonPtr) { + *specifiedSkeletonPtr = nullptr; + } + + // find boot entry + char16_t baseChar = skeleton.getFirstChar(); + if ((curElem=getHeader(baseChar))==nullptr) { + return nullptr; // no match + } + + do { + UBool equal; + if (specifiedSkeletonPtr != nullptr) { // called from DateTimePatternGenerator::getBestRaw or addPattern, use original + equal = curElem->skeleton->original == skeleton.original; + } else { // called from DateTimePatternGenerator::getRedundants, use baseOriginal + equal = curElem->skeleton->baseOriginal == skeleton.baseOriginal; + } + if (equal) { + if (specifiedSkeletonPtr && curElem->skeletonWasSpecified) { + *specifiedSkeletonPtr = curElem->skeleton.getAlias(); + } + return &(curElem->pattern); + } + curElem = curElem->next.getAlias(); + } while (curElem != nullptr); + + return nullptr; +} + +UBool +PatternMap::equals(const PatternMap& other) const { + if ( this==&other ) { + return true; + } + for (int32_t bootIndex = 0; bootIndex < MAX_PATTERN_ENTRIES; ++bootIndex) { + if (boot[bootIndex] == other.boot[bootIndex]) { + continue; + } + if ((boot[bootIndex] == nullptr) || (other.boot[bootIndex] == nullptr)) { + return false; + } + PtnElem *otherElem = other.boot[bootIndex]; + PtnElem *myElem = boot[bootIndex]; + while ((otherElem != nullptr) || (myElem != nullptr)) { + if ( myElem == otherElem ) { + break; + } + if ((otherElem == nullptr) || (myElem == nullptr)) { + return false; + } + if ( (myElem->basePattern != otherElem->basePattern) || + (myElem->pattern != otherElem->pattern) ) { + return false; + } + if ((myElem->skeleton.getAlias() != otherElem->skeleton.getAlias()) && + !myElem->skeleton->equals(*(otherElem->skeleton))) { + return false; + } + myElem = myElem->next.getAlias(); + otherElem = otherElem->next.getAlias(); + } + } + return true; +} + +// find any key existing in the mapping table already. +// return true if there is an existing key, otherwise return false. +PtnElem* +PatternMap::getDuplicateElem( + const UnicodeString &basePattern, + const PtnSkeleton &skeleton, + PtnElem *baseElem) { + PtnElem *curElem; + + if ( baseElem == nullptr ) { + return nullptr; + } + else { + curElem = baseElem; + } + do { + if ( basePattern.compare(curElem->basePattern)==0 ) { + UBool isEqual = true; + for (int32_t i = 0; i < UDATPG_FIELD_COUNT; ++i) { + if (curElem->skeleton->type[i] != skeleton.type[i] ) { + isEqual = false; + break; + } + } + if (isEqual) { + return curElem; + } + } + curElem = curElem->next.getAlias(); + } while( curElem != nullptr ); + + // end of the list + return nullptr; + +} // PatternMap::getDuplicateElem + +DateTimeMatcher::DateTimeMatcher() { +} + +DateTimeMatcher::~DateTimeMatcher() {} + +DateTimeMatcher::DateTimeMatcher(const DateTimeMatcher& other) { + copyFrom(other.skeleton); +} + +DateTimeMatcher& DateTimeMatcher::operator=(const DateTimeMatcher& other) { + copyFrom(other.skeleton); + return *this; +} + + +void +DateTimeMatcher::set(const UnicodeString& pattern, FormatParser* fp) { + PtnSkeleton localSkeleton; + return set(pattern, fp, localSkeleton); +} + +void +DateTimeMatcher::set(const UnicodeString& pattern, FormatParser* fp, PtnSkeleton& skeletonResult) { + int32_t i; + for (i=0; iset(pattern); + for (i=0; i < fp->itemNumber; i++) { + const UnicodeString& value = fp->items[i]; + // don't skip 'a' anymore, dayPeriod handled specially below + + if ( fp->isQuoteLiteral(value) ) { + UnicodeString quoteLiteral; + fp->getQuoteLiteral(quoteLiteral, &i); + continue; + } + int32_t canonicalIndex = fp->getCanonicalIndex(value); + if (canonicalIndex < 0) { + continue; + } + const dtTypeElem *row = &dtTypes[canonicalIndex]; + int32_t field = row->field; + skeletonResult.original.populate(field, value); + char16_t repeatChar = row->patternChar; + int32_t repeatCount = row->minLen; + skeletonResult.baseOriginal.populate(field, repeatChar, repeatCount); + int16_t subField = row->type; + if (row->type > 0) { + U_ASSERT(value.length() < INT16_MAX); + subField += static_cast(value.length()); + } + skeletonResult.type[field] = subField; + } + + // #20739, we have a skeleton with minutes and milliseconds, but no seconds + // + // Theoretically we would need to check and fix all fields with "gaps": + // for example year-day (no month), month-hour (no day), and so on, All the possible field combinations. + // Plus some smartness: year + hour => should we add month, or add day-of-year? + // What about month + day-of-week, or month + am/pm indicator. + // I think beyond a certain point we should not try to fix bad developer input and try guessing what they mean. + // Garbage in, garbage out. + if (!skeletonResult.original.isFieldEmpty(UDATPG_MINUTE_FIELD) + && !skeletonResult.original.isFieldEmpty(UDATPG_FRACTIONAL_SECOND_FIELD) + && skeletonResult.original.isFieldEmpty(UDATPG_SECOND_FIELD)) { + // Force the use of seconds + for (i = 0; dtTypes[i].patternChar != 0; i++) { + if (dtTypes[i].field == UDATPG_SECOND_FIELD) { + // first entry for UDATPG_SECOND_FIELD + skeletonResult.original.populate(UDATPG_SECOND_FIELD, dtTypes[i].patternChar, dtTypes[i].minLen); + skeletonResult.baseOriginal.populate(UDATPG_SECOND_FIELD, dtTypes[i].patternChar, dtTypes[i].minLen); + // We add value.length, same as above, when type is first initialized. + // The value we want to "fake" here is "s", and 1 means "s".length() + int16_t subField = dtTypes[i].type; + skeletonResult.type[UDATPG_SECOND_FIELD] = (subField > 0) ? subField + 1 : subField; + break; + } + } + } + + // #13183, handle special behavior for day period characters (a, b, B) + if (!skeletonResult.original.isFieldEmpty(UDATPG_HOUR_FIELD)) { + if (skeletonResult.original.getFieldChar(UDATPG_HOUR_FIELD)==LOW_H || skeletonResult.original.getFieldChar(UDATPG_HOUR_FIELD)==CAP_K) { + // We have a skeleton with 12-hour-cycle format + if (skeletonResult.original.isFieldEmpty(UDATPG_DAYPERIOD_FIELD)) { + // But we do not have a day period in the skeleton; add the default DAYPERIOD (currently "a") + for (i = 0; dtTypes[i].patternChar != 0; i++) { + if ( dtTypes[i].field == UDATPG_DAYPERIOD_FIELD ) { + // first entry for UDATPG_DAYPERIOD_FIELD + skeletonResult.original.populate(UDATPG_DAYPERIOD_FIELD, dtTypes[i].patternChar, dtTypes[i].minLen); + skeletonResult.baseOriginal.populate(UDATPG_DAYPERIOD_FIELD, dtTypes[i].patternChar, dtTypes[i].minLen); + skeletonResult.type[UDATPG_DAYPERIOD_FIELD] = dtTypes[i].type; + skeletonResult.addedDefaultDayPeriod = true; + break; + } + } + } + } else { + // Skeleton has 24-hour-cycle hour format and has dayPeriod, delete dayPeriod (i.e. ignore it) + skeletonResult.original.clearField(UDATPG_DAYPERIOD_FIELD); + skeletonResult.baseOriginal.clearField(UDATPG_DAYPERIOD_FIELD); + skeletonResult.type[UDATPG_DAYPERIOD_FIELD] = NONE; + } + } + copyFrom(skeletonResult); +} + +void +DateTimeMatcher::getBasePattern(UnicodeString &result ) { + result.remove(); // Reset the result first. + skeleton.baseOriginal.appendTo(result); +} + +UnicodeString +DateTimeMatcher::getPattern() { + UnicodeString result; + return skeleton.original.appendTo(result); +} + +int32_t +DateTimeMatcher::getDistance(const DateTimeMatcher& other, int32_t includeMask, DistanceInfo& distanceInfo) const { + int32_t result = 0; + distanceInfo.clear(); + for (int32_t i=0; iskeleton.original; +} + +int32_t +DateTimeMatcher::getFieldMask() const { + int32_t result = 0; + + for (int32_t i=0; i= pattern.length()) { + return DONE; + } + // check the current char is between A-Z or a-z + do { + char16_t c=pattern.charAt(curLoc); + if ( (c>=CAP_A && c<=CAP_Z) || (c>=LOW_A && c<=LOW_Z) ) { + curLoc++; + } + else { + startPos = curLoc; + *len=1; + return ADD_TOKEN; + } + + if ( pattern.charAt(curLoc)!= pattern.charAt(startPos) ) { + break; // not the same token + } + } while(curLoc <= pattern.length()); + *len = curLoc-startPos; + return ADD_TOKEN; +} + +void +FormatParser::set(const UnicodeString& pattern) { + int32_t startPos = 0; + TokenStatus result = START; + int32_t len = 0; + itemNumber = 0; + + do { + result = setTokens( pattern, startPos, &len ); + if ( result == ADD_TOKEN ) + { + items[itemNumber++] = UnicodeString(pattern, startPos, len ); + startPos += len; + } + else { + break; + } + } while (result==ADD_TOKEN && itemNumber < MAX_DT_TOKEN); +} + +int32_t +FormatParser::getCanonicalIndex(const UnicodeString& s, UBool strict) { + int32_t len = s.length(); + if (len == 0) { + return -1; + } + char16_t ch = s.charAt(0); + + // Verify that all are the same character. + for (int32_t l = 1; l < len; l++) { + if (ch != s.charAt(l)) { + return -1; + } + } + int32_t i = 0; + int32_t bestRow = -1; + while (dtTypes[i].patternChar != 0x0000) { + if ( dtTypes[i].patternChar != ch ) { + ++i; + continue; + } + bestRow = i; + if (dtTypes[i].patternChar != dtTypes[i+1].patternChar) { + return i; + } + if (dtTypes[i+1].minLen <= len) { + ++i; + continue; + } + return i; + } + return strict ? -1 : bestRow; +} + +UBool +FormatParser::isQuoteLiteral(const UnicodeString& s) { + return (UBool)(s.charAt(0) == SINGLE_QUOTE); +} + +// This function assumes the current itemIndex points to the quote literal. +// Please call isQuoteLiteral prior to this function. +void +FormatParser::getQuoteLiteral(UnicodeString& quote, int32_t *itemIndex) { + int32_t i = *itemIndex; + + quote.remove(); + if (items[i].charAt(0)==SINGLE_QUOTE) { + quote += items[i]; + ++i; + } + while ( i < itemNumber ) { + if ( items[i].charAt(0)==SINGLE_QUOTE ) { + if ( (i+1patternMap=&newPatternMap; +} + +PtnSkeleton* +PatternMapIterator::getSkeleton() const { + if ( nodePtr == nullptr ) { + return nullptr; + } + else { + return nodePtr->skeleton.getAlias(); + } +} + +UBool +PatternMapIterator::hasNext() const { + int32_t headIndex = bootIndex; + PtnElem *curPtr = nodePtr; + + if (patternMap==nullptr) { + return false; + } + while ( headIndex < MAX_PATTERN_ENTRIES ) { + if ( curPtr != nullptr ) { + if ( curPtr->next != nullptr ) { + return true; + } + else { + headIndex++; + curPtr=nullptr; + continue; + } + } + else { + if ( patternMap->boot[headIndex] != nullptr ) { + return true; + } + else { + headIndex++; + continue; + } + } + } + return false; +} + +DateTimeMatcher& +PatternMapIterator::next() { + while ( bootIndex < MAX_PATTERN_ENTRIES ) { + if ( nodePtr != nullptr ) { + if ( nodePtr->next != nullptr ) { + nodePtr = nodePtr->next.getAlias(); + break; + } + else { + bootIndex++; + nodePtr=nullptr; + continue; + } + } + else { + if ( patternMap->boot[bootIndex] != nullptr ) { + nodePtr = patternMap->boot[bootIndex]; + break; + } + else { + bootIndex++; + continue; + } + } + } + if (nodePtr!=nullptr) { + matcher->copyFrom(*nodePtr->skeleton); + } + else { + matcher->copyFrom(); + } + return *matcher; +} + + +SkeletonFields::SkeletonFields() { + // Set initial values to zero + clear(); +} + +void SkeletonFields::clear() { + uprv_memset(chars, 0, sizeof(chars)); + uprv_memset(lengths, 0, sizeof(lengths)); +} + +void SkeletonFields::copyFrom(const SkeletonFields& other) { + uprv_memcpy(chars, other.chars, sizeof(chars)); + uprv_memcpy(lengths, other.lengths, sizeof(lengths)); +} + +void SkeletonFields::clearField(int32_t field) { + chars[field] = 0; + lengths[field] = 0; +} + +char16_t SkeletonFields::getFieldChar(int32_t field) const { + return chars[field]; +} + +int32_t SkeletonFields::getFieldLength(int32_t field) const { + return lengths[field]; +} + +void SkeletonFields::populate(int32_t field, const UnicodeString& value) { + populate(field, value.charAt(0), value.length()); +} + +void SkeletonFields::populate(int32_t field, char16_t ch, int32_t length) { + chars[field] = (int8_t) ch; + lengths[field] = (int8_t) length; +} + +UBool SkeletonFields::isFieldEmpty(int32_t field) const { + return lengths[field] == 0; +} + +UnicodeString& SkeletonFields::appendTo(UnicodeString& string) const { + for (int32_t i = 0; i < UDATPG_FIELD_COUNT; ++i) { + appendFieldTo(i, string); + } + return string; +} + +UnicodeString& SkeletonFields::appendFieldTo(int32_t field, UnicodeString& string) const { + char16_t ch(chars[field]); + int32_t length = (int32_t) lengths[field]; + + for (int32_t i=0; i= 0) { + // for backward compatibility: if DateTimeMatcher.set added a single 'a' that + // was not in the provided skeleton, remove it here before returning skeleton. + result.remove(pos, 1); + } + return result; +} + +UnicodeString +PtnSkeleton::getBaseSkeleton() const { + UnicodeString result; + result = baseOriginal.appendTo(result); + int32_t pos; + if (addedDefaultDayPeriod && (pos = result.indexOf(LOW_A)) >= 0) { + // for backward compatibility: if DateTimeMatcher.set added a single 'a' that + // was not in the provided skeleton, remove it here before returning skeleton. + result.remove(pos, 1); + } + return result; +} + +char16_t +PtnSkeleton::getFirstChar() const { + return baseOriginal.getFirstChar(); +} + +PtnSkeleton::~PtnSkeleton() { +} + +PtnElem::PtnElem(const UnicodeString &basePat, const UnicodeString &pat) : + basePattern(basePat), skeleton(nullptr), pattern(pat), next(nullptr) +{ +} + +PtnElem::~PtnElem() { +} + +DTSkeletonEnumeration::DTSkeletonEnumeration(PatternMap& patternMap, dtStrEnum type, UErrorCode& status) : fSkeletons(nullptr) { + PtnElem *curElem; + PtnSkeleton *curSkeleton; + UnicodeString s; + int32_t bootIndex; + + pos=0; + fSkeletons.adoptInsteadAndCheckErrorCode(new UVector(status), status); + if (U_FAILURE(status)) { + return; + } + + for (bootIndex=0; bootIndexbasePattern; + break; + case DT_PATTERN: + s=curElem->pattern; + break; + case DT_SKELETON: + curSkeleton=curElem->skeleton.getAlias(); + s=curSkeleton->getSkeleton(); + break; + } + if ( !isCanonicalItem(s) ) { + LocalPointer newElem(s.clone(), status); + if (U_FAILURE(status)) { + return; + } + fSkeletons->addElement(newElem.getAlias(), status); + if (U_FAILURE(status)) { + fSkeletons.adoptInstead(nullptr); + return; + } + newElem.orphan(); // fSkeletons vector now owns the UnicodeString (although it + // does not use a deleter function to manage the ownership). + } + curElem = curElem->next.getAlias(); + } + } + if ((bootIndex==MAX_PATTERN_ENTRIES) && (curElem!=nullptr) ) { + status = U_BUFFER_OVERFLOW_ERROR; + } +} + +const UnicodeString* +DTSkeletonEnumeration::snext(UErrorCode& status) { + if (U_SUCCESS(status) && fSkeletons.isValid() && pos < fSkeletons->size()) { + return (const UnicodeString*)fSkeletons->elementAt(pos++); + } + return nullptr; +} + +void +DTSkeletonEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +DTSkeletonEnumeration::count(UErrorCode& /*status*/) const { + return (fSkeletons.isNull()) ? 0 : fSkeletons->size(); +} + +UBool +DTSkeletonEnumeration::isCanonicalItem(const UnicodeString& item) { + if ( item.length() != 1 ) { + return false; + } + for (int32_t i=0; isize(); ++i) { + if ((s = (UnicodeString *)fSkeletons->elementAt(i)) != nullptr) { + delete s; + } + } + } +} + +DTRedundantEnumeration::DTRedundantEnumeration() : pos(0), fPatterns(nullptr) { +} + +void +DTRedundantEnumeration::add(const UnicodeString& pattern, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + if (fPatterns.isNull()) { + fPatterns.adoptInsteadAndCheckErrorCode(new UVector(status), status); + if (U_FAILURE(status)) { + return; + } + } + LocalPointer newElem(new UnicodeString(pattern), status); + if (U_FAILURE(status)) { + return; + } + fPatterns->addElement(newElem.getAlias(), status); + if (U_FAILURE(status)) { + fPatterns.adoptInstead(nullptr); + return; + } + newElem.orphan(); // fPatterns now owns the string, although a UVector + // deleter function is not used to manage that ownership. +} + +const UnicodeString* +DTRedundantEnumeration::snext(UErrorCode& status) { + if (U_SUCCESS(status) && fPatterns.isValid() && pos < fPatterns->size()) { + return (const UnicodeString*)fPatterns->elementAt(pos++); + } + return nullptr; +} + +void +DTRedundantEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +DTRedundantEnumeration::count(UErrorCode& /*status*/) const { + return (fPatterns.isNull()) ? 0 : fPatterns->size(); +} + +UBool +DTRedundantEnumeration::isCanonicalItem(const UnicodeString& item) const { + if ( item.length() != 1 ) { + return false; + } + for (int32_t i=0; isize(); ++i) { + if ((s = (UnicodeString *)fPatterns->elementAt(i)) != nullptr) { + delete s; + } + } + } +} + +U_NAMESPACE_END + + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/dtptngen_impl.h b/intl/icu/source/i18n/dtptngen_impl.h new file mode 100644 index 0000000000..74fe925b2f --- /dev/null +++ b/intl/icu/source/i18n/dtptngen_impl.h @@ -0,0 +1,310 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File DTPTNGEN.H +* +******************************************************************************* +*/ + +#ifndef __DTPTNGEN_IMPL_H__ +#define __DTPTNGEN_IMPL_H__ + +#include "unicode/udatpg.h" + +#include "unicode/strenum.h" +#include "unicode/unistr.h" +#include "uvector.h" + +// TODO(claireho): Split off Builder class. +// TODO(claireho): If splitting off Builder class: As subclass or independent? + +#define MAX_PATTERN_ENTRIES 52 +#define MAX_CLDR_FIELD_LEN 60 +#define MAX_DT_TOKEN 50 +#define MAX_RESOURCE_FIELD 12 +#define MAX_AVAILABLE_FORMATS 12 +#define NONE 0 +#define EXTRA_FIELD 0x10000 +#define MISSING_FIELD 0x1000 +#define MAX_STRING_ENUMERATION 200 +#define SINGLE_QUOTE ((char16_t)0x0027) +#define FORWARDSLASH ((char16_t)0x002F) +#define BACKSLASH ((char16_t)0x005C) +#define SPACE ((char16_t)0x0020) +#define QUOTATION_MARK ((char16_t)0x0022) +#define ASTERISK ((char16_t)0x002A) +#define PLUSSITN ((char16_t)0x002B) +#define COMMA ((char16_t)0x002C) +#define HYPHEN ((char16_t)0x002D) +#define DOT ((char16_t)0x002E) +#define COLON ((char16_t)0x003A) +#define CAP_A ((char16_t)0x0041) +#define CAP_B ((char16_t)0x0042) +#define CAP_C ((char16_t)0x0043) +#define CAP_D ((char16_t)0x0044) +#define CAP_E ((char16_t)0x0045) +#define CAP_F ((char16_t)0x0046) +#define CAP_G ((char16_t)0x0047) +#define CAP_H ((char16_t)0x0048) +#define CAP_J ((char16_t)0x004A) +#define CAP_K ((char16_t)0x004B) +#define CAP_L ((char16_t)0x004C) +#define CAP_M ((char16_t)0x004D) +#define CAP_O ((char16_t)0x004F) +#define CAP_Q ((char16_t)0x0051) +#define CAP_S ((char16_t)0x0053) +#define CAP_T ((char16_t)0x0054) +#define CAP_U ((char16_t)0x0055) +#define CAP_V ((char16_t)0x0056) +#define CAP_W ((char16_t)0x0057) +#define CAP_X ((char16_t)0x0058) +#define CAP_Y ((char16_t)0x0059) +#define CAP_Z ((char16_t)0x005A) +#define LOWLINE ((char16_t)0x005F) +#define LOW_A ((char16_t)0x0061) +#define LOW_B ((char16_t)0x0062) +#define LOW_C ((char16_t)0x0063) +#define LOW_D ((char16_t)0x0064) +#define LOW_E ((char16_t)0x0065) +#define LOW_F ((char16_t)0x0066) +#define LOW_G ((char16_t)0x0067) +#define LOW_H ((char16_t)0x0068) +#define LOW_I ((char16_t)0x0069) +#define LOW_J ((char16_t)0x006A) +#define LOW_K ((char16_t)0x006B) +#define LOW_L ((char16_t)0x006C) +#define LOW_M ((char16_t)0x006D) +#define LOW_N ((char16_t)0x006E) +#define LOW_O ((char16_t)0x006F) +#define LOW_P ((char16_t)0x0070) +#define LOW_Q ((char16_t)0x0071) +#define LOW_R ((char16_t)0x0072) +#define LOW_S ((char16_t)0x0073) +#define LOW_T ((char16_t)0x0074) +#define LOW_U ((char16_t)0x0075) +#define LOW_V ((char16_t)0x0076) +#define LOW_W ((char16_t)0x0077) +#define LOW_X ((char16_t)0x0078) +#define LOW_Y ((char16_t)0x0079) +#define LOW_Z ((char16_t)0x007A) +#define DT_NARROW -0x101 +#define DT_SHORTER -0x102 +#define DT_SHORT -0x103 +#define DT_LONG -0x104 +#define DT_NUMERIC 0x100 +#define DT_DELTA 0x10 + +U_NAMESPACE_BEGIN + +const int32_t UDATPG_FRACTIONAL_MASK = 1< skeleton; + UnicodeString pattern; + UBool skeletonWasSpecified; // if specified in availableFormats, not derived + LocalPointer next; + + PtnElem(const UnicodeString &basePattern, const UnicodeString &pattern); + virtual ~PtnElem(); +}; + +class FormatParser : public UMemory { +public: + UnicodeString items[MAX_DT_TOKEN]; + int32_t itemNumber; + + FormatParser(); + virtual ~FormatParser(); + void set(const UnicodeString& patternString); + void getQuoteLiteral(UnicodeString& quote, int32_t *itemIndex); + UBool isPatternSeparator(const UnicodeString& field) const; + static UBool isQuoteLiteral(const UnicodeString& s); + static int32_t getCanonicalIndex(const UnicodeString& s) { return getCanonicalIndex(s, true); } + static int32_t getCanonicalIndex(const UnicodeString& s, UBool strict); + +private: + typedef enum TokenStatus { + START, + ADD_TOKEN, + SYNTAX_ERROR, + DONE + } TokenStatus; + + TokenStatus status; + virtual TokenStatus setTokens(const UnicodeString& pattern, int32_t startPos, int32_t *len); +}; + +class DistanceInfo : public UMemory { +public: + int32_t missingFieldMask; + int32_t extraFieldMask; + + DistanceInfo() {} + virtual ~DistanceInfo(); + void clear() { missingFieldMask = extraFieldMask = 0; } + void setTo(const DistanceInfo& other); + void addMissing(int32_t field) { missingFieldMask |= (1< matcher; + PatternMap *patternMap; +}; + +class DTSkeletonEnumeration : public StringEnumeration { +public: + DTSkeletonEnumeration(PatternMap& patternMap, dtStrEnum type, UErrorCode& status); + virtual ~DTSkeletonEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; +private: + int32_t pos; + UBool isCanonicalItem(const UnicodeString& item); + LocalPointer fSkeletons; +}; + +class DTRedundantEnumeration : public StringEnumeration { +public: + DTRedundantEnumeration(); + virtual ~DTRedundantEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; + void add(const UnicodeString &pattern, UErrorCode& status); +private: + int32_t pos; + UBool isCanonicalItem(const UnicodeString& item) const; + LocalPointer fPatterns; +}; + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/dtrule.cpp b/intl/icu/source/i18n/dtrule.cpp new file mode 100644 index 0000000000..7322cbfdad --- /dev/null +++ b/intl/icu/source/i18n/dtrule.cpp @@ -0,0 +1,141 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2012, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/dtrule.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DateTimeRule) + +DateTimeRule::DateTimeRule(int32_t month, + int32_t dayOfMonth, + int32_t millisInDay, + TimeRuleType timeType) +: fMonth(month), fDayOfMonth(dayOfMonth), fDayOfWeek(0), fWeekInMonth(0), fMillisInDay(millisInDay), + fDateRuleType(DateTimeRule::DOM), fTimeRuleType(timeType) { +} + +DateTimeRule::DateTimeRule(int32_t month, + int32_t weekInMonth, + int32_t dayOfWeek, + int32_t millisInDay, + TimeRuleType timeType) +: fMonth(month), fDayOfMonth(0), fDayOfWeek(dayOfWeek), fWeekInMonth(weekInMonth), fMillisInDay(millisInDay), + fDateRuleType(DateTimeRule::DOW), fTimeRuleType(timeType) { +} + +DateTimeRule::DateTimeRule(int32_t month, + int32_t dayOfMonth, + int32_t dayOfWeek, + UBool after, + int32_t millisInDay, + TimeRuleType timeType) +: UObject(), + fMonth(month), fDayOfMonth(dayOfMonth), fDayOfWeek(dayOfWeek), fWeekInMonth(0), fMillisInDay(millisInDay), + fTimeRuleType(timeType) { + if (after) { + fDateRuleType = DateTimeRule::DOW_GEQ_DOM; + } else { + fDateRuleType = DateTimeRule::DOW_LEQ_DOM; + } +} + +DateTimeRule::DateTimeRule(const DateTimeRule& source) +: UObject(source), + fMonth(source.fMonth), fDayOfMonth(source.fDayOfMonth), fDayOfWeek(source.fDayOfWeek), + fWeekInMonth(source.fWeekInMonth), fMillisInDay(source.fMillisInDay), + fDateRuleType(source.fDateRuleType), fTimeRuleType(source.fTimeRuleType) { +} + +DateTimeRule::~DateTimeRule() { +} + +DateTimeRule* +DateTimeRule::clone() const { + return new DateTimeRule(*this); +} + +DateTimeRule& +DateTimeRule::operator=(const DateTimeRule& right) { + if (this != &right) { + fMonth = right.fMonth; + fDayOfMonth = right.fDayOfMonth; + fDayOfWeek = right.fDayOfWeek; + fWeekInMonth = right.fWeekInMonth; + fMillisInDay = right.fMillisInDay; + fDateRuleType = right.fDateRuleType; + fTimeRuleType = right.fTimeRuleType; + } + return *this; +} + +bool +DateTimeRule::operator==(const DateTimeRule& that) const { + return ((this == &that) || + (typeid(*this) == typeid(that) && + fMonth == that.fMonth && + fDayOfMonth == that.fDayOfMonth && + fDayOfWeek == that.fDayOfWeek && + fWeekInMonth == that.fWeekInMonth && + fMillisInDay == that.fMillisInDay && + fDateRuleType == that.fDateRuleType && + fTimeRuleType == that.fTimeRuleType)); +} + +bool +DateTimeRule::operator!=(const DateTimeRule& that) const { + return !operator==(that); +} + +DateTimeRule::DateRuleType +DateTimeRule::getDateRuleType() const { + return fDateRuleType; +} + +DateTimeRule::TimeRuleType +DateTimeRule::getTimeRuleType() const { + return fTimeRuleType; +} + +int32_t +DateTimeRule::getRuleMonth() const { + return fMonth; +} + +int32_t +DateTimeRule::getRuleDayOfMonth() const { + return fDayOfMonth; +} + +int32_t +DateTimeRule::getRuleDayOfWeek() const { + return fDayOfWeek; +} + +int32_t +DateTimeRule::getRuleWeekInMonth() const { + return fWeekInMonth; +} + +int32_t +DateTimeRule::getRuleMillisInDay() const { + return fMillisInDay; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/erarules.cpp b/intl/icu/source/i18n/erarules.cpp new file mode 100644 index 0000000000..65405bb84a --- /dev/null +++ b/intl/icu/source/i18n/erarules.cpp @@ -0,0 +1,326 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include +#include "unicode/ucal.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "unicode/timezone.h" +#include "cmemory.h" +#include "cstring.h" +#include "erarules.h" +#include "gregoimp.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +static const int32_t MAX_ENCODED_START_YEAR = 32767; +static const int32_t MIN_ENCODED_START_YEAR = -32768; +static const int32_t MIN_ENCODED_START = -2147483391; // encodeDate(MIN_ENCODED_START_YEAR, 1, 1, ...); + +static const int32_t YEAR_MASK = 0xFFFF0000; +static const int32_t MONTH_MASK = 0x0000FF00; +static const int32_t DAY_MASK = 0x000000FF; + +static const int32_t MAX_INT32 = 0x7FFFFFFF; +static const int32_t MIN_INT32 = 0xFFFFFFFF; + +static const char16_t VAL_FALSE[] = {0x66, 0x61, 0x6c, 0x73, 0x65}; // "false" +static const char16_t VAL_FALSE_LEN = 5; + +static UBool isSet(int startDate) { + return startDate != 0; +} + +static UBool isValidRuleStartDate(int32_t year, int32_t month, int32_t day) { + return year >= MIN_ENCODED_START_YEAR && year <= MAX_ENCODED_START_YEAR + && month >= 1 && month <= 12 && day >=1 && day <= 31; +} + +/** + * Encode year/month/date to a single integer. + * year is high 16 bits (-32768 to 32767), month is + * next 8 bits and day of month is last 8 bits. + * + * @param year year + * @param month month (1-base) + * @param day day of month + * @return an encoded date. + */ +static int32_t encodeDate(int32_t year, int32_t month, int32_t day) { + return (int32_t)((uint32_t)year << 16) | month << 8 | day; +} + +static void decodeDate(int32_t encodedDate, int32_t (&fields)[3]) { + if (encodedDate == MIN_ENCODED_START) { + fields[0] = MIN_INT32; + fields[1] = 1; + fields[2] = 1; + } else { + fields[0] = (encodedDate & YEAR_MASK) >> 16; + fields[1] = (encodedDate & MONTH_MASK) >> 8; + fields[2] = encodedDate & DAY_MASK; + } +} + +/** + * Compare an encoded date with another date specified by year/month/day. + * @param encoded An encoded date + * @param year Year of another date + * @param month Month of another date + * @param day Day of another date + * @return -1 when encoded date is earlier, 0 when two dates are same, + * and 1 when encoded date is later. + */ +static int32_t compareEncodedDateWithYMD(int encoded, int year, int month, int day) { + if (year < MIN_ENCODED_START_YEAR) { + if (encoded == MIN_ENCODED_START) { + if (year > MIN_INT32 || month > 1 || day > 1) { + return -1; + } + return 0; + } else { + return 1; + } + } else if (year > MAX_ENCODED_START_YEAR) { + return -1; + } else { + int tmp = encodeDate(year, month, day); + if (encoded < tmp) { + return -1; + } else if (encoded == tmp) { + return 0; + } else { + return 1; + } + } +} + +EraRules::EraRules(LocalMemory& eraStartDates, int32_t numEras) + : numEras(numEras) { + startDates = std::move(eraStartDates); + initCurrentEra(); +} + +EraRules::~EraRules() { +} + +EraRules* EraRules::createInstance(const char *calType, UBool includeTentativeEra, UErrorCode& status) { + if(U_FAILURE(status)) { + return nullptr; + } + LocalUResourceBundlePointer rb(ures_openDirect(nullptr, "supplementalData", &status)); + ures_getByKey(rb.getAlias(), "calendarData", rb.getAlias(), &status); + ures_getByKey(rb.getAlias(), calType, rb.getAlias(), &status); + ures_getByKey(rb.getAlias(), "eras", rb.getAlias(), &status); + + if (U_FAILURE(status)) { + return nullptr; + } + + int32_t numEras = ures_getSize(rb.getAlias()); + int32_t firstTentativeIdx = MAX_INT32; + + LocalMemory startDates(static_cast(uprv_malloc(numEras * sizeof(int32_t)))); + if (startDates.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + uprv_memset(startDates.getAlias(), 0 , numEras * sizeof(int32_t)); + + while (ures_hasNext(rb.getAlias())) { + LocalUResourceBundlePointer eraRuleRes(ures_getNextResource(rb.getAlias(), nullptr, &status)); + if (U_FAILURE(status)) { + return nullptr; + } + const char *eraIdxStr = ures_getKey(eraRuleRes.getAlias()); + char *endp; + int32_t eraIdx = (int32_t)strtol(eraIdxStr, &endp, 10); + if ((size_t)(endp - eraIdxStr) != uprv_strlen(eraIdxStr)) { + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + if (eraIdx < 0 || eraIdx >= numEras) { + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + if (isSet(startDates[eraIdx])) { + // start date of the index was already set + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + + UBool hasName = true; + UBool hasEnd = true; + int32_t len; + while (ures_hasNext(eraRuleRes.getAlias())) { + LocalUResourceBundlePointer res(ures_getNextResource(eraRuleRes.getAlias(), nullptr, &status)); + if (U_FAILURE(status)) { + return nullptr; + } + const char *key = ures_getKey(res.getAlias()); + if (uprv_strcmp(key, "start") == 0) { + const int32_t *fields = ures_getIntVector(res.getAlias(), &len, &status); + if (U_FAILURE(status)) { + return nullptr; + } + if (len != 3 || !isValidRuleStartDate(fields[0], fields[1], fields[2])) { + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + startDates[eraIdx] = encodeDate(fields[0], fields[1], fields[2]); + } else if (uprv_strcmp(key, "named") == 0) { + const char16_t *val = ures_getString(res.getAlias(), &len, &status); + if (u_strncmp(val, VAL_FALSE, VAL_FALSE_LEN) == 0) { + hasName = false; + } + } else if (uprv_strcmp(key, "end") == 0) { + hasEnd = true; + } + } + + if (isSet(startDates[eraIdx])) { + if (hasEnd) { + // This implementation assumes either start or end is available, not both. + // For now, just ignore the end rule. + } + } else { + if (hasEnd) { + if (eraIdx != 0) { + // This implementation does not support end only rule for eras other than + // the first one. + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + U_ASSERT(eraIdx == 0); + startDates[eraIdx] = MIN_ENCODED_START; + } else { + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + } + + if (hasName) { + if (eraIdx >= firstTentativeIdx) { + status = U_INVALID_FORMAT_ERROR; + return nullptr; + } + } else { + if (eraIdx < firstTentativeIdx) { + firstTentativeIdx = eraIdx; + } + } + } + + EraRules *result; + if (firstTentativeIdx < MAX_INT32 && !includeTentativeEra) { + result = new EraRules(startDates, firstTentativeIdx); + } else { + result = new EraRules(startDates, numEras); + } + + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return result; +} + +void EraRules::getStartDate(int32_t eraIdx, int32_t (&fields)[3], UErrorCode& status) const { + if(U_FAILURE(status)) { + return; + } + if (eraIdx < 0 || eraIdx >= numEras) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + decodeDate(startDates[eraIdx], fields); +} + +int32_t EraRules::getStartYear(int32_t eraIdx, UErrorCode& status) const { + int year = MAX_INT32; // bogus value + if(U_FAILURE(status)) { + return year; + } + if (eraIdx < 0 || eraIdx >= numEras) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return year; + } + int fields[3]; + decodeDate(startDates[eraIdx], fields); + year = fields[0]; + + return year; +} + +int32_t EraRules::getEraIndex(int32_t year, int32_t month, int32_t day, UErrorCode& status) const { + if(U_FAILURE(status)) { + return -1; + } + + if (month < 1 || month > 12 || day < 1 || day > 31) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return -1; + } + int32_t high = numEras; // last index + 1 + int32_t low; + + // Short circuit for recent years. Most modern computations will + // occur in the last few eras. + if (compareEncodedDateWithYMD(startDates[getCurrentEraIndex()], year, month, day) <= 0) { + low = getCurrentEraIndex(); + } else { + low = 0; + } + + // Do binary search + while (low < high - 1) { + int i = (low + high) / 2; + if (compareEncodedDateWithYMD(startDates[i], year, month, day) <= 0) { + low = i; + } else { + high = i; + } + } + return low; +} + +void EraRules::initCurrentEra() { + // Compute local wall time in millis using ICU's default time zone. + UErrorCode ec = U_ZERO_ERROR; + UDate localMillis = ucal_getNow(); + + int32_t rawOffset, dstOffset; + TimeZone* zone = TimeZone::createDefault(); + // If we failed to create the default time zone, we are in a bad state and don't + // really have many options. Carry on using UTC millis as a fallback. + if (zone != nullptr) { + zone->getOffset(localMillis, false, rawOffset, dstOffset, ec); + delete zone; + localMillis += (rawOffset + dstOffset); + } + + int year, month0, dom, dow, doy, mid; + Grego::timeToFields(localMillis, year, month0, dom, dow, doy, mid); + int currentEncodedDate = encodeDate(year, month0 + 1 /* changes to 1-base */, dom); + int eraIdx = numEras - 1; + while (eraIdx > 0) { + if (currentEncodedDate >= startDates[eraIdx]) { + break; + } + eraIdx--; + } + // Note: current era could be before the first era. + // In this case, this implementation returns the first era index (0). + currentEra = eraIdx; +} + +U_NAMESPACE_END +#endif /* #if !UCONFIG_NO_FORMATTING */ + + diff --git a/intl/icu/source/i18n/erarules.h b/intl/icu/source/i18n/erarules.h new file mode 100644 index 0000000000..74b7862da4 --- /dev/null +++ b/intl/icu/source/i18n/erarules.h @@ -0,0 +1,99 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#ifndef ERARULES_H_ +#define ERARULES_H_ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/localpointer.h" +#include "unicode/uobject.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +// Export an explicit template instantiation of LocalMemory used as a data member of EraRules. +// When building DLLs for Windows this is required even though no direct access leaks out of the i18n library. +// See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples. +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#if defined(_MSC_VER) +// Ignore warning 4661 as LocalPointerBase does not use operator== or operator!= +#pragma warning(push) +#pragma warning(disable: 4661) +#endif +template class U_I18N_API LocalPointerBase; +template class U_I18N_API LocalMemory; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + +class U_I18N_API EraRules : public UMemory { +public: + ~EraRules(); + + static EraRules* createInstance(const char *calType, UBool includeTentativeEra, UErrorCode& status); + + /** + * Gets number of effective eras + * @return number of effective eras + */ + inline int32_t getNumberOfEras() const { + return numEras; + } + + /** + * Gets start date of an era + * @param eraIdx Era index + * @param fields Receives date fields. The result includes values of year, month, + * day of month in this order. When an era has no start date, the result + * will be January 1st in year whose value is minimum integer. + * @param status Receives status. + */ + void getStartDate(int32_t eraIdx, int32_t (&fields)[3], UErrorCode& status) const; + + /** + * Gets start year of an era + * @param eraIdx Era index + * @param status Receives status. + * @return The first year of an era. When a era has no start date, minimum int32 + * value is returned. + */ + int32_t getStartYear(int32_t eraIdx, UErrorCode& status) const; + + /** + * Returns era index for the specified year/month/day. + * @param year Year + * @param month Month (1-base) + * @param day Day of month + * @param status Receives status + * @return era index (or 0, when the specified date is before the first era) + */ + int32_t getEraIndex(int32_t year, int32_t month, int32_t day, UErrorCode& status) const; + + /** + * Gets the current era index. This is calculated only once for an instance of + * EraRules. The current era calculation is based on the default time zone at + * the time of instantiation. + * + * @return era index of current era (or 0, when current date is before the first era) + */ + inline int32_t getCurrentEraIndex() const { + return currentEra; + } + +private: + EraRules(LocalMemory& eraStartDates, int32_t numEra); + + void initCurrentEra(); + + LocalMemory startDates; + int32_t numEras; + int32_t currentEra; +}; + +U_NAMESPACE_END +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif /* ERARULES_H_ */ diff --git a/intl/icu/source/i18n/esctrn.cpp b/intl/icu/source/i18n/esctrn.cpp new file mode 100644 index 0000000000..24fa6f58e4 --- /dev/null +++ b/intl/icu/source/i18n/esctrn.cpp @@ -0,0 +1,181 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2001-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/19/2001 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/utf16.h" +#include "esctrn.h" +#include "util.h" + +U_NAMESPACE_BEGIN + +static const char16_t UNIPRE[] = {85,43,0}; // "U+" +static const char16_t BS_u[] = {92,117,0}; // "\\u" +static const char16_t BS_U[] = {92,85,0}; // "\\U" +static const char16_t XMLPRE[] = {38,35,120,0}; // "&#x" +static const char16_t XML10PRE[] = {38,35,0}; // "&#" +static const char16_t PERLPRE[] = {92,120,123,0}; // "\\x{" +static const char16_t SEMI[] = {59,0}; // ";" +static const char16_t RBRACE[] = {125,0}; // "}" + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(EscapeTransliterator) + +/** + * Factory methods + */ +static Transliterator* _createEscUnicode(const UnicodeString& ID, Transliterator::Token /*context*/) { + // Unicode: "U+10FFFF" hex, min=4, max=6 + return new EscapeTransliterator(ID, UnicodeString(true, UNIPRE, 2), UnicodeString(), 16, 4, true, nullptr); +} +static Transliterator* _createEscJava(const UnicodeString& ID, Transliterator::Token /*context*/) { + // Java: "\\uFFFF" hex, min=4, max=4 + return new EscapeTransliterator(ID, UnicodeString(true, BS_u, 2), UnicodeString(), 16, 4, false, nullptr); +} +static Transliterator* _createEscC(const UnicodeString& ID, Transliterator::Token /*context*/) { + // C: "\\uFFFF" hex, min=4, max=4; \\U0010FFFF hex, min=8, max=8 + return new EscapeTransliterator(ID, UnicodeString(true, BS_u, 2), UnicodeString(), 16, 4, true, + new EscapeTransliterator(UnicodeString(), UnicodeString(true, BS_U, 2), UnicodeString(), 16, 8, true, nullptr)); +} +static Transliterator* _createEscXML(const UnicodeString& ID, Transliterator::Token /*context*/) { + // XML: "􏿿" hex, min=1, max=6 + return new EscapeTransliterator(ID, UnicodeString(true, XMLPRE, 3), UnicodeString(SEMI[0]), 16, 1, true, nullptr); +} +static Transliterator* _createEscXML10(const UnicodeString& ID, Transliterator::Token /*context*/) { + // XML10: "&1114111;" dec, min=1, max=7 (not really "Any-Hex") + return new EscapeTransliterator(ID, UnicodeString(true, XML10PRE, 2), UnicodeString(SEMI[0]), 10, 1, true, nullptr); +} +static Transliterator* _createEscPerl(const UnicodeString& ID, Transliterator::Token /*context*/) { + // Perl: "\\x{263A}" hex, min=1, max=6 + return new EscapeTransliterator(ID, UnicodeString(true, PERLPRE, 3), UnicodeString(RBRACE[0]), 16, 1, true, nullptr); +} + +/** + * Registers standard variants with the system. Called by + * Transliterator during initialization. + */ +void EscapeTransliterator::registerIDs() { + Token t = integerToken(0); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/Unicode"), _createEscUnicode, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/Java"), _createEscJava, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/C"), _createEscC, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/XML"), _createEscXML, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/XML10"), _createEscXML10, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex/Perl"), _createEscPerl, t); + + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-Hex"), _createEscJava, t); +} + +/** + * Constructs an escape transliterator with the given ID and + * parameters. See the class member documentation for details. + */ +EscapeTransliterator::EscapeTransliterator(const UnicodeString& newID, + const UnicodeString& _prefix, const UnicodeString& _suffix, + int32_t _radix, int32_t _minDigits, + UBool _grokSupplementals, + EscapeTransliterator* adoptedSupplementalHandler) : + Transliterator(newID, nullptr) +{ + this->prefix = _prefix; + this->suffix = _suffix; + this->radix = _radix; + this->minDigits = _minDigits; + this->grokSupplementals = _grokSupplementals; + this->supplementalHandler = adoptedSupplementalHandler; +} + +/** + * Copy constructor. + */ +EscapeTransliterator::EscapeTransliterator(const EscapeTransliterator& o) : + Transliterator(o), + prefix(o.prefix), + suffix(o.suffix), + radix(o.radix), + minDigits(o.minDigits), + grokSupplementals(o.grokSupplementals) { + supplementalHandler = (o.supplementalHandler != 0) ? + new EscapeTransliterator(*o.supplementalHandler) : nullptr; +} + +EscapeTransliterator::~EscapeTransliterator() { + delete supplementalHandler; +} + +/** + * Transliterator API. + */ +EscapeTransliterator* EscapeTransliterator::clone() const { + return new EscapeTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void EscapeTransliterator::handleTransliterate(Replaceable& text, + UTransPosition& pos, + UBool /*isIncremental*/) const +{ + /* TODO: Verify that isIncremental can be ignored */ + int32_t start = pos.start; + int32_t limit = pos.limit; + + UnicodeString buf(prefix); + int32_t prefixLen = prefix.length(); + UBool redoPrefix = false; + + while (start < limit) { + int32_t c = grokSupplementals ? text.char32At(start) : text.charAt(start); + int32_t charLen = grokSupplementals ? U16_LENGTH(c) : 1; + + if ((c & 0xFFFF0000) != 0 && supplementalHandler != nullptr) { + buf.truncate(0); + buf.append(supplementalHandler->prefix); + ICU_Utility::appendNumber(buf, c, supplementalHandler->radix, + supplementalHandler->minDigits); + buf.append(supplementalHandler->suffix); + redoPrefix = true; + } else { + if (redoPrefix) { + buf.truncate(0); + buf.append(prefix); + redoPrefix = false; + } else { + buf.truncate(prefixLen); + } + ICU_Utility::appendNumber(buf, c, radix, minDigits); + buf.append(suffix); + } + + text.handleReplaceBetween(start, start + charLen, buf); + start += buf.length(); + limit += buf.length() - charLen; + } + + pos.contextLimit += limit - pos.limit; + pos.limit = limit; + pos.start = start; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +//eof diff --git a/intl/icu/source/i18n/esctrn.h b/intl/icu/source/i18n/esctrn.h new file mode 100644 index 0000000000..a4282ea86a --- /dev/null +++ b/intl/icu/source/i18n/esctrn.h @@ -0,0 +1,144 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/20/2001 aliu Creation. +********************************************************************** +*/ +#ifndef ESCTRN_H +#define ESCTRN_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that converts Unicode characters to an escape + * form. Examples of escape forms are "U+4E01" and "􏿿". + * Escape forms have a prefix and suffix, either of which may be + * empty, a radix, typically 16 or 10, a minimum digit count, + * typically 1, 4, or 8, and a boolean that specifies whether + * supplemental characters are handled as 32-bit code points or as two + * 16-bit code units. Most escape forms handle 32-bit code points, + * but some, such as the Java form, intentionally break them into two + * surrogate pairs, for backward compatibility. + * + *

Some escape forms actually have two different patterns, one for + * BMP characters (0..FFFF) and one for supplements (>FFFF). To + * handle this, a second EscapeTransliterator may be defined that + * specifies the pattern to be produced for supplementals. An example + * of a form that requires this is the C form, which uses "\\uFFFF" + * for BMP characters and "\\U0010FFFF" for supplementals. + * + *

This class is package private. It registers several standard + * variants with the system which are then accessed via their IDs. + * + * @author Alan Liu + */ +class EscapeTransliterator : public Transliterator { + + private: + + /** + * The prefix of the escape form; may be empty, but usually isn't. + */ + UnicodeString prefix; + + /** + * The prefix of the escape form; often empty. + */ + UnicodeString suffix; + + /** + * The radix to display the number in. Typically 16 or 10. Must + * be in the range 2 to 36. + */ + int32_t radix; + + /** + * The minimum number of digits. Typically 1, 4, or 8. Values + * less than 1 are equivalent to 1. + */ + int32_t minDigits; + + /** + * If true, supplementals are handled as 32-bit code points. If + * false, they are handled as two 16-bit code units. + */ + UBool grokSupplementals; + + /** + * The form to be used for supplementals. If this is null then + * the same form is used for BMP characters and supplementals. If + * this is not null and if grokSupplementals is true then the + * prefix, suffix, radix, and minDigits of this object are used + * for supplementals. This pointer is owned. + */ + EscapeTransliterator* supplementalHandler; + + public: + + /** + * Registers standard variants with the system. Called by + * Transliterator during initialization. + */ + static void registerIDs(); + + /** + * Constructs an escape transliterator with the given ID and + * parameters. See the class member documentation for details. + */ + EscapeTransliterator(const UnicodeString& ID, + const UnicodeString& prefix, const UnicodeString& suffix, + int32_t radix, int32_t minDigits, + UBool grokSupplementals, + EscapeTransliterator* adoptedSupplementalHandler); + + /** + * Copy constructor. + */ + EscapeTransliterator(const EscapeTransliterator&); + + /** + * Destructor. + */ + virtual ~EscapeTransliterator(); + + /** + * Transliterator API. + */ + virtual EscapeTransliterator* clone() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + protected: + + /** + * Implements {@link Transliterator#handleTransliterate}. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/ethpccal.cpp b/intl/icu/source/i18n/ethpccal.cpp new file mode 100644 index 0000000000..be4010843a --- /dev/null +++ b/intl/icu/source/i18n/ethpccal.cpp @@ -0,0 +1,259 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2013, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "umutex.h" +#include "ethpccal.h" +#include "cecal.h" +#include + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(EthiopicCalendar) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(EthiopicAmeteAlemCalendar) + +//static const int32_t JD_EPOCH_OFFSET_AMETE_ALEM = -285019; +static const int32_t JD_EPOCH_OFFSET_AMETE_MIHRET = 1723856; +static const int32_t AMETE_MIHRET_DELTA = 5500; // 5501 - 1 (Amete Alem 5501 = Amete Mihret 1) + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +EthiopicCalendar::EthiopicCalendar(const Locale& aLocale, + UErrorCode& success) +: CECalendar(aLocale, success) +{ +} + +EthiopicCalendar::~EthiopicCalendar() +{ +} + +EthiopicCalendar* +EthiopicCalendar::clone() const +{ + return new EthiopicCalendar(*this); +} + +const char * +EthiopicCalendar::getType() const +{ + return "ethiopic"; +} + +//------------------------------------------------------------------------- +// Calendar framework +//------------------------------------------------------------------------- + +int32_t +EthiopicCalendar::handleGetExtendedYear() +{ + // Ethiopic calendar uses EXTENDED_YEAR aligned to + // Amelete Hihret year always. + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + return internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } + // The year defaults to the epoch start, the era to AMETE_MIHRET + if (internalGet(UCAL_ERA, AMETE_MIHRET) == AMETE_MIHRET) { + return internalGet(UCAL_YEAR, 1); // Default to year 1 + } + return internalGet(UCAL_YEAR, 1) - AMETE_MIHRET_DELTA; +} + +void +EthiopicCalendar::handleComputeFields(int32_t julianDay, UErrorCode &/*status*/) +{ + int32_t eyear, month, day; + jdToCE(julianDay, getJDEpochOffset(), eyear, month, day); + + internalSet(UCAL_EXTENDED_YEAR, eyear); + internalSet(UCAL_ERA, (eyear > 0) ? AMETE_MIHRET : AMETE_ALEM); + internalSet(UCAL_YEAR, (eyear > 0) ? eyear : (eyear + AMETE_MIHRET_DELTA)); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DATE, day); + internalSet(UCAL_DAY_OF_YEAR, (30 * month) + day); +} + +constexpr uint32_t kEthiopicRelatedYearDiff = 8; + +int32_t EthiopicCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kEthiopicRelatedYearDiff; +} + +void EthiopicCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kEthiopicRelatedYearDiff); +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + +static void U_CALLCONV initializeSystemDefaultCentury() +{ + UErrorCode status = U_ZERO_ERROR; + EthiopicCalendar calendar(Locale("@calendar=ethiopic"), status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate +EthiopicCalendar::defaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t +EthiopicCalendar::defaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + + +int32_t +EthiopicCalendar::getJDEpochOffset() const +{ + return JD_EPOCH_OFFSET_AMETE_MIHRET; +} + + +#if 0 +// We do not want to introduce this API in ICU4C. +// It was accidentally introduced in ICU4J as a public API. + +//------------------------------------------------------------------------- +// Calendar system Conversion methods... +//------------------------------------------------------------------------- + +int32_t +EthiopicCalendar::ethiopicToJD(int32_t year, int32_t month, int32_t date) +{ + return ceToJD(year, month, date, JD_EPOCH_OFFSET_AMETE_MIHRET); +} +#endif + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +EthiopicAmeteAlemCalendar::EthiopicAmeteAlemCalendar(const Locale& aLocale, + UErrorCode& success) +: EthiopicCalendar(aLocale, success) +{ +} + +EthiopicAmeteAlemCalendar::~EthiopicAmeteAlemCalendar() +{ +} + +EthiopicAmeteAlemCalendar* +EthiopicAmeteAlemCalendar::clone() const +{ + return new EthiopicAmeteAlemCalendar(*this); +} + +//------------------------------------------------------------------------- +// Calendar framework +//------------------------------------------------------------------------- + +const char * +EthiopicAmeteAlemCalendar::getType() const +{ + return "ethiopic-amete-alem"; +} + +int32_t +EthiopicAmeteAlemCalendar::handleGetExtendedYear() +{ + // Ethiopic calendar uses EXTENDED_YEAR aligned to + // Amelete Hihret year always. + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + return internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } + return internalGet(UCAL_YEAR, 1 + AMETE_MIHRET_DELTA) + - AMETE_MIHRET_DELTA; // Default to year 1 of Amelete Mihret +} + +void +EthiopicAmeteAlemCalendar::handleComputeFields(int32_t julianDay, UErrorCode &/*status*/) +{ + int32_t eyear, month, day; + jdToCE(julianDay, getJDEpochOffset(), eyear, month, day); + + internalSet(UCAL_EXTENDED_YEAR, eyear); + internalSet(UCAL_ERA, AMETE_ALEM); + internalSet(UCAL_YEAR, eyear + AMETE_MIHRET_DELTA); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DATE, day); + internalSet(UCAL_DAY_OF_YEAR, (30 * month) + day); +} + +int32_t +EthiopicAmeteAlemCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const +{ + if (field == UCAL_ERA) { + return 0; // Only one era in this mode, era is always 0 + } + return EthiopicCalendar::handleGetLimit(field, limitType); +} + +constexpr uint32_t kEthiopicAmeteAlemRelatedYearDiff = -5492; + +int32_t EthiopicAmeteAlemCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kEthiopicAmeteAlemRelatedYearDiff; +} + +void EthiopicAmeteAlemCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kEthiopicAmeteAlemRelatedYearDiff); +} + +int32_t +EthiopicAmeteAlemCalendar::defaultCenturyStartYear() const +{ + return EthiopicCalendar::defaultCenturyStartYear() + AMETE_MIHRET_DELTA; +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/ethpccal.h b/intl/icu/source/i18n/ethpccal.h new file mode 100644 index 0000000000..1a5cd4f41a --- /dev/null +++ b/intl/icu/source/i18n/ethpccal.h @@ -0,0 +1,367 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003 - 2013, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef ETHPCCAL_H +#define ETHPCCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "cecal.h" + +U_NAMESPACE_BEGIN + +/** + * Implement the Ethiopic calendar system. + * @internal + */ +class EthiopicCalendar : public CECalendar { + +public: + /** + * Useful constants for EthiopicCalendar. + * @internal + */ + enum EMonths { + /** + * Constant for መስከረም, the 1st month of the Ethiopic year. + */ + MESKEREM, + + /** + * Constant for ጥቅምት, the 2nd month of the Ethiopic year. + */ + TEKEMT, + + /** + * Constant for ኅዳር, the 3rd month of the Ethiopic year. + */ + HEDAR, + + /** + * Constant for ታኅሣሥ, the 4th month of the Ethiopic year. + */ + TAHSAS, + + /** + * Constant for ጥር, the 5th month of the Ethiopic year. + */ + TER, + + /** + * Constant for የካቲት, the 6th month of the Ethiopic year. + */ + YEKATIT, + + /** + * Constant for መጋቢት, the 7th month of the Ethiopic year. + */ + MEGABIT, + + /** + * Constant for ሚያዝያ, the 8th month of the Ethiopic year. + */ + MIAZIA, + + /** + * Constant for ግንቦት, the 9th month of the Ethiopic year. + */ + GENBOT, + + /** + * Constant for ሰኔ, the 10th month of the Ethiopic year. + */ + SENE, + + /** + * Constant for ሐምሌ, the 11th month of the Ethiopic year. + */ + HAMLE, + + /** + * Constant for ነሐሴ, the 12th month of the Ethiopic year. + */ + NEHASSA, + + /** + * Constant for ጳጉሜን, the 13th month of the Ethiopic year. + */ + PAGUMEN + }; + + enum EEras { + AMETE_ALEM, // Before the epoch + AMETE_MIHRET // After the epoch + }; + + /** + * Constructs a EthiopicCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of EthiopicCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @param type Whether this Ethiopic calendar use Amete Mihrret (default) or + * only use Amete Alem for all the time. + * @internal + */ + EthiopicCalendar(const Locale& aLocale, UErrorCode& success); + + /** + * Copy Constructor + * @internal + */ + EthiopicCalendar(const EthiopicCalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~EthiopicCalendar(); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual EthiopicCalendar* clone() const override; + + /** + * Return the calendar type, "ethiopic" + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + +protected: + //------------------------------------------------------------------------- + // Calendar framework + //------------------------------------------------------------------------- + + /** + * Return the extended year defined by the current fields. + * This calendar uses both AMETE_ALEM and AMETE_MIHRET. + * + * EXTENDED_YEAR ERA YEAR + * 0 AMETE_ALEM 5500 + * 1 AMETE_MIHRET 1 + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Compute fields from the JD + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + + /** + * Return the date offset from Julian + * @internal + */ + virtual int32_t getJDEpochOffset() const override; + +public: + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + +#if 0 +// We do not want to introduce this API in ICU4C. +// It was accidentally introduced in ICU4J as a public API. + +public: + //------------------------------------------------------------------------- + // Calendar system Conversion methods... + //------------------------------------------------------------------------- + + /** + * Convert an Ethiopic year, month, and day to a Julian day. + * + * @param year the extended year + * @param month the month + * @param day the day + * @return Julian day + * @internal + */ + int32_t ethiopicToJD(int32_t year, int32_t month, int32_t day); +#endif +}; + +/** + * Implement the Ethiopic Amete Alem calendar system. + * @internal + */ +class EthiopicAmeteAlemCalendar : public EthiopicCalendar { + +public: + /** + * Constructs a EthiopicAmeteAlemCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of EthiopicCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + EthiopicAmeteAlemCalendar(const Locale& aLocale, UErrorCode& success); + + /** + * Copy Constructor + * @internal + */ + EthiopicAmeteAlemCalendar(const EthiopicAmeteAlemCalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~EthiopicAmeteAlemCalendar(); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual EthiopicAmeteAlemCalendar* clone() const override; + + /** + * Return the calendar type, "ethiopic-amete-alem" + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + +protected: + //------------------------------------------------------------------------- + // Calendar framework + //------------------------------------------------------------------------- + + /** + * Return the extended year defined by the current fields. + * This calendar use only AMETE_ALEM for the era. + * + * EXTENDED_YEAR ERA YEAR + * 0 AMETE_ALEM 5500 + * 1 AMETE_ALEM 5501 + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Compute fields from the JD + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + /** + * Calculate the limit for a specified type of limit and field + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; +}; + +U_NAMESPACE_END +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif /* ETHPCCAL_H */ +//eof diff --git a/intl/icu/source/i18n/fmtable.cpp b/intl/icu/source/i18n/fmtable.cpp new file mode 100644 index 0000000000..618868c0a2 --- /dev/null +++ b/intl/icu/source/i18n/fmtable.cpp @@ -0,0 +1,1042 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File FMTABLE.CPP +* +* Modification History: +* +* Date Name Description +* 03/25/97 clhuang Initial Implementation. +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include +#include +#include "unicode/fmtable.h" +#include "unicode/ustring.h" +#include "unicode/measure.h" +#include "unicode/curramt.h" +#include "unicode/uformattable.h" +#include "charstr.h" +#include "cmemory.h" +#include "cstring.h" +#include "fmtableimp.h" +#include "number_decimalquantity.h" + +// ***************************************************************************** +// class Formattable +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Formattable) + +using number::impl::DecimalQuantity; + + +//-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. + +// NOTE: As of 3.0, there are limitations to the UObject API. It does +// not (yet) support cloning, operator=, nor operator==. To +// work around this, I implement some simple inlines here. Later +// these can be modified or removed. [alan] + +// NOTE: These inlines assume that all fObjects are in fact instances +// of the Measure class, which is true as of 3.0. [alan] + +// Return true if *a == *b. +static inline UBool objectEquals(const UObject* a, const UObject* b) { + // LATER: return *a == *b; + return *((const Measure*) a) == *((const Measure*) b); +} + +// Return a clone of *a. +static inline UObject* objectClone(const UObject* a) { + // LATER: return a->clone(); + return ((const Measure*) a)->clone(); +} + +// Return true if *a is an instance of Measure. +static inline UBool instanceOfMeasure(const UObject* a) { + return dynamic_cast(a) != nullptr; +} + +/** + * Creates a new Formattable array and copies the values from the specified + * original. + * @param array the original array + * @param count the original array count + * @return the new Formattable array. + */ +static Formattable* createArrayCopy(const Formattable* array, int32_t count) { + Formattable *result = new Formattable[count]; + if (result != nullptr) { + for (int32_t i=0; i INT32_MAX) { + status = U_INVALID_FORMAT_ERROR; + return INT32_MAX; + } else if (fValue.fInt64 < INT32_MIN) { + status = U_INVALID_FORMAT_ERROR; + return INT32_MIN; + } else { + return (int32_t)fValue.fInt64; + } + case Formattable::kDouble: + if (fValue.fDouble > INT32_MAX) { + status = U_INVALID_FORMAT_ERROR; + return INT32_MAX; + } else if (fValue.fDouble < INT32_MIN) { + status = U_INVALID_FORMAT_ERROR; + return INT32_MIN; + } else { + return (int32_t)fValue.fDouble; // loses fraction + } + case Formattable::kObject: + if (fValue.fObject == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + // TODO Later replace this with instanceof call + if (instanceOfMeasure(fValue.fObject)) { + return ((const Measure*) fValue.fObject)-> + getNumber().getLong(status); + } + U_FALLTHROUGH; + default: + status = U_INVALID_FORMAT_ERROR; + return 0; + } +} + +// ------------------------------------- +// Maximum int that can be represented exactly in a double. (53 bits) +// Larger ints may be rounded to a near-by value as not all are representable. +// TODO: move this constant elsewhere, possibly configure it for different +// floating point formats, if any non-standard ones are still in use. +static const int64_t U_DOUBLE_MAX_EXACT_INT = 9007199254740992LL; + +int64_t +Formattable::getInt64(UErrorCode& status) const +{ + if (U_FAILURE(status)) { + return 0; + } + + switch (fType) { + case Formattable::kLong: + case Formattable::kInt64: + return fValue.fInt64; + case Formattable::kDouble: + if (fValue.fDouble > (double)U_INT64_MAX) { + status = U_INVALID_FORMAT_ERROR; + return U_INT64_MAX; + } else if (fValue.fDouble < (double)U_INT64_MIN) { + status = U_INVALID_FORMAT_ERROR; + return U_INT64_MIN; + } else if (fabs(fValue.fDouble) > U_DOUBLE_MAX_EXACT_INT && fDecimalQuantity != nullptr) { + if (fDecimalQuantity->fitsInLong(true)) { + return fDecimalQuantity->toLong(); + } else { + // Unexpected + status = U_INVALID_FORMAT_ERROR; + return fDecimalQuantity->isNegative() ? U_INT64_MIN : U_INT64_MAX; + } + } else { + return (int64_t)fValue.fDouble; + } + case Formattable::kObject: + if (fValue.fObject == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + if (instanceOfMeasure(fValue.fObject)) { + return ((const Measure*) fValue.fObject)-> + getNumber().getInt64(status); + } + U_FALLTHROUGH; + default: + status = U_INVALID_FORMAT_ERROR; + return 0; + } +} + +// ------------------------------------- +double +Formattable::getDouble(UErrorCode& status) const +{ + if (U_FAILURE(status)) { + return 0; + } + + switch (fType) { + case Formattable::kLong: + case Formattable::kInt64: // loses precision + return (double)fValue.fInt64; + case Formattable::kDouble: + return fValue.fDouble; + case Formattable::kObject: + if (fValue.fObject == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + // TODO Later replace this with instanceof call + if (instanceOfMeasure(fValue.fObject)) { + return ((const Measure*) fValue.fObject)-> + getNumber().getDouble(status); + } + U_FALLTHROUGH; + default: + status = U_INVALID_FORMAT_ERROR; + return 0; + } +} + +const UObject* +Formattable::getObject() const { + return (fType == kObject) ? fValue.fObject : nullptr; +} + +// ------------------------------------- +// Sets the value to a double value d. + +void +Formattable::setDouble(double d) +{ + dispose(); + fType = kDouble; + fValue.fDouble = d; +} + +// ------------------------------------- +// Sets the value to a long value l. + +void +Formattable::setLong(int32_t l) +{ + dispose(); + fType = kLong; + fValue.fInt64 = l; +} + +// ------------------------------------- +// Sets the value to an int64 value ll. + +void +Formattable::setInt64(int64_t ll) +{ + dispose(); + fType = kInt64; + fValue.fInt64 = ll; +} + +// ------------------------------------- +// Sets the value to a Date instance d. + +void +Formattable::setDate(UDate d) +{ + dispose(); + fType = kDate; + fValue.fDate = d; +} + +// ------------------------------------- +// Sets the value to a string value stringToCopy. + +void +Formattable::setString(const UnicodeString& stringToCopy) +{ + dispose(); + fType = kString; + fValue.fString = new UnicodeString(stringToCopy); +} + +// ------------------------------------- +// Sets the value to an array of Formattable objects. + +void +Formattable::setArray(const Formattable* array, int32_t count) +{ + dispose(); + fType = kArray; + fValue.fArrayAndCount.fArray = createArrayCopy(array, count); + fValue.fArrayAndCount.fCount = count; +} + +// ------------------------------------- +// Adopts the stringToAdopt value. + +void +Formattable::adoptString(UnicodeString* stringToAdopt) +{ + dispose(); + fType = kString; + fValue.fString = stringToAdopt; +} + +// ------------------------------------- +// Adopts the array value and its count. + +void +Formattable::adoptArray(Formattable* array, int32_t count) +{ + dispose(); + fType = kArray; + fValue.fArrayAndCount.fArray = array; + fValue.fArrayAndCount.fCount = count; +} + +void +Formattable::adoptObject(UObject* objectToAdopt) { + dispose(); + fType = kObject; + fValue.fObject = objectToAdopt; +} + +// ------------------------------------- +UnicodeString& +Formattable::getString(UnicodeString& result, UErrorCode& status) const +{ + if (fType != kString) { + setError(status, U_INVALID_FORMAT_ERROR); + result.setToBogus(); + } else { + if (fValue.fString == nullptr) { + setError(status, U_MEMORY_ALLOCATION_ERROR); + } else { + result = *fValue.fString; + } + } + return result; +} + +// ------------------------------------- +const UnicodeString& +Formattable::getString(UErrorCode& status) const +{ + if (fType != kString) { + setError(status, U_INVALID_FORMAT_ERROR); + return *getBogus(); + } + if (fValue.fString == nullptr) { + setError(status, U_MEMORY_ALLOCATION_ERROR); + return *getBogus(); + } + return *fValue.fString; +} + +// ------------------------------------- +UnicodeString& +Formattable::getString(UErrorCode& status) +{ + if (fType != kString) { + setError(status, U_INVALID_FORMAT_ERROR); + return *getBogus(); + } + if (fValue.fString == nullptr) { + setError(status, U_MEMORY_ALLOCATION_ERROR); + return *getBogus(); + } + return *fValue.fString; +} + +// ------------------------------------- +const Formattable* +Formattable::getArray(int32_t& count, UErrorCode& status) const +{ + if (fType != kArray) { + setError(status, U_INVALID_FORMAT_ERROR); + count = 0; + return nullptr; + } + count = fValue.fArrayAndCount.fCount; + return fValue.fArrayAndCount.fArray; +} + +// ------------------------------------- +// Gets the bogus string, ensures mondo bogosity. + +UnicodeString* +Formattable::getBogus() const +{ + return (UnicodeString*)&fBogus; /* cast away const :-( */ +} + + +// -------------------------------------- +StringPiece Formattable::getDecimalNumber(UErrorCode &status) { + if (U_FAILURE(status)) { + return ""; + } + if (fDecimalStr != nullptr) { + return fDecimalStr->toStringPiece(); + } + + CharString *decimalStr = internalGetCharString(status); + if(decimalStr == nullptr) { + return ""; // getDecimalNumber returns "" for error cases + } else { + return decimalStr->toStringPiece(); + } +} + +CharString *Formattable::internalGetCharString(UErrorCode &status) { + if(fDecimalStr == nullptr) { + if (fDecimalQuantity == nullptr) { + // No decimal number for the formattable yet. Which means the value was + // set directly by the user as an int, int64 or double. If the value came + // from parsing, or from the user setting a decimal number, fDecimalNum + // would already be set. + // + LocalPointer dq(new DecimalQuantity(), status); + if (U_FAILURE(status)) { return nullptr; } + populateDecimalQuantity(*dq, status); + if (U_FAILURE(status)) { return nullptr; } + fDecimalQuantity = dq.orphan(); + } + + fDecimalStr = new CharString(); + if (fDecimalStr == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + // Older ICUs called uprv_decNumberToString here, which is not exactly the same as + // DecimalQuantity::toScientificString(). The biggest difference is that uprv_decNumberToString does + // not print scientific notation for magnitudes greater than -5 and smaller than some amount (+5?). + if (fDecimalQuantity->isInfinite()) { + fDecimalStr->append("Infinity", status); + } else if (fDecimalQuantity->isNaN()) { + fDecimalStr->append("NaN", status); + } else if (fDecimalQuantity->isZeroish()) { + fDecimalStr->append("0", -1, status); + } else if (fType==kLong || fType==kInt64 || // use toPlainString for integer types + (fDecimalQuantity->getMagnitude() != INT32_MIN && std::abs(fDecimalQuantity->getMagnitude()) < 5)) { + fDecimalStr->appendInvariantChars(fDecimalQuantity->toPlainString(), status); + } else { + fDecimalStr->appendInvariantChars(fDecimalQuantity->toScientificString(), status); + } + } + return fDecimalStr; +} + +void +Formattable::populateDecimalQuantity(number::impl::DecimalQuantity& output, UErrorCode& status) const { + if (fDecimalQuantity != nullptr) { + output = *fDecimalQuantity; + return; + } + + switch (fType) { + case kDouble: + output.setToDouble(this->getDouble()); + output.roundToInfinity(); + break; + case kLong: + output.setToInt(this->getLong()); + break; + case kInt64: + output.setToLong(this->getInt64()); + break; + default: + // The formattable's value is not a numeric type. + status = U_INVALID_STATE_ERROR; + } +} + +// --------------------------------------- +void +Formattable::adoptDecimalQuantity(DecimalQuantity *dq) { + if (fDecimalQuantity != nullptr) { + delete fDecimalQuantity; + } + fDecimalQuantity = dq; + if (dq == nullptr) { // allow adoptDigitList(nullptr) to clear + return; + } + + // Set the value into the Union of simple type values. + // Cannot use the set() functions because they would delete the fDecimalNum value. + if (fDecimalQuantity->fitsInLong()) { + fValue.fInt64 = fDecimalQuantity->toLong(); + if (fValue.fInt64 <= INT32_MAX && fValue.fInt64 >= INT32_MIN) { + fType = kLong; + } else { + fType = kInt64; + } + } else { + fType = kDouble; + fValue.fDouble = fDecimalQuantity->toDouble(); + } +} + + +// --------------------------------------- +void +Formattable::setDecimalNumber(StringPiece numberString, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + dispose(); + + auto* dq = new DecimalQuantity(); + dq->setToDecNumber(numberString, status); + adoptDecimalQuantity(dq); + + // Note that we do not hang on to the caller's input string. + // If we are asked for the string, we will regenerate one from fDecimalQuantity. +} + +#if 0 +//---------------------------------------------------- +// console I/O +//---------------------------------------------------- +#ifdef _DEBUG + +#include +using namespace std; + +#include "unicode/datefmt.h" +#include "unistrm.h" + +class FormattableStreamer /* not : public UObject because all methods are static */ { +public: + static void streamOut(ostream& stream, const Formattable& obj); + +private: + FormattableStreamer() {} // private - forbid instantiation +}; + +// This is for debugging purposes only. This will send a displayable +// form of the Formattable object to the output stream. + +void +FormattableStreamer::streamOut(ostream& stream, const Formattable& obj) +{ + static DateFormat *defDateFormat = 0; + + UnicodeString buffer; + switch(obj.getType()) { + case Formattable::kDate : + // Creates a DateFormat instance for formatting the + // Date instance. + if (defDateFormat == 0) { + defDateFormat = DateFormat::createInstance(); + } + defDateFormat->format(obj.getDate(), buffer); + stream << buffer; + break; + case Formattable::kDouble : + // Output the double as is. + stream << obj.getDouble() << 'D'; + break; + case Formattable::kLong : + // Output the double as is. + stream << obj.getLong() << 'L'; + break; + case Formattable::kString: + // Output the double as is. Please see UnicodeString console + // I/O routine for more details. + stream << '"' << obj.getString(buffer) << '"'; + break; + case Formattable::kArray: + int32_t i, count; + const Formattable* array; + array = obj.getArray(count); + stream << '['; + // Recursively calling the console I/O routine for each element in the array. + for (i=0; itoUFormattable(); + + if( fmt == nullptr ) { + *status = U_MEMORY_ALLOCATION_ERROR; + } + return fmt; +} + +U_CAPI void U_EXPORT2 +ufmt_close(UFormattable *fmt) { + Formattable *obj = Formattable::fromUFormattable(fmt); + + delete obj; +} + +U_CAPI UFormattableType U_EXPORT2 +ufmt_getType(const UFormattable *fmt, UErrorCode *status) { + if(U_FAILURE(*status)) { + return (UFormattableType)UFMT_COUNT; + } + const Formattable *obj = Formattable::fromUFormattable(fmt); + return (UFormattableType)obj->getType(); +} + + +U_CAPI UBool U_EXPORT2 +ufmt_isNumeric(const UFormattable *fmt) { + const Formattable *obj = Formattable::fromUFormattable(fmt); + return obj->isNumeric(); +} + +U_CAPI UDate U_EXPORT2 +ufmt_getDate(const UFormattable *fmt, UErrorCode *status) { + const Formattable *obj = Formattable::fromUFormattable(fmt); + + return obj->getDate(*status); +} + +U_CAPI double U_EXPORT2 +ufmt_getDouble(UFormattable *fmt, UErrorCode *status) { + Formattable *obj = Formattable::fromUFormattable(fmt); + + return obj->getDouble(*status); +} + +U_CAPI int32_t U_EXPORT2 +ufmt_getLong(UFormattable *fmt, UErrorCode *status) { + Formattable *obj = Formattable::fromUFormattable(fmt); + + return obj->getLong(*status); +} + + +U_CAPI const void *U_EXPORT2 +ufmt_getObject(const UFormattable *fmt, UErrorCode *status) { + const Formattable *obj = Formattable::fromUFormattable(fmt); + + const void *ret = obj->getObject(); + if( ret==nullptr && + (obj->getType() != Formattable::kObject) && + U_SUCCESS( *status )) { + *status = U_INVALID_FORMAT_ERROR; + } + return ret; +} + +U_CAPI const char16_t* U_EXPORT2 +ufmt_getUChars(UFormattable *fmt, int32_t *len, UErrorCode *status) { + Formattable *obj = Formattable::fromUFormattable(fmt); + + // avoid bogosity by checking the type first. + if( obj->getType() != Formattable::kString ) { + if( U_SUCCESS(*status) ){ + *status = U_INVALID_FORMAT_ERROR; + } + return nullptr; + } + + // This should return a valid string + UnicodeString &str = obj->getString(*status); + if( U_SUCCESS(*status) && len != nullptr ) { + *len = str.length(); + } + return str.getTerminatedBuffer(); +} + +U_CAPI int32_t U_EXPORT2 +ufmt_getArrayLength(const UFormattable* fmt, UErrorCode *status) { + const Formattable *obj = Formattable::fromUFormattable(fmt); + + int32_t count; + (void)obj->getArray(count, *status); + return count; +} + +U_CAPI UFormattable * U_EXPORT2 +ufmt_getArrayItemByIndex(UFormattable* fmt, int32_t n, UErrorCode *status) { + Formattable *obj = Formattable::fromUFormattable(fmt); + int32_t count; + (void)obj->getArray(count, *status); + if(U_FAILURE(*status)) { + return nullptr; + } else if(n<0 || n>=count) { + setError(*status, U_INDEX_OUTOFBOUNDS_ERROR); + return nullptr; + } else { + return (*obj)[n].toUFormattable(); // returns non-const Formattable + } +} + +U_CAPI const char * U_EXPORT2 +ufmt_getDecNumChars(UFormattable *fmt, int32_t *len, UErrorCode *status) { + if(U_FAILURE(*status)) { + return ""; + } + Formattable *obj = Formattable::fromUFormattable(fmt); + CharString *charString = obj->internalGetCharString(*status); + if(U_FAILURE(*status)) { + return ""; + } + if(charString == nullptr) { + *status = U_MEMORY_ALLOCATION_ERROR; + return ""; + } else { + if(len!=nullptr) { + *len = charString->length(); + } + return charString->data(); + } +} + +U_CAPI int64_t U_EXPORT2 +ufmt_getInt64(UFormattable *fmt, UErrorCode *status) { + Formattable *obj = Formattable::fromUFormattable(fmt); + return obj->getInt64(*status); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/fmtable_cnv.cpp b/intl/icu/source/i18n/fmtable_cnv.cpp new file mode 100644 index 0000000000..bc3847b696 --- /dev/null +++ b/intl/icu/source/i18n/fmtable_cnv.cpp @@ -0,0 +1,44 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2010, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File FMTABLE.CPP +* +* Modification History: +* +* Date Name Description +* 03/25/97 clhuang Initial Implementation. +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING && !UCONFIG_NO_CONVERSION + +#include "unicode/fmtable.h" + +// ***************************************************************************** +// class Formattable +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +// ------------------------------------- +// Creates a formattable object with a char* string. +// This API is useless. The API that takes a UnicodeString is actually just as good. +Formattable::Formattable(const char* stringToCopy) +{ + init(); + fType = kString; + fValue.fString = new UnicodeString(stringToCopy); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING || !UCONFIG_NO_CONVERSION */ + +//eof diff --git a/intl/icu/source/i18n/fmtableimp.h b/intl/icu/source/i18n/fmtableimp.h new file mode 100644 index 0000000000..2707d6ece2 --- /dev/null +++ b/intl/icu/source/i18n/fmtableimp.h @@ -0,0 +1,31 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2014, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef FMTABLEIMP_H +#define FMTABLEIMP_H + +#include "number_decimalquantity.h" + +#if !UCONFIG_NO_FORMATTING + +U_NAMESPACE_BEGIN + +/** + * Maximum int64_t value that can be stored in a double without chancing losing precision. + * IEEE doubles have 53 bits of mantissa, 10 bits exponent, 1 bit sign. + * IBM Mainframes have 56 bits of mantissa, 7 bits of base 16 exponent, 1 bit sign. + * Define this constant to the smallest value from those for supported platforms. + * @internal + */ +static const int64_t MAX_INT64_IN_DOUBLE = 0x001FFFFFFFFFFFFFLL; + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_FORMATTING +#endif diff --git a/intl/icu/source/i18n/format.cpp b/intl/icu/source/i18n/format.cpp new file mode 100644 index 0000000000..10856a4acb --- /dev/null +++ b/intl/icu/source/i18n/format.cpp @@ -0,0 +1,219 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2012, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File FORMAT.CPP +* +* Modification History: +* +* Date Name Description +* 02/19/97 aliu Converted from java. +* 03/17/97 clhuang Implemented with new APIs. +* 03/27/97 helena Updated to pass the simple test after code review. +* 07/20/98 stephen Added explicit init values for Field/ParsePosition +******************************************************************************** +*/ +// ***************************************************************************** +// This file was generated from the java source file Format.java +// ***************************************************************************** + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#ifndef U_I18N_IMPLEMENTATION +#error U_I18N_IMPLEMENTATION not set - must be set for all ICU source files in i18n/ - see https://unicode-org.github.io/icu/userguide/howtouseicu +#endif + +/* + * Dummy code: + * If all modules in the I18N library are switched off, then there are no + * library exports and MSVC 6 writes a .dll but not a .lib file. + * Unless we export _something_ in that case... + */ +#if UCONFIG_NO_COLLATION && UCONFIG_NO_FORMATTING && UCONFIG_NO_TRANSLITERATION +U_CAPI int32_t U_EXPORT2 +uprv_icuin_lib_dummy(int32_t i) { + return -i; +} +#endif + +/* Format class implementation ---------------------------------------------- */ + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/format.h" +#include "unicode/ures.h" +#include "cstring.h" +#include "locbased.h" + +// ***************************************************************************** +// class Format +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(FieldPosition) + +FieldPosition::~FieldPosition() {} + +FieldPosition * +FieldPosition::clone() const { + return new FieldPosition(*this); +} + +// ------------------------------------- +// default constructor + +Format::Format() + : UObject() +{ + *validLocale = *actualLocale = 0; +} + +// ------------------------------------- + +Format::~Format() +{ +} + +// ------------------------------------- +// copy constructor + +Format::Format(const Format &that) + : UObject(that) +{ + *this = that; +} + +// ------------------------------------- +// assignment operator + +Format& +Format::operator=(const Format& that) +{ + if (this != &that) { + uprv_strcpy(validLocale, that.validLocale); + uprv_strcpy(actualLocale, that.actualLocale); + } + return *this; +} + +// ------------------------------------- +// Formats the obj and append the result in the buffer, toAppendTo. +// This calls the actual implementation in the concrete subclasses. + +UnicodeString& +Format::format(const Formattable& obj, + UnicodeString& toAppendTo, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return toAppendTo; + + FieldPosition pos(FieldPosition::DONT_CARE); + + return format(obj, toAppendTo, pos, status); +} + +// ------------------------------------- +// Default implementation sets unsupported error; subclasses should +// override. + +UnicodeString& +Format::format(const Formattable& /* unused obj */, + UnicodeString& toAppendTo, + FieldPositionIterator* /* unused posIter */, + UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + status = U_UNSUPPORTED_ERROR; + } + return toAppendTo; +} + +// ------------------------------------- +// Parses the source string and create the corresponding +// result object. Checks the parse position for errors. + +void +Format::parseObject(const UnicodeString& source, + Formattable& result, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return; + + ParsePosition parsePosition(0); + parseObject(source, result, parsePosition); + if (parsePosition.getIndex() == 0) { + status = U_INVALID_FORMAT_ERROR; + } +} + +// ------------------------------------- + +bool +Format::operator==(const Format& that) const +{ + // Subclasses: Call this method and then add more specific checks. + return typeid(*this) == typeid(that); +} +//--------------------------------------- + +/** + * Simple function for initializing a UParseError from a UnicodeString. + * + * @param pattern The pattern to copy into the parseError + * @param pos The position in pattern where the error occurred + * @param parseError The UParseError object to fill in + * @draft ICU 2.4 + */ +void Format::syntaxError(const UnicodeString& pattern, + int32_t pos, + UParseError& parseError) { + parseError.offset = pos; + parseError.line=0; // we are not using line number + + // for pre-context + int32_t start = (pos < U_PARSE_CONTEXT_LEN)? 0 : (pos - (U_PARSE_CONTEXT_LEN-1 + /* subtract 1 so that we have room for null*/)); + int32_t stop = pos; + pattern.extract(start,stop-start,parseError.preContext,0); + //null terminate the buffer + parseError.preContext[stop-start] = 0; + + //for post-context + start = pos+1; + stop = ((pos+U_PARSE_CONTEXT_LEN)<=pattern.length()) ? (pos+(U_PARSE_CONTEXT_LEN-1)) : + pattern.length(); + pattern.extract(start,stop-start,parseError.postContext,0); + //null terminate the buffer + parseError.postContext[stop-start]= 0; +} + +Locale +Format::getLocale(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocale(type, status); +} + +const char * +Format::getLocaleID(ULocDataLocaleType type, UErrorCode& status) const { + U_LOCALE_BASED(locBased, *this); + return locBased.getLocaleID(type, status); +} + +void +Format::setLocaleIDs(const char* valid, const char* actual) { + U_LOCALE_BASED(locBased, *this); + locBased.setLocaleIDs(valid, actual); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/formatted_string_builder.cpp b/intl/icu/source/i18n/formatted_string_builder.cpp new file mode 100644 index 0000000000..8dbf954af9 --- /dev/null +++ b/intl/icu/source/i18n/formatted_string_builder.cpp @@ -0,0 +1,470 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "formatted_string_builder.h" +#include "putilimp.h" +#include "unicode/ustring.h" +#include "unicode/utf16.h" +#include "unicode/unum.h" // for UNumberFormatFields literals + +namespace { + +// A version of uprv_memcpy that checks for length 0. +// By default, uprv_memcpy requires a length of at least 1. +inline void uprv_memcpy2(void* dest, const void* src, size_t len) { + if (len > 0) { + uprv_memcpy(dest, src, len); + } +} + +// A version of uprv_memmove that checks for length 0. +// By default, uprv_memmove requires a length of at least 1. +inline void uprv_memmove2(void* dest, const void* src, size_t len) { + if (len > 0) { + uprv_memmove(dest, src, len); + } +} + +} // namespace + + +U_NAMESPACE_BEGIN + +FormattedStringBuilder::FormattedStringBuilder() { +#if U_DEBUG + // Initializing the memory to non-zero helps catch some bugs that involve + // reading from an improperly terminated string. + for (int32_t i=0; i DEFAULT_CAPACITY) { + // FIXME: uprv_malloc + // C++ note: malloc appears in two places: here and in prepareForInsertHelper. + auto newChars = static_cast (uprv_malloc(sizeof(char16_t) * capacity)); + auto newFields = static_cast(uprv_malloc(sizeof(Field) * capacity)); + if (newChars == nullptr || newFields == nullptr) { + // UErrorCode is not available; fail silently. + uprv_free(newChars); + uprv_free(newFields); + *this = FormattedStringBuilder(); // can't fail + return *this; + } + + fUsingHeap = true; + fChars.heap.capacity = capacity; + fChars.heap.ptr = newChars; + fFields.heap.capacity = capacity; + fFields.heap.ptr = newFields; + } + + uprv_memcpy2(getCharPtr(), other.getCharPtr(), sizeof(char16_t) * capacity); + uprv_memcpy2(getFieldPtr(), other.getFieldPtr(), sizeof(Field) * capacity); + + fZero = other.fZero; + fLength = other.fLength; + return *this; +} + +int32_t FormattedStringBuilder::length() const { + return fLength; +} + +int32_t FormattedStringBuilder::codePointCount() const { + return u_countChar32(getCharPtr() + fZero, fLength); +} + +UChar32 FormattedStringBuilder::getFirstCodePoint() const { + if (fLength == 0) { + return -1; + } + UChar32 cp; + U16_GET(getCharPtr() + fZero, 0, 0, fLength, cp); + return cp; +} + +UChar32 FormattedStringBuilder::getLastCodePoint() const { + if (fLength == 0) { + return -1; + } + int32_t offset = fLength; + U16_BACK_1(getCharPtr() + fZero, 0, offset); + UChar32 cp; + U16_GET(getCharPtr() + fZero, 0, offset, fLength, cp); + return cp; +} + +UChar32 FormattedStringBuilder::codePointAt(int32_t index) const { + UChar32 cp; + U16_GET(getCharPtr() + fZero, 0, index, fLength, cp); + return cp; +} + +UChar32 FormattedStringBuilder::codePointBefore(int32_t index) const { + int32_t offset = index; + U16_BACK_1(getCharPtr() + fZero, 0, offset); + UChar32 cp; + U16_GET(getCharPtr() + fZero, 0, offset, fLength, cp); + return cp; +} + +FormattedStringBuilder &FormattedStringBuilder::clear() { + // TODO: Reset the heap here? + fZero = getCapacity() / 2; + fLength = 0; + return *this; +} + +int32_t +FormattedStringBuilder::insertCodePoint(int32_t index, UChar32 codePoint, Field field, UErrorCode &status) { + int32_t count = U16_LENGTH(codePoint); + int32_t position = prepareForInsert(index, count, status); + if (U_FAILURE(status)) { + return count; + } + if (count == 1) { + getCharPtr()[position] = (char16_t) codePoint; + getFieldPtr()[position] = field; + } else { + getCharPtr()[position] = U16_LEAD(codePoint); + getCharPtr()[position + 1] = U16_TRAIL(codePoint); + getFieldPtr()[position] = getFieldPtr()[position + 1] = field; + } + return count; +} + +int32_t FormattedStringBuilder::insert(int32_t index, const UnicodeString &unistr, Field field, + UErrorCode &status) { + if (unistr.length() == 0) { + // Nothing to insert. + return 0; + } else if (unistr.length() == 1) { + // Fast path: insert using insertCodePoint. + return insertCodePoint(index, unistr.charAt(0), field, status); + } else { + return insert(index, unistr, 0, unistr.length(), field, status); + } +} + +int32_t +FormattedStringBuilder::insert(int32_t index, const UnicodeString &unistr, int32_t start, int32_t end, + Field field, UErrorCode &status) { + int32_t count = end - start; + int32_t position = prepareForInsert(index, count, status); + if (U_FAILURE(status)) { + return count; + } + for (int32_t i = 0; i < count; i++) { + getCharPtr()[position + i] = unistr.charAt(start + i); + getFieldPtr()[position + i] = field; + } + return count; +} + +int32_t +FormattedStringBuilder::splice(int32_t startThis, int32_t endThis, const UnicodeString &unistr, + int32_t startOther, int32_t endOther, Field field, UErrorCode& status) { + int32_t thisLength = endThis - startThis; + int32_t otherLength = endOther - startOther; + int32_t count = otherLength - thisLength; + if (U_FAILURE(status)) { + return count; + } + int32_t position; + if (count > 0) { + // Overall, chars need to be added. + position = prepareForInsert(startThis, count, status); + } else { + // Overall, chars need to be removed or kept the same. + position = remove(startThis, -count); + } + if (U_FAILURE(status)) { + return count; + } + for (int32_t i = 0; i < otherLength; i++) { + getCharPtr()[position + i] = unistr.charAt(startOther + i); + getFieldPtr()[position + i] = field; + } + return count; +} + +int32_t FormattedStringBuilder::append(const FormattedStringBuilder &other, UErrorCode &status) { + return insert(fLength, other, status); +} + +int32_t +FormattedStringBuilder::insert(int32_t index, const FormattedStringBuilder &other, UErrorCode &status) { + if (U_FAILURE(status)) { + return 0; + } + if (this == &other) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + int32_t count = other.fLength; + if (count == 0) { + // Nothing to insert. + return 0; + } + int32_t position = prepareForInsert(index, count, status); + if (U_FAILURE(status)) { + return count; + } + for (int32_t i = 0; i < count; i++) { + getCharPtr()[position + i] = other.charAt(i); + getFieldPtr()[position + i] = other.fieldAt(i); + } + return count; +} + +void FormattedStringBuilder::writeTerminator(UErrorCode& status) { + int32_t position = prepareForInsert(fLength, 1, status); + if (U_FAILURE(status)) { + return; + } + getCharPtr()[position] = 0; + getFieldPtr()[position] = kUndefinedField; + fLength--; +} + +int32_t FormattedStringBuilder::prepareForInsert(int32_t index, int32_t count, UErrorCode &status) { + U_ASSERT(index >= 0); + U_ASSERT(index <= fLength); + U_ASSERT(count >= 0); + U_ASSERT(fZero >= 0); + U_ASSERT(fLength >= 0); + U_ASSERT(getCapacity() - fZero >= fLength); + if (U_FAILURE(status)) { + return count; + } + if (index == 0 && fZero - count >= 0) { + // Append to start + fZero -= count; + fLength += count; + return fZero; + } else if (index == fLength && count <= getCapacity() - fZero - fLength) { + // Append to end + fLength += count; + return fZero + fLength - count; + } else { + // Move chars around and/or allocate more space + return prepareForInsertHelper(index, count, status); + } +} + +int32_t FormattedStringBuilder::prepareForInsertHelper(int32_t index, int32_t count, UErrorCode &status) { + int32_t oldCapacity = getCapacity(); + int32_t oldZero = fZero; + char16_t *oldChars = getCharPtr(); + Field *oldFields = getFieldPtr(); + int32_t newLength; + if (uprv_add32_overflow(fLength, count, &newLength)) { + status = U_INPUT_TOO_LONG_ERROR; + return -1; + } + int32_t newZero; + if (newLength > oldCapacity) { + if (newLength > INT32_MAX / 2) { + // We do not support more than 1G char16_t in this code because + // dealing with >2G *bytes* can cause subtle bugs. + status = U_INPUT_TOO_LONG_ERROR; + return -1; + } + // Keep newCapacity also to at most 1G char16_t. + int32_t newCapacity = newLength * 2; + newZero = (newCapacity - newLength) / 2; + + // C++ note: malloc appears in two places: here and in the assignment operator. + auto newChars = static_cast (uprv_malloc(sizeof(char16_t) * static_cast(newCapacity))); + auto newFields = static_cast(uprv_malloc(sizeof(Field) * static_cast(newCapacity))); + if (newChars == nullptr || newFields == nullptr) { + uprv_free(newChars); + uprv_free(newFields); + status = U_MEMORY_ALLOCATION_ERROR; + return -1; + } + + // First copy the prefix and then the suffix, leaving room for the new chars that the + // caller wants to insert. + // C++ note: memcpy is OK because the src and dest do not overlap. + uprv_memcpy2(newChars + newZero, oldChars + oldZero, sizeof(char16_t) * index); + uprv_memcpy2(newChars + newZero + index + count, + oldChars + oldZero + index, + sizeof(char16_t) * (fLength - index)); + uprv_memcpy2(newFields + newZero, oldFields + oldZero, sizeof(Field) * index); + uprv_memcpy2(newFields + newZero + index + count, + oldFields + oldZero + index, + sizeof(Field) * (fLength - index)); + + if (fUsingHeap) { + uprv_free(oldChars); + uprv_free(oldFields); + } + fUsingHeap = true; + fChars.heap.ptr = newChars; + fChars.heap.capacity = newCapacity; + fFields.heap.ptr = newFields; + fFields.heap.capacity = newCapacity; + } else { + newZero = (oldCapacity - newLength) / 2; + + // C++ note: memmove is required because src and dest may overlap. + // First copy the entire string to the location of the prefix, and then move the suffix + // to make room for the new chars that the caller wants to insert. + uprv_memmove2(oldChars + newZero, oldChars + oldZero, sizeof(char16_t) * fLength); + uprv_memmove2(oldChars + newZero + index + count, + oldChars + newZero + index, + sizeof(char16_t) * (fLength - index)); + uprv_memmove2(oldFields + newZero, oldFields + oldZero, sizeof(Field) * fLength); + uprv_memmove2(oldFields + newZero + index + count, + oldFields + newZero + index, + sizeof(Field) * (fLength - index)); + } + fZero = newZero; + fLength = newLength; + return fZero + index; +} + +int32_t FormattedStringBuilder::remove(int32_t index, int32_t count) { + U_ASSERT(0 <= index); + U_ASSERT(index <= fLength); + U_ASSERT(count <= (fLength - index)); + U_ASSERT(index <= getCapacity() - fZero); + + int32_t position = index + fZero; + // TODO: Reset the heap here? (If the string after removal can fit on stack?) + uprv_memmove2(getCharPtr() + position, + getCharPtr() + position + count, + sizeof(char16_t) * (fLength - index - count)); + uprv_memmove2(getFieldPtr() + position, + getFieldPtr() + position + count, + sizeof(Field) * (fLength - index - count)); + fLength -= count; + return position; +} + +UnicodeString FormattedStringBuilder::toUnicodeString() const { + return UnicodeString(getCharPtr() + fZero, fLength); +} + +const UnicodeString FormattedStringBuilder::toTempUnicodeString() const { + // Readonly-alias constructor: + return UnicodeString(false, getCharPtr() + fZero, fLength); +} + +UnicodeString FormattedStringBuilder::toDebugString() const { + UnicodeString sb; + sb.append(u"", -1); + return sb; +} + +const char16_t *FormattedStringBuilder::chars() const { + return getCharPtr() + fZero; +} + +bool FormattedStringBuilder::contentEquals(const FormattedStringBuilder &other) const { + if (fLength != other.fLength) { + return false; + } + for (int32_t i = 0; i < fLength; i++) { + if (charAt(i) != other.charAt(i) || fieldAt(i) != other.fieldAt(i)) { + return false; + } + } + return true; +} + +bool FormattedStringBuilder::containsField(Field field) const { + for (int32_t i = 0; i < fLength; i++) { + if (field == fieldAt(i)) { + return true; + } + } + return false; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/formatted_string_builder.h b/intl/icu/source/i18n/formatted_string_builder.h new file mode 100644 index 0000000000..32e0900ae2 --- /dev/null +++ b/intl/icu/source/i18n/formatted_string_builder.h @@ -0,0 +1,272 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_STRINGBUILDER_H__ +#define __NUMBER_STRINGBUILDER_H__ + + +#include +#include + +#include "cstring.h" +#include "uassert.h" +#include "fphdlimp.h" + +U_NAMESPACE_BEGIN + +class FormattedValueStringBuilderImpl; + +/** + * A StringBuilder optimized for formatting. It implements the following key + * features beyond a UnicodeString: + * + *

    + *
  1. Efficient prepend as well as append. + *
  2. Keeps track of Fields in an efficient manner. + *
+ * + * See also FormattedValueStringBuilderImpl. + * + * @author sffc (Shane Carr) + */ +class U_I18N_API FormattedStringBuilder : public UMemory { + private: + static const int32_t DEFAULT_CAPACITY = 40; + + template + union ValueOrHeapArray { + T value[DEFAULT_CAPACITY]; + struct { + T *ptr; + int32_t capacity; + } heap; + }; + + public: + FormattedStringBuilder(); + + ~FormattedStringBuilder(); + + FormattedStringBuilder(const FormattedStringBuilder &other); + + // Convention: bottom 4 bits for field, top 4 bits for field category. + // Field category 0 implies the number category so that the number field + // literals can be directly passed as a Field type. + // Exported as U_I18N_API so it can be used by other exports on Windows. + struct U_I18N_API Field { + uint8_t bits; + + Field() = default; + constexpr Field(uint8_t category, uint8_t field); + + inline UFieldCategory getCategory() const; + inline int32_t getField() const; + inline bool isNumeric() const; + inline bool isUndefined() const; + inline bool operator==(const Field& other) const; + inline bool operator!=(const Field& other) const; + }; + + FormattedStringBuilder &operator=(const FormattedStringBuilder &other); + + int32_t length() const; + + int32_t codePointCount() const; + + inline char16_t charAt(int32_t index) const { + U_ASSERT(index >= 0); + U_ASSERT(index < fLength); + return getCharPtr()[fZero + index]; + } + + inline Field fieldAt(int32_t index) const { + U_ASSERT(index >= 0); + U_ASSERT(index < fLength); + return getFieldPtr()[fZero + index]; + } + + UChar32 getFirstCodePoint() const; + + UChar32 getLastCodePoint() const; + + UChar32 codePointAt(int32_t index) const; + + UChar32 codePointBefore(int32_t index) const; + + FormattedStringBuilder &clear(); + + /** Appends a UTF-16 code unit. */ + inline int32_t appendChar16(char16_t codeUnit, Field field, UErrorCode& status) { + // appendCodePoint handles both code units and code points. + return insertCodePoint(fLength, codeUnit, field, status); + } + + /** Inserts a UTF-16 code unit. Note: insert at index 0 is very efficient. */ + inline int32_t insertChar16(int32_t index, char16_t codeUnit, Field field, UErrorCode& status) { + // insertCodePoint handles both code units and code points. + return insertCodePoint(index, codeUnit, field, status); + } + + /** Appends a Unicode code point. */ + inline int32_t appendCodePoint(UChar32 codePoint, Field field, UErrorCode &status) { + return insertCodePoint(fLength, codePoint, field, status); + } + + /** Inserts a Unicode code point. Note: insert at index 0 is very efficient. */ + int32_t insertCodePoint(int32_t index, UChar32 codePoint, Field field, UErrorCode &status); + + /** Appends a string. */ + inline int32_t append(const UnicodeString &unistr, Field field, UErrorCode &status) { + return insert(fLength, unistr, field, status); + } + + /** Inserts a string. Note: insert at index 0 is very efficient. */ + int32_t insert(int32_t index, const UnicodeString &unistr, Field field, UErrorCode &status); + + /** Inserts a substring. Note: insert at index 0 is very efficient. + * + * @param start Start index of the substring of unistr to be inserted. + * @param end End index of the substring of unistr to be inserted (exclusive). + */ + int32_t insert(int32_t index, const UnicodeString &unistr, int32_t start, int32_t end, Field field, + UErrorCode &status); + + /** Deletes a substring and then inserts a string at that same position. + * Similar to JavaScript Array.prototype.splice(). + * + * @param startThis Start of the span to delete. + * @param endThis End of the span to delete (exclusive). + * @param unistr The string to insert at the deletion position. + * @param startOther Start index of the substring of unistr to be inserted. + * @param endOther End index of the substring of unistr to be inserted (exclusive). + */ + int32_t splice(int32_t startThis, int32_t endThis, const UnicodeString &unistr, + int32_t startOther, int32_t endOther, Field field, UErrorCode& status); + + /** Appends a formatted string. */ + int32_t append(const FormattedStringBuilder &other, UErrorCode &status); + + /** Inserts a formatted string. Note: insert at index 0 is very efficient. */ + int32_t insert(int32_t index, const FormattedStringBuilder &other, UErrorCode &status); + + /** + * Ensures that the string buffer contains a NUL terminator. The NUL terminator does + * not count toward the string length. Any further changes to the string (insert or + * append) may invalidate the NUL terminator. + * + * You should call this method after the formatted string is completely built if you + * plan to return a pointer to the string from a C API. + */ + void writeTerminator(UErrorCode& status); + + /** + * Gets a "safe" UnicodeString that can be used even after the FormattedStringBuilder is destructed. + */ + UnicodeString toUnicodeString() const; + + /** + * Gets an "unsafe" UnicodeString that is valid only as long as the FormattedStringBuilder is alive and + * unchanged. Slightly faster than toUnicodeString(). + */ + const UnicodeString toTempUnicodeString() const; + + UnicodeString toDebugString() const; + + const char16_t *chars() const; + + bool contentEquals(const FormattedStringBuilder &other) const; + + bool containsField(Field field) const; + + private: + bool fUsingHeap = false; + ValueOrHeapArray fChars; + ValueOrHeapArray fFields; + int32_t fZero = DEFAULT_CAPACITY / 2; + int32_t fLength = 0; + + inline char16_t *getCharPtr() { + return fUsingHeap ? fChars.heap.ptr : fChars.value; + } + + inline const char16_t *getCharPtr() const { + return fUsingHeap ? fChars.heap.ptr : fChars.value; + } + + inline Field *getFieldPtr() { + return fUsingHeap ? fFields.heap.ptr : fFields.value; + } + + inline const Field *getFieldPtr() const { + return fUsingHeap ? fFields.heap.ptr : fFields.value; + } + + inline int32_t getCapacity() const { + return fUsingHeap ? fChars.heap.capacity : DEFAULT_CAPACITY; + } + + int32_t prepareForInsert(int32_t index, int32_t count, UErrorCode &status); + + int32_t prepareForInsertHelper(int32_t index, int32_t count, UErrorCode &status); + + int32_t remove(int32_t index, int32_t count); + + friend class FormattedValueStringBuilderImpl; +}; + +static_assert( + // std::is_pod<> is deprecated. + std::is_standard_layout::value && + std::is_trivial::value, + "Field should be a POD type for efficient initialization"); + +constexpr FormattedStringBuilder::Field::Field(uint8_t category, uint8_t field) + : bits(( + U_ASSERT(category <= 0xf), + U_ASSERT(field <= 0xf), + static_cast((category << 4) | field) + )) {} + +/** + * Internal constant for the undefined field for use in FormattedStringBuilder. + */ +constexpr FormattedStringBuilder::Field kUndefinedField = {UFIELD_CATEGORY_UNDEFINED, 0}; + +/** + * Internal field to signal "numeric" when fields are not supported in NumberFormat. + */ +constexpr FormattedStringBuilder::Field kGeneralNumericField = {UFIELD_CATEGORY_UNDEFINED, 1}; + +inline UFieldCategory FormattedStringBuilder::Field::getCategory() const { + return static_cast(bits >> 4); +} + +inline int32_t FormattedStringBuilder::Field::getField() const { + return bits & 0xf; +} + +inline bool FormattedStringBuilder::Field::isNumeric() const { + return getCategory() == UFIELD_CATEGORY_NUMBER || *this == kGeneralNumericField; +} + +inline bool FormattedStringBuilder::Field::isUndefined() const { + return getCategory() == UFIELD_CATEGORY_UNDEFINED; +} + +inline bool FormattedStringBuilder::Field::operator==(const Field& other) const { + return bits == other.bits; +} + +inline bool FormattedStringBuilder::Field::operator!=(const Field& other) const { + return bits != other.bits; +} + +U_NAMESPACE_END + + +#endif //__NUMBER_STRINGBUILDER_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/formattedval_impl.h b/intl/icu/source/i18n/formattedval_impl.h new file mode 100644 index 0000000000..e19392c5b9 --- /dev/null +++ b/intl/icu/source/i18n/formattedval_impl.h @@ -0,0 +1,318 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#ifndef __FORMVAL_IMPL_H__ +#define __FORMVAL_IMPL_H__ + +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +// This file contains compliant implementations of FormattedValue which can be +// leveraged by ICU formatters. +// +// Each implementation is defined in its own cpp file in order to split +// dependencies more modularly. + +#include "unicode/formattedvalue.h" +#include "capi_helper.h" +#include "fphdlimp.h" +#include "util.h" +#include "uvectr32.h" +#include "formatted_string_builder.h" + + +/** + * Represents the type of constraint for ConstrainedFieldPosition. + * + * Constraints are used to control the behavior of iteration in FormattedValue. + * + * @internal + */ +typedef enum UCFPosConstraintType { + /** + * Represents the lack of a constraint. + * + * This is the value of fConstraint if no "constrain" methods were called. + * + * @internal + */ + UCFPOS_CONSTRAINT_NONE = 0, + + /** + * Represents that the field category is constrained. + * + * This is the value of fConstraint if constraintCategory was called. + * + * FormattedValue implementations should not change the field category + * while this constraint is active. + * + * @internal + */ + UCFPOS_CONSTRAINT_CATEGORY, + + /** + * Represents that the field and field category are constrained. + * + * This is the value of fConstraint if constraintField was called. + * + * FormattedValue implementations should not change the field or field category + * while this constraint is active. + * + * @internal + */ + UCFPOS_CONSTRAINT_FIELD +} UCFPosConstraintType; + + +U_NAMESPACE_BEGIN + + +/** + * Implementation of FormattedValue using FieldPositionHandler to accept fields. + * + * TODO(ICU-20897): This class is unused. If it is not needed when fixing ICU-20897, + * it should be deleted. + */ +class FormattedValueFieldPositionIteratorImpl : public UMemory, public FormattedValue { +public: + + /** @param initialFieldCapacity Initially allocate space for this many fields. */ + FormattedValueFieldPositionIteratorImpl(int32_t initialFieldCapacity, UErrorCode& status); + + virtual ~FormattedValueFieldPositionIteratorImpl(); + + // Implementation of FormattedValue (const): + + UnicodeString toString(UErrorCode& status) const override; + UnicodeString toTempString(UErrorCode& status) const override; + Appendable& appendTo(Appendable& appendable, UErrorCode& status) const override; + UBool nextPosition(ConstrainedFieldPosition& cfpos, UErrorCode& status) const override; + + // Additional methods used during construction phase only (non-const): + + FieldPositionIteratorHandler getHandler(UErrorCode& status); + void appendString(UnicodeString string, UErrorCode& status); + + /** + * Computes the spans for duplicated values. + * For example, if the string has fields: + * + * ...aa..[b.cc]..d.[bb.e.c]..a.. + * + * then the spans will be the bracketed regions. + * + * Assumes that the currently known fields are sorted + * and all in the same category. + */ + void addOverlapSpans(UFieldCategory spanCategory, int8_t firstIndex, UErrorCode& status); + + /** + * Sorts the fields: start index first, length second. + */ + void sort(); + +private: + UnicodeString fString; + UVector32 fFields; +}; + + +// Internal struct that must be exported for MSVC +struct U_I18N_API SpanInfo { + UFieldCategory category; + int32_t spanValue; + int32_t start; + int32_t length; +}; + +// Export an explicit template instantiation of the MaybeStackArray that +// is used as a data member of CEBuffer. +// +// When building DLLs for Windows this is required even though +// no direct access to the MaybeStackArray leaks out of the i18n library. +// +// See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples. +// +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +#endif + +/** + * Implementation of FormattedValue based on FormattedStringBuilder. + * + * The implementation currently revolves around numbers and number fields. + * However, it can be generalized in the future when there is a need. + * + * @author sffc (Shane Carr) + */ +// Exported as U_I18N_API for tests +class U_I18N_API FormattedValueStringBuilderImpl : public UMemory, public FormattedValue { +public: + + FormattedValueStringBuilderImpl(FormattedStringBuilder::Field numericField); + + virtual ~FormattedValueStringBuilderImpl(); + + FormattedValueStringBuilderImpl(FormattedValueStringBuilderImpl&&) = default; + FormattedValueStringBuilderImpl& operator=(FormattedValueStringBuilderImpl&&) = default; + + // Implementation of FormattedValue (const): + + UnicodeString toString(UErrorCode& status) const override; + UnicodeString toTempString(UErrorCode& status) const override; + Appendable& appendTo(Appendable& appendable, UErrorCode& status) const override; + UBool nextPosition(ConstrainedFieldPosition& cfpos, UErrorCode& status) const override; + + // Additional helper functions: + UBool nextFieldPosition(FieldPosition& fp, UErrorCode& status) const; + void getAllFieldPositions(FieldPositionIteratorHandler& fpih, UErrorCode& status) const; + inline FormattedStringBuilder& getStringRef() { + return fString; + } + inline const FormattedStringBuilder& getStringRef() const { + return fString; + } + void resetString(); + + /** + * Adds additional metadata used for span fields. + * + * category: the category to use for the span field. + * spanValue: the value of the span field: index of the list item, for example. + * start: the start position within the string of the span. -1 if unknown. + * length: the length of the span, used to split adjacent fields. + */ + void appendSpanInfo(UFieldCategory category, int32_t spanValue, int32_t start, int32_t length, UErrorCode& status); + void prependSpanInfo(UFieldCategory category, int32_t spanValue, int32_t start, int32_t length, UErrorCode& status); + +private: + FormattedStringBuilder fString; + FormattedStringBuilder::Field fNumericField; + MaybeStackArray spanIndices; + int32_t spanIndicesCount = 0; + + bool nextPositionImpl(ConstrainedFieldPosition& cfpos, FormattedStringBuilder::Field numericField, UErrorCode& status) const; + static bool isIntOrGroup(FormattedStringBuilder::Field field); + static bool isTrimmable(FormattedStringBuilder::Field field); + int32_t trimBack(int32_t limit) const; + int32_t trimFront(int32_t start) const; +}; + + +// C API Helpers for FormattedValue +// Magic number as ASCII == "UFV" +struct UFormattedValueImpl; +typedef IcuCApiHelper UFormattedValueApiHelper; +struct UFormattedValueImpl : public UMemory, public UFormattedValueApiHelper { + // This pointer should be set by the child class. + FormattedValue* fFormattedValue = nullptr; +}; + + +/** Boilerplate to check for valid status before dereferencing the fData pointer. */ +#define UPRV_FORMATTED_VALUE_METHOD_GUARD(returnExpression) \ + if (U_FAILURE(status)) { \ + return returnExpression; \ + } \ + if (fData == nullptr) { \ + status = fErrorCode; \ + return returnExpression; \ + } \ + + +/** Implementation of the methods from U_FORMATTED_VALUE_SUBCLASS_AUTO. */ +#define UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(Name) \ + Name::Name(Name&& src) noexcept \ + : fData(src.fData), fErrorCode(src.fErrorCode) { \ + src.fData = nullptr; \ + src.fErrorCode = U_INVALID_STATE_ERROR; \ + } \ + Name::~Name() { \ + delete fData; \ + fData = nullptr; \ + } \ + Name& Name::operator=(Name&& src) noexcept { \ + delete fData; \ + fData = src.fData; \ + src.fData = nullptr; \ + fErrorCode = src.fErrorCode; \ + src.fErrorCode = U_INVALID_STATE_ERROR; \ + return *this; \ + } \ + UnicodeString Name::toString(UErrorCode& status) const { \ + UPRV_FORMATTED_VALUE_METHOD_GUARD(ICU_Utility::makeBogusString()) \ + return fData->toString(status); \ + } \ + UnicodeString Name::toTempString(UErrorCode& status) const { \ + UPRV_FORMATTED_VALUE_METHOD_GUARD(ICU_Utility::makeBogusString()) \ + return fData->toTempString(status); \ + } \ + Appendable& Name::appendTo(Appendable& appendable, UErrorCode& status) const { \ + UPRV_FORMATTED_VALUE_METHOD_GUARD(appendable) \ + return fData->appendTo(appendable, status); \ + } \ + UBool Name::nextPosition(ConstrainedFieldPosition& cfpos, UErrorCode& status) const { \ + UPRV_FORMATTED_VALUE_METHOD_GUARD(false) \ + return fData->nextPosition(cfpos, status); \ + } + + +/** Like UPRV_FORMATTED_VALUE_CAPI_AUTO_IMPL but without impl type declarations. */ +#define UPRV_FORMATTED_VALUE_CAPI_NO_IMPLTYPE_AUTO_IMPL(CType, ImplType, HelperType, Prefix) \ + U_CAPI CType* U_EXPORT2 \ + Prefix ## _openResult (UErrorCode* ec) { \ + if (U_FAILURE(*ec)) { \ + return nullptr; \ + } \ + ImplType* impl = new ImplType(); \ + if (impl == nullptr) { \ + *ec = U_MEMORY_ALLOCATION_ERROR; \ + return nullptr; \ + } \ + return static_cast(impl)->exportForC(); \ + } \ + U_CAPI const UFormattedValue* U_EXPORT2 \ + Prefix ## _resultAsValue (const CType* uresult, UErrorCode* ec) { \ + const ImplType* result = HelperType::validate(uresult, *ec); \ + if (U_FAILURE(*ec)) { return nullptr; } \ + return static_cast(result)->exportConstForC(); \ + } \ + U_CAPI void U_EXPORT2 \ + Prefix ## _closeResult (CType* uresult) { \ + UErrorCode localStatus = U_ZERO_ERROR; \ + const ImplType* impl = HelperType::validate(uresult, localStatus); \ + delete impl; \ + } + + +/** + * Implementation of the standard methods for a UFormattedValue "subclass" C API. + * @param CPPType The public C++ type, like FormattedList + * @param CType The public C type, like UFormattedList + * @param ImplType A name to use for the implementation class + * @param HelperType A name to use for the "mixin" typedef for C API conversion + * @param Prefix The C API prefix, like ulistfmt + * @param MagicNumber A unique 32-bit number to use to identify this type + */ +#define UPRV_FORMATTED_VALUE_CAPI_AUTO_IMPL(CPPType, CType, ImplType, HelperType, Prefix, MagicNumber) \ + U_NAMESPACE_BEGIN \ + class ImplType; \ + typedef IcuCApiHelper HelperType; \ + class ImplType : public UFormattedValueImpl, public HelperType { \ + public: \ + ImplType(); \ + ~ImplType(); \ + CPPType fImpl; \ + }; \ + ImplType::ImplType() { \ + fFormattedValue = &fImpl; \ + } \ + ImplType::~ImplType() {} \ + U_NAMESPACE_END \ + UPRV_FORMATTED_VALUE_CAPI_NO_IMPLTYPE_AUTO_IMPL(CType, ImplType, HelperType, Prefix) + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif // __FORMVAL_IMPL_H__ diff --git a/intl/icu/source/i18n/formattedval_iterimpl.cpp b/intl/icu/source/i18n/formattedval_iterimpl.cpp new file mode 100644 index 0000000000..ec770e2191 --- /dev/null +++ b/intl/icu/source/i18n/formattedval_iterimpl.cpp @@ -0,0 +1,176 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// This file contains one implementation of FormattedValue. +// Other independent implementations should go into their own cpp file for +// better dependency modularization. + +#include "formattedval_impl.h" +#include "putilimp.h" + +U_NAMESPACE_BEGIN + + +FormattedValueFieldPositionIteratorImpl::FormattedValueFieldPositionIteratorImpl( + int32_t initialFieldCapacity, + UErrorCode& status) + : fFields(initialFieldCapacity * 4, status) { +} + +FormattedValueFieldPositionIteratorImpl::~FormattedValueFieldPositionIteratorImpl() = default; + +UnicodeString FormattedValueFieldPositionIteratorImpl::toString( + UErrorCode&) const { + return fString; +} + +UnicodeString FormattedValueFieldPositionIteratorImpl::toTempString( + UErrorCode&) const { + // The alias must point to memory owned by this object; + // fastCopyFrom doesn't do this when using a stack buffer. + return UnicodeString(true, fString.getBuffer(), fString.length()); +} + +Appendable& FormattedValueFieldPositionIteratorImpl::appendTo( + Appendable& appendable, + UErrorCode&) const { + appendable.appendString(fString.getBuffer(), fString.length()); + return appendable; +} + +UBool FormattedValueFieldPositionIteratorImpl::nextPosition( + ConstrainedFieldPosition& cfpos, + UErrorCode&) const { + U_ASSERT(fFields.size() % 4 == 0); + int32_t numFields = fFields.size() / 4; + int32_t i = static_cast(cfpos.getInt64IterationContext()); + for (; i < numFields; i++) { + UFieldCategory category = static_cast(fFields.elementAti(i * 4)); + int32_t field = fFields.elementAti(i * 4 + 1); + if (cfpos.matchesField(category, field)) { + int32_t start = fFields.elementAti(i * 4 + 2); + int32_t limit = fFields.elementAti(i * 4 + 3); + cfpos.setState(category, field, start, limit); + break; + } + } + cfpos.setInt64IterationContext(i == numFields ? i : i + 1); + return i < numFields; +} + + +FieldPositionIteratorHandler FormattedValueFieldPositionIteratorImpl::getHandler( + UErrorCode& status) { + return FieldPositionIteratorHandler(&fFields, status); +} + +void FormattedValueFieldPositionIteratorImpl::appendString( + UnicodeString string, + UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + fString.append(string); + // Make the string NUL-terminated + if (fString.getTerminatedBuffer() == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } +} + + +void FormattedValueFieldPositionIteratorImpl::addOverlapSpans( + UFieldCategory spanCategory, + int8_t firstIndex, + UErrorCode& status) { + // In order to avoid fancy data structures, this is an O(N^2) algorithm, + // which should be fine for all real-life applications of this function. + int32_t s1a = INT32_MAX; + int32_t s1b = 0; + int32_t s2a = INT32_MAX; + int32_t s2b = 0; + int32_t numFields = fFields.size() / 4; + for (int32_t i = 0; i higher rank + comparison = start2 - start1; + } else if (limit1 != limit2) { + // Higher length (end index) -> lower rank + comparison = limit1 - limit2; + } else if (categ1 != categ2) { + // Higher field category -> lower rank + comparison = categ1 - categ2; + } else if (field1 != field2) { + // Higher field -> higher rank + comparison = field2 - field1; + } + if (comparison < 0) { + // Perform a swap + isSorted = false; + fFields.setElementAt(categ2, i*4 + 0); + fFields.setElementAt(field2, i*4 + 1); + fFields.setElementAt(start2, i*4 + 2); + fFields.setElementAt(limit2, i*4 + 3); + fFields.setElementAt(categ1, i*4 + 4); + fFields.setElementAt(field1, i*4 + 5); + fFields.setElementAt(start1, i*4 + 6); + fFields.setElementAt(limit1, i*4 + 7); + } + } + if (isSorted) { + break; + } + } +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/formattedval_sbimpl.cpp b/intl/icu/source/i18n/formattedval_sbimpl.cpp new file mode 100644 index 0000000000..72197cdd8c --- /dev/null +++ b/intl/icu/source/i18n/formattedval_sbimpl.cpp @@ -0,0 +1,350 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// This file contains one implementation of FormattedValue. +// Other independent implementations should go into their own cpp file for +// better dependency modularization. + +#include "unicode/ustring.h" +#include "formattedval_impl.h" +#include "number_types.h" +#include "formatted_string_builder.h" +#include "number_utils.h" +#include "static_unicode_sets.h" +#include "unicode/listformatter.h" + +U_NAMESPACE_BEGIN + + +typedef FormattedStringBuilder::Field Field; + + +FormattedValueStringBuilderImpl::FormattedValueStringBuilderImpl(Field numericField) + : fNumericField(numericField) { +} + +FormattedValueStringBuilderImpl::~FormattedValueStringBuilderImpl() { +} + + +UnicodeString FormattedValueStringBuilderImpl::toString(UErrorCode&) const { + return fString.toUnicodeString(); +} + +UnicodeString FormattedValueStringBuilderImpl::toTempString(UErrorCode&) const { + return fString.toTempUnicodeString(); +} + +Appendable& FormattedValueStringBuilderImpl::appendTo(Appendable& appendable, UErrorCode&) const { + appendable.appendString(fString.chars(), fString.length()); + return appendable; +} + +UBool FormattedValueStringBuilderImpl::nextPosition(ConstrainedFieldPosition& cfpos, UErrorCode& status) const { + // NOTE: MSVC sometimes complains when implicitly converting between bool and UBool + return nextPositionImpl(cfpos, fNumericField, status) ? true : false; +} + +UBool FormattedValueStringBuilderImpl::nextFieldPosition(FieldPosition& fp, UErrorCode& status) const { + int32_t rawField = fp.getField(); + + if (rawField == FieldPosition::DONT_CARE) { + return false; + } + + if (rawField < 0 || rawField >= UNUM_FIELD_COUNT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return false; + } + + ConstrainedFieldPosition cfpos; + cfpos.constrainField(UFIELD_CATEGORY_NUMBER, rawField); + cfpos.setState(UFIELD_CATEGORY_NUMBER, rawField, fp.getBeginIndex(), fp.getEndIndex()); + if (nextPositionImpl(cfpos, kUndefinedField, status)) { + fp.setBeginIndex(cfpos.getStart()); + fp.setEndIndex(cfpos.getLimit()); + return true; + } + + // Special case: fraction should start after integer if fraction is not present + if (rawField == UNUM_FRACTION_FIELD && fp.getEndIndex() == 0) { + bool inside = false; + int32_t i = fString.fZero; + for (; i < fString.fZero + fString.fLength; i++) { + if (isIntOrGroup(fString.getFieldPtr()[i]) || fString.getFieldPtr()[i] == Field(UFIELD_CATEGORY_NUMBER, UNUM_DECIMAL_SEPARATOR_FIELD)) { + inside = true; + } else if (inside) { + break; + } + } + fp.setBeginIndex(i - fString.fZero); + fp.setEndIndex(i - fString.fZero); + } + + return false; +} + +void FormattedValueStringBuilderImpl::getAllFieldPositions(FieldPositionIteratorHandler& fpih, + UErrorCode& status) const { + ConstrainedFieldPosition cfpos; + while (nextPositionImpl(cfpos, kUndefinedField, status)) { + fpih.addAttribute(cfpos.getField(), cfpos.getStart(), cfpos.getLimit()); + } +} + +void FormattedValueStringBuilderImpl::resetString() { + fString.clear(); + spanIndicesCount = 0; +} + +// Signal the end of the string using a field that doesn't exist and that is +// different from kUndefinedField, which is used for "null field". +static constexpr Field kEndField = Field(0xf, 0xf); + +bool FormattedValueStringBuilderImpl::nextPositionImpl(ConstrainedFieldPosition& cfpos, Field numericField, UErrorCode& /*status*/) const { + int32_t fieldStart = -1; + Field currField = kUndefinedField; + bool prevIsSpan = false; + int32_t nextSpanStart = -1; + if (spanIndicesCount > 0) { + int64_t si = cfpos.getInt64IterationContext(); + U_ASSERT(si <= spanIndicesCount); + if (si < spanIndicesCount) { + nextSpanStart = spanIndices[si].start; + } + if (si > 0) { + prevIsSpan = cfpos.getCategory() == spanIndices[si-1].category + && cfpos.getField() == spanIndices[si-1].spanValue; + } + } + bool prevIsNumeric = false; + if (numericField != kUndefinedField) { + prevIsNumeric = cfpos.getCategory() == numericField.getCategory() + && cfpos.getField() == numericField.getField(); + } + bool prevIsInteger = cfpos.getCategory() == UFIELD_CATEGORY_NUMBER + && cfpos.getField() == UNUM_INTEGER_FIELD; + + for (int32_t i = fString.fZero + cfpos.getLimit(); i <= fString.fZero + fString.fLength; i++) { + Field _field = (i < fString.fZero + fString.fLength) ? fString.getFieldPtr()[i] : kEndField; + // Case 1: currently scanning a field. + if (currField != kUndefinedField) { + if (currField != _field) { + int32_t end = i - fString.fZero; + // Grouping separators can be whitespace; don't throw them out! + if (isTrimmable(currField)) { + end = trimBack(i - fString.fZero); + } + if (end <= fieldStart) { + // Entire field position is ignorable; skip. + fieldStart = -1; + currField = kUndefinedField; + i--; // look at this index again + continue; + } + int32_t start = fieldStart; + if (isTrimmable(currField)) { + start = trimFront(start); + } + cfpos.setState(currField.getCategory(), currField.getField(), start, end); + return true; + } + continue; + } + // Special case: emit normalField if we are pointing at the end of spanField. + if (i > fString.fZero && prevIsSpan) { + int64_t si = cfpos.getInt64IterationContext() - 1; + U_ASSERT(si >= 0); + int32_t previ = i - spanIndices[si].length; + U_ASSERT(previ >= fString.fZero); + Field prevField = fString.getFieldPtr()[previ]; + if (prevField == Field(UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD)) { + // Special handling for ULISTFMT_ELEMENT_FIELD + if (cfpos.matchesField(UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD)) { + fieldStart = i - fString.fZero - spanIndices[si].length; + int32_t end = fieldStart + spanIndices[si].length; + cfpos.setState( + UFIELD_CATEGORY_LIST, + ULISTFMT_ELEMENT_FIELD, + fieldStart, + end); + return true; + } else { + prevIsSpan = false; + } + } else { + // Re-wind, since there may be multiple fields in the span. + i = previ; + _field = prevField; + } + } + // Special case: coalesce the INTEGER if we are pointing at the end of the INTEGER. + if (cfpos.matchesField(UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD) + && i > fString.fZero + && !prevIsInteger + && !prevIsNumeric + && isIntOrGroup(fString.getFieldPtr()[i - 1]) + && !isIntOrGroup(_field)) { + int j = i - 1; + for (; j >= fString.fZero && isIntOrGroup(fString.getFieldPtr()[j]); j--) {} + cfpos.setState( + UFIELD_CATEGORY_NUMBER, + UNUM_INTEGER_FIELD, + j - fString.fZero + 1, + i - fString.fZero); + return true; + } + // Special case: coalesce NUMERIC if we are pointing at the end of the NUMERIC. + if (numericField != kUndefinedField + && cfpos.matchesField(numericField.getCategory(), numericField.getField()) + && i > fString.fZero + && !prevIsNumeric + && fString.getFieldPtr()[i - 1].isNumeric() + && !_field.isNumeric()) { + // Re-wind to the beginning of the field and then emit it + int32_t j = i - 1; + for (; j >= fString.fZero && fString.getFieldPtr()[j].isNumeric(); j--) {} + cfpos.setState( + numericField.getCategory(), + numericField.getField(), + j - fString.fZero + 1, + i - fString.fZero); + return true; + } + // Check for span field + if (!prevIsSpan && ( + _field == Field(UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD) || + i - fString.fZero == nextSpanStart)) { + int64_t si = cfpos.getInt64IterationContext(); + if (si >= spanIndicesCount) { + break; + } + UFieldCategory spanCategory = spanIndices[si].category; + int32_t spanValue = spanIndices[si].spanValue; + int32_t length = spanIndices[si].length; + cfpos.setInt64IterationContext(si + 1); + if (si + 1 < spanIndicesCount) { + nextSpanStart = spanIndices[si + 1].start; + } + if (length == 0) { + // ICU-21871: Don't return fields on empty spans + i--; + continue; + } + if (cfpos.matchesField(spanCategory, spanValue)) { + fieldStart = i - fString.fZero; + int32_t end = fieldStart + length; + cfpos.setState( + spanCategory, + spanValue, + fieldStart, + end); + return true; + } else if (_field == Field(UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD)) { + // Special handling for ULISTFMT_ELEMENT_FIELD + if (cfpos.matchesField(UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD)) { + fieldStart = i - fString.fZero; + int32_t end = fieldStart + length; + cfpos.setState( + UFIELD_CATEGORY_LIST, + ULISTFMT_ELEMENT_FIELD, + fieldStart, + end); + return true; + } else { + // Failed to match; jump ahead + i += length - 1; + // goto loopend + } + } + } + // Special case: skip over INTEGER; will be coalesced later. + else if (_field == Field(UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD)) { + _field = kUndefinedField; + } + // No field starting at this position. + else if (_field.isUndefined() || _field == kEndField) { + // goto loopend + } + // No SpanField + else if (cfpos.matchesField(_field.getCategory(), _field.getField())) { + fieldStart = i - fString.fZero; + currField = _field; + } + // loopend: + prevIsSpan = false; + prevIsNumeric = false; + prevIsInteger = false; + } + + U_ASSERT(currField == kUndefinedField); + // Always set the position to the end so that we don't revisit previous sections + cfpos.setState( + cfpos.getCategory(), + cfpos.getField(), + fString.fLength, + fString.fLength); + return false; +} + +void FormattedValueStringBuilderImpl::appendSpanInfo(UFieldCategory category, int32_t spanValue, int32_t start, int32_t length, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + U_ASSERT(spanIndices.getCapacity() >= spanIndicesCount); + if (spanIndices.getCapacity() == spanIndicesCount) { + if (!spanIndices.resize(spanIndicesCount * 2, spanIndicesCount)) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + spanIndices[spanIndicesCount] = {category, spanValue, start, length}; + spanIndicesCount++; +} + +void FormattedValueStringBuilderImpl::prependSpanInfo(UFieldCategory category, int32_t spanValue, int32_t start, int32_t length, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + U_ASSERT(spanIndices.getCapacity() >= spanIndicesCount); + if (spanIndices.getCapacity() == spanIndicesCount) { + if (!spanIndices.resize(spanIndicesCount * 2, spanIndicesCount)) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + for (int32_t i = spanIndicesCount - 1; i >= 0; i--) { + spanIndices[i+1] = spanIndices[i]; + } + spanIndices[0] = {category, spanValue, start, length}; + spanIndicesCount++; +} + +bool FormattedValueStringBuilderImpl::isIntOrGroup(Field field) { + return field == Field(UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD) + || field == Field(UFIELD_CATEGORY_NUMBER, UNUM_GROUPING_SEPARATOR_FIELD); +} + +bool FormattedValueStringBuilderImpl::isTrimmable(Field field) { + return field != Field(UFIELD_CATEGORY_NUMBER, UNUM_GROUPING_SEPARATOR_FIELD) + && field.getCategory() != UFIELD_CATEGORY_LIST; +} + +int32_t FormattedValueStringBuilderImpl::trimBack(int32_t limit) const { + return unisets::get(unisets::DEFAULT_IGNORABLES)->spanBack( + fString.getCharPtr() + fString.fZero, + limit, + USET_SPAN_CONTAINED); +} + +int32_t FormattedValueStringBuilderImpl::trimFront(int32_t start) const { + return start + unisets::get(unisets::DEFAULT_IGNORABLES)->span( + fString.getCharPtr() + fString.fZero + start, + fString.fLength - start, + USET_SPAN_CONTAINED); +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/formattedvalue.cpp b/intl/icu/source/i18n/formattedvalue.cpp new file mode 100644 index 0000000000..aacd6ac70e --- /dev/null +++ b/intl/icu/source/i18n/formattedvalue.cpp @@ -0,0 +1,234 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/formattedvalue.h" +#include "formattedval_impl.h" +#include "capi_helper.h" + +U_NAMESPACE_BEGIN + + +ConstrainedFieldPosition::ConstrainedFieldPosition() {} + +ConstrainedFieldPosition::~ConstrainedFieldPosition() {} + +void ConstrainedFieldPosition::reset() { + fContext = 0LL; + fField = 0; + fStart = 0; + fLimit = 0; + fConstraint = UCFPOS_CONSTRAINT_NONE; + fCategory = UFIELD_CATEGORY_UNDEFINED; +} + +void ConstrainedFieldPosition::constrainCategory(int32_t category) { + fConstraint = UCFPOS_CONSTRAINT_CATEGORY; + fCategory = category; +} + +void ConstrainedFieldPosition::constrainField(int32_t category, int32_t field) { + fConstraint = UCFPOS_CONSTRAINT_FIELD; + fCategory = category; + fField = field; +} + +void ConstrainedFieldPosition::setInt64IterationContext(int64_t context) { + fContext = context; +} + +UBool ConstrainedFieldPosition::matchesField(int32_t category, int32_t field) const { + switch (fConstraint) { + case UCFPOS_CONSTRAINT_NONE: + return true; + case UCFPOS_CONSTRAINT_CATEGORY: + return fCategory == category; + case UCFPOS_CONSTRAINT_FIELD: + return fCategory == category && fField == field; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +void ConstrainedFieldPosition::setState( + int32_t category, + int32_t field, + int32_t start, + int32_t limit) { + fCategory = category; + fField = field; + fStart = start; + fLimit = limit; +} + + +FormattedValue::~FormattedValue() = default; + + +/////////////////////// +/// C API FUNCTIONS /// +/////////////////////// + +struct UConstrainedFieldPositionImpl : public UMemory, + // Magic number as ASCII == "UCF" + public IcuCApiHelper { + ConstrainedFieldPosition fImpl; +}; + +U_CAPI UConstrainedFieldPosition* U_EXPORT2 +ucfpos_open(UErrorCode* ec) { + auto* impl = new UConstrainedFieldPositionImpl(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return impl->exportForC(); +} + +U_CAPI void U_EXPORT2 +ucfpos_reset(UConstrainedFieldPosition* ptr, UErrorCode* ec) { + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + impl->fImpl.reset(); +} + +U_CAPI void U_EXPORT2 +ucfpos_constrainCategory(UConstrainedFieldPosition* ptr, int32_t category, UErrorCode* ec) { + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + impl->fImpl.constrainCategory(category); +} + +U_CAPI void U_EXPORT2 +ucfpos_constrainField(UConstrainedFieldPosition* ptr, int32_t category, int32_t field, UErrorCode* ec) { + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + impl->fImpl.constrainField(category, field); +} + +U_CAPI int32_t U_EXPORT2 +ucfpos_getCategory(const UConstrainedFieldPosition* ptr, UErrorCode* ec) { + const auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return UFIELD_CATEGORY_UNDEFINED; + } + return impl->fImpl.getCategory(); +} + +U_CAPI int32_t U_EXPORT2 +ucfpos_getField(const UConstrainedFieldPosition* ptr, UErrorCode* ec) { + const auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + return impl->fImpl.getField(); +} + +U_CAPI void U_EXPORT2 +ucfpos_getIndexes(const UConstrainedFieldPosition* ptr, int32_t* pStart, int32_t* pLimit, UErrorCode* ec) { + const auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + *pStart = impl->fImpl.getStart(); + *pLimit = impl->fImpl.getLimit(); +} + +U_CAPI int64_t U_EXPORT2 +ucfpos_getInt64IterationContext(const UConstrainedFieldPosition* ptr, UErrorCode* ec) { + const auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + return impl->fImpl.getInt64IterationContext(); +} + +U_CAPI void U_EXPORT2 +ucfpos_setInt64IterationContext(UConstrainedFieldPosition* ptr, int64_t context, UErrorCode* ec) { + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + impl->fImpl.setInt64IterationContext(context); +} + +U_CAPI UBool U_EXPORT2 +ucfpos_matchesField(const UConstrainedFieldPosition* ptr, int32_t category, int32_t field, UErrorCode* ec) { + const auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + return impl->fImpl.matchesField(category, field); +} + +U_CAPI void U_EXPORT2 +ucfpos_setState( + UConstrainedFieldPosition* ptr, + int32_t category, + int32_t field, + int32_t start, + int32_t limit, + UErrorCode* ec) { + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, *ec); + if (U_FAILURE(*ec)) { + return; + } + impl->fImpl.setState(category, field, start, limit); +} + +U_CAPI void U_EXPORT2 +ucfpos_close(UConstrainedFieldPosition* ptr) { + UErrorCode localStatus = U_ZERO_ERROR; + auto* impl = UConstrainedFieldPositionImpl::validate(ptr, localStatus); + delete impl; +} + + +U_CAPI const char16_t* U_EXPORT2 +ufmtval_getString( + const UFormattedValue* ufmtval, + int32_t* pLength, + UErrorCode* ec) { + const auto* impl = UFormattedValueApiHelper::validate(ufmtval, *ec); + if (U_FAILURE(*ec)) { + return nullptr; + } + UnicodeString readOnlyAlias = impl->fFormattedValue->toTempString(*ec); + if (U_FAILURE(*ec)) { + return nullptr; + } + if (pLength != nullptr) { + *pLength = readOnlyAlias.length(); + } + // Note: this line triggers -Wreturn-local-addr, but it is safe because toTempString is + // defined to return memory owned by the ufmtval argument. + return readOnlyAlias.getBuffer(); +} + + +U_CAPI UBool U_EXPORT2 +ufmtval_nextPosition( + const UFormattedValue* ufmtval, + UConstrainedFieldPosition* ucfpos, + UErrorCode* ec) { + const auto* fmtval = UFormattedValueApiHelper::validate(ufmtval, *ec); + auto* cfpos = UConstrainedFieldPositionImpl::validate(ucfpos, *ec); + if (U_FAILURE(*ec)) { + return false; + } + return fmtval->fFormattedValue->nextPosition(cfpos->fImpl, *ec); +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/fphdlimp.cpp b/intl/icu/source/i18n/fphdlimp.cpp new file mode 100644 index 0000000000..e170dc4b99 --- /dev/null +++ b/intl/icu/source/i18n/fphdlimp.cpp @@ -0,0 +1,125 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2009-2015, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "fphdlimp.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +// utility FieldPositionHandler +// base class, null implementation + +FieldPositionHandler::~FieldPositionHandler() { +} + +void FieldPositionHandler::setShift(int32_t delta) { + fShift = delta; +} + + +// utility subclass FieldPositionOnlyHandler + +FieldPositionOnlyHandler::FieldPositionOnlyHandler(FieldPosition& _pos) + : pos(_pos) { +} + +FieldPositionOnlyHandler::~FieldPositionOnlyHandler() { +} + +void +FieldPositionOnlyHandler::addAttribute(int32_t id, int32_t start, int32_t limit) { + if (pos.getField() == id && (!acceptFirstOnly || !seenFirst)) { + seenFirst = true; + pos.setBeginIndex(start + fShift); + pos.setEndIndex(limit + fShift); + } +} + +void +FieldPositionOnlyHandler::shiftLast(int32_t delta) { + if (delta != 0 && pos.getField() != FieldPosition::DONT_CARE && pos.getBeginIndex() != -1) { + pos.setBeginIndex(delta + pos.getBeginIndex()); + pos.setEndIndex(delta + pos.getEndIndex()); + } +} + +UBool +FieldPositionOnlyHandler::isRecording() const { + return pos.getField() != FieldPosition::DONT_CARE; +} + +void FieldPositionOnlyHandler::setAcceptFirstOnly(UBool acceptFirstOnly) { + this->acceptFirstOnly = acceptFirstOnly; +} + + +// utility subclass FieldPositionIteratorHandler + +FieldPositionIteratorHandler::FieldPositionIteratorHandler(FieldPositionIterator* posIter, + UErrorCode& _status) + : iter(posIter), vec(nullptr), status(_status), fCategory(UFIELD_CATEGORY_UNDEFINED) { + if (iter && U_SUCCESS(status)) { + vec = new UVector32(status); + } +} + +FieldPositionIteratorHandler::FieldPositionIteratorHandler( + UVector32* vec, + UErrorCode& status) + : iter(nullptr), vec(vec), status(status), fCategory(UFIELD_CATEGORY_UNDEFINED) { +} + +FieldPositionIteratorHandler::~FieldPositionIteratorHandler() { + // setData adopts the vec regardless of status, so it's safe to null it + if (iter) { + iter->setData(vec, status); + } + // if iter is null, we never allocated vec, so no need to free it + vec = nullptr; +} + +void +FieldPositionIteratorHandler::addAttribute(int32_t id, int32_t start, int32_t limit) { + if (vec && U_SUCCESS(status) && start < limit) { + int32_t size = vec->size(); + vec->addElement(fCategory, status); + vec->addElement(id, status); + vec->addElement(start + fShift, status); + vec->addElement(limit + fShift, status); + if (!U_SUCCESS(status)) { + vec->setSize(size); + } + } +} + +void +FieldPositionIteratorHandler::shiftLast(int32_t delta) { + if (U_SUCCESS(status) && delta != 0) { + int32_t i = vec->size(); + if (i > 0) { + --i; + vec->setElementAt(delta + vec->elementAti(i), i); + --i; + vec->setElementAt(delta + vec->elementAti(i), i); + } + } +} + +UBool +FieldPositionIteratorHandler::isRecording() const { + return U_SUCCESS(status); +} + +U_NAMESPACE_END + +#endif /* !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/fphdlimp.h b/intl/icu/source/i18n/fphdlimp.h new file mode 100644 index 0000000000..ad09c6c903 --- /dev/null +++ b/intl/icu/source/i18n/fphdlimp.h @@ -0,0 +1,109 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2009-2015, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef FPHDLIMP_H +#define FPHDLIMP_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/fieldpos.h" +#include "unicode/fpositer.h" +#include "unicode/formattedvalue.h" + +U_NAMESPACE_BEGIN + +// utility FieldPositionHandler +// base class, null implementation + +class U_I18N_API FieldPositionHandler: public UMemory { + protected: + int32_t fShift = 0; + + public: + virtual ~FieldPositionHandler(); + virtual void addAttribute(int32_t id, int32_t start, int32_t limit) = 0; + virtual void shiftLast(int32_t delta) = 0; + virtual UBool isRecording() const = 0; + + void setShift(int32_t delta); +}; + + +// utility subclass FieldPositionOnlyHandler + +class FieldPositionOnlyHandler : public FieldPositionHandler { + FieldPosition& pos; + UBool acceptFirstOnly = false; + UBool seenFirst = false; + + public: + FieldPositionOnlyHandler(FieldPosition& pos); + virtual ~FieldPositionOnlyHandler(); + + void addAttribute(int32_t id, int32_t start, int32_t limit) override; + void shiftLast(int32_t delta) override; + UBool isRecording() const override; + + /** + * Enable this option to lock in the FieldPosition value after seeing the + * first occurrence of the field. The default behavior is to take the last + * occurrence. + */ + void setAcceptFirstOnly(UBool acceptFirstOnly); +}; + + +// utility subclass FieldPositionIteratorHandler +// exported as U_I18N_API for tests + +class U_I18N_API FieldPositionIteratorHandler : public FieldPositionHandler { + FieldPositionIterator* iter; // can be nullptr + UVector32* vec; + UErrorCode status; + UFieldCategory fCategory; + + // Note, we keep a reference to status, so if status is on the stack, we have + // to be destroyed before status goes out of scope. Easiest thing is to + // allocate us on the stack in the same (or narrower) scope as status has. + // This attempts to encourage that by blocking heap allocation. + static void* U_EXPORT2 operator new(size_t) noexcept = delete; + static void* U_EXPORT2 operator new[](size_t) noexcept = delete; +#if U_HAVE_PLACEMENT_NEW + static void* U_EXPORT2 operator new(size_t, void*) noexcept = delete; +#endif + + public: + FieldPositionIteratorHandler(FieldPositionIterator* posIter, UErrorCode& status); + /** If using this constructor, you must call getError() when done formatting! */ + FieldPositionIteratorHandler(UVector32* vec, UErrorCode& status); + ~FieldPositionIteratorHandler(); + + void addAttribute(int32_t id, int32_t start, int32_t limit) override; + void shiftLast(int32_t delta) override; + UBool isRecording() const override; + + /** Copies a failed error code into _status. */ + inline void getError(UErrorCode& _status) { + if (U_SUCCESS(_status) && U_FAILURE(status)) { + _status = status; + } + } + + inline void setCategory(UFieldCategory category) { + fCategory = category; + } +}; + +U_NAMESPACE_END + +#endif /* !UCONFIG_NO_FORMATTING */ + +#endif /* FPHDLIMP_H */ diff --git a/intl/icu/source/i18n/fpositer.cpp b/intl/icu/source/i18n/fpositer.cpp new file mode 100644 index 0000000000..68f1edcc17 --- /dev/null +++ b/intl/icu/source/i18n/fpositer.cpp @@ -0,0 +1,114 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2009-2012, International Business Machines Corporation and +* others. All Rights Reserved. +****************************************************************************** +* Date Name Description +* 12/14/09 doug Creation. +****************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/fpositer.h" +#include "cmemory.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +FieldPositionIterator::~FieldPositionIterator() { + delete data; + data = nullptr; + pos = -1; +} + +FieldPositionIterator::FieldPositionIterator() + : data(nullptr), pos(-1) { +} + +FieldPositionIterator::FieldPositionIterator(const FieldPositionIterator &rhs) + : UObject(rhs), data(nullptr), pos(rhs.pos) { + + if (rhs.data) { + UErrorCode status = U_ZERO_ERROR; + data = new UVector32(status); + data->assign(*rhs.data, status); + if (status != U_ZERO_ERROR) { + delete data; + data = nullptr; + pos = -1; + } + } +} + +bool FieldPositionIterator::operator==(const FieldPositionIterator &rhs) const { + if (&rhs == this) { + return true; + } + if (pos != rhs.pos) { + return false; + } + if (!data) { + return rhs.data == nullptr; + } + return rhs.data ? data->operator==(*rhs.data) : false; +} + +void FieldPositionIterator::setData(UVector32 *adopt, UErrorCode& status) { + // Verify that adopt has valid data, and update status if it doesn't. + if (U_SUCCESS(status)) { + if (adopt) { + if (adopt->size() == 0) { + delete adopt; + adopt = nullptr; + } else if ((adopt->size() % 4) != 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + for (int i = 2; i < adopt->size(); i += 4) { + if (adopt->elementAti(i) >= adopt->elementAti(i+1)) { + status = U_ILLEGAL_ARGUMENT_ERROR; + break; + } + } + } + } + } + + // We own the data, even if status is in error, so we need to delete it now + // if we're not keeping track of it. + if (!U_SUCCESS(status)) { + delete adopt; + return; + } + + delete data; + data = adopt; + pos = adopt == nullptr ? -1 : 0; +} + +UBool FieldPositionIterator::next(FieldPosition& fp) { + if (pos == -1) { + return false; + } + + // Ignore the first element of the tetrad: used for field category + pos++; + fp.setField(data->elementAti(pos++)); + fp.setBeginIndex(data->elementAti(pos++)); + fp.setEndIndex(data->elementAti(pos++)); + + if (pos == data->size()) { + pos = -1; + } + + return true; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + diff --git a/intl/icu/source/i18n/funcrepl.cpp b/intl/icu/source/i18n/funcrepl.cpp new file mode 100644 index 0000000000..f093a1e437 --- /dev/null +++ b/intl/icu/source/i18n/funcrepl.cpp @@ -0,0 +1,130 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2002-2012, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 02/04/2002 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" +#include "unicode/uniset.h" +#include "funcrepl.h" + +static const char16_t AMPERSAND = 38; // '&' +static const char16_t OPEN[] = {40,32,0}; // "( " +static const char16_t CLOSE[] = {32,41,0}; // " )" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(FunctionReplacer) + +/** + * Construct a replacer that takes the output of the given + * replacer, passes it through the given transliterator, and emits + * the result as output. + */ +FunctionReplacer::FunctionReplacer(Transliterator* adoptedTranslit, + UnicodeFunctor* adoptedReplacer) { + translit = adoptedTranslit; + replacer = adoptedReplacer; +} + +/** + * Copy constructor. + */ +FunctionReplacer::FunctionReplacer(const FunctionReplacer& other) : + UnicodeFunctor(other), + UnicodeReplacer(other) +{ + translit = other.translit->clone(); + replacer = other.replacer->clone(); +} + +/** + * Destructor + */ +FunctionReplacer::~FunctionReplacer() { + delete translit; + delete replacer; +} + +/** + * Implement UnicodeFunctor + */ +FunctionReplacer* FunctionReplacer::clone() const { + return new FunctionReplacer(*this); +} + +/** + * UnicodeFunctor API. Cast 'this' to a UnicodeReplacer* pointer + * and return the pointer. + */ +UnicodeReplacer* FunctionReplacer::toReplacer() const { + FunctionReplacer *nonconst_this = const_cast(this); + UnicodeReplacer *nonconst_base = static_cast(nonconst_this); + + return nonconst_base; +} + +/** + * UnicodeReplacer API + */ +int32_t FunctionReplacer::replace(Replaceable& text, + int32_t start, + int32_t limit, + int32_t& cursor) +{ + + // First delegate to subordinate replacer + int32_t len = replacer->toReplacer()->replace(text, start, limit, cursor); + limit = start + len; + + // Now transliterate + limit = translit->transliterate(text, start, limit); + + return limit - start; +} + +/** + * UnicodeReplacer API + */ +UnicodeString& FunctionReplacer::toReplacerPattern(UnicodeString& rule, + UBool escapeUnprintable) const { + UnicodeString str; + rule.truncate(0); + rule.append(AMPERSAND); + rule.append(translit->getID()); + rule.append(OPEN, 2); + rule.append(replacer->toReplacer()->toReplacerPattern(str, escapeUnprintable)); + rule.append(CLOSE, 2); + return rule; +} + +/** + * Implement UnicodeReplacer + */ +void FunctionReplacer::addReplacementSetTo(UnicodeSet& toUnionTo) const { + UnicodeSet set; + toUnionTo.addAll(translit->getTargetSet(set)); +} + +/** + * UnicodeFunctor API + */ +void FunctionReplacer::setData(const TransliterationRuleData* d) { + replacer->setData(d); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +//eof diff --git a/intl/icu/source/i18n/funcrepl.h b/intl/icu/source/i18n/funcrepl.h new file mode 100644 index 0000000000..529a10ebbf --- /dev/null +++ b/intl/icu/source/i18n/funcrepl.h @@ -0,0 +1,121 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2002-2011, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 02/04/2002 aliu Creation. +********************************************************************** +*/ + +#ifndef FUNCREPL_H +#define FUNCREPL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unifunct.h" +#include "unicode/unirepl.h" + +U_NAMESPACE_BEGIN + +class Transliterator; + +/** + * A replacer that calls a transliterator to generate its output text. + * The input text to the transliterator is the output of another + * UnicodeReplacer object. That is, this replacer wraps another + * replacer with a transliterator. + * + * @author Alan Liu + */ +class FunctionReplacer : public UnicodeFunctor, public UnicodeReplacer { + + private: + + /** + * The transliterator. Must not be null. OWNED. + */ + Transliterator* translit; + + /** + * The replacer object. This generates text that is then + * processed by 'translit'. Must not be null. OWNED. + */ + UnicodeFunctor* replacer; + + public: + + /** + * Construct a replacer that takes the output of the given + * replacer, passes it through the given transliterator, and emits + * the result as output. + */ + FunctionReplacer(Transliterator* adoptedTranslit, + UnicodeFunctor* adoptedReplacer); + + /** + * Copy constructor. + */ + FunctionReplacer(const FunctionReplacer& other); + + /** + * Destructor + */ + virtual ~FunctionReplacer(); + + /** + * Implement UnicodeFunctor + */ + virtual FunctionReplacer* clone() const override; + + /** + * UnicodeFunctor API. Cast 'this' to a UnicodeReplacer* pointer + * and return the pointer. + */ + virtual UnicodeReplacer* toReplacer() const override; + + /** + * UnicodeReplacer API + */ + virtual int32_t replace(Replaceable& text, + int32_t start, + int32_t limit, + int32_t& cursor) override; + + /** + * UnicodeReplacer API + */ + virtual UnicodeString& toReplacerPattern(UnicodeString& rule, + UBool escapeUnprintable) const override; + + /** + * Implement UnicodeReplacer + */ + virtual void addReplacementSetTo(UnicodeSet& toUnionTo) const override; + + /** + * UnicodeFunctor API + */ + virtual void setData(const TransliterationRuleData*) override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ +#endif + +//eof diff --git a/intl/icu/source/i18n/gender.cpp b/intl/icu/source/i18n/gender.cpp new file mode 100644 index 0000000000..97d161de32 --- /dev/null +++ b/intl/icu/source/i18n/gender.cpp @@ -0,0 +1,254 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2008-2013, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* +* File GENDER.CPP +* +* Modification History:* +* Date Name Description +* +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/gender.h" +#include "unicode/ugender.h" +#include "unicode/ures.h" + +#include "cmemory.h" +#include "cstring.h" +#include "mutex.h" +#include "uassert.h" +#include "ucln_in.h" +#include "umutex.h" +#include "uhash.h" + +static UHashtable* gGenderInfoCache = nullptr; + +static const char* gNeutralStr = "neutral"; +static const char* gMailTaintsStr = "maleTaints"; +static const char* gMixedNeutralStr = "mixedNeutral"; +static icu::GenderInfo* gObjs = nullptr; +static icu::UInitOnce gGenderInitOnce {}; + +enum GenderStyle { + NEUTRAL, + MIXED_NEUTRAL, + MALE_TAINTS, + GENDER_STYLE_LENGTH +}; + +U_CDECL_BEGIN + +static UBool U_CALLCONV gender_cleanup() { + if (gGenderInfoCache != nullptr) { + uhash_close(gGenderInfoCache); + gGenderInfoCache = nullptr; + delete [] gObjs; + } + gGenderInitOnce.reset(); + return true; +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +void U_CALLCONV GenderInfo_initCache(UErrorCode &status) { + ucln_i18n_registerCleanup(UCLN_I18N_GENDERINFO, gender_cleanup); + U_ASSERT(gGenderInfoCache == nullptr); + if (U_FAILURE(status)) { + return; + } + gObjs = new GenderInfo[GENDER_STYLE_LENGTH]; + if (gObjs == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (int i = 0; i < GENDER_STYLE_LENGTH; i++) { + gObjs[i]._style = i; + } + gGenderInfoCache = uhash_open(uhash_hashChars, uhash_compareChars, nullptr, &status); + if (U_FAILURE(status)) { + delete [] gObjs; + return; + } + uhash_setKeyDeleter(gGenderInfoCache, uprv_free); +} + + +GenderInfo::GenderInfo() { +} + +GenderInfo::~GenderInfo() { +} + +const GenderInfo* GenderInfo::getInstance(const Locale& locale, UErrorCode& status) { + // Make sure our cache exists. + umtx_initOnce(gGenderInitOnce, &GenderInfo_initCache, status); + if (U_FAILURE(status)) { + return nullptr; + } + + static UMutex gGenderMetaLock; + const GenderInfo* result = nullptr; + const char* key = locale.getName(); + { + Mutex lock(&gGenderMetaLock); + result = (const GenderInfo*) uhash_get(gGenderInfoCache, key); + } + if (result) { + return result; + } + + // On cache miss, try to create GenderInfo from CLDR data + result = loadInstance(locale, status); + if (U_FAILURE(status)) { + return nullptr; + } + + // Try to put our GenderInfo object in cache. If there is a race condition, + // favor the GenderInfo object that is already in the cache. + { + Mutex lock(&gGenderMetaLock); + GenderInfo* temp = (GenderInfo*) uhash_get(gGenderInfoCache, key); + if (temp) { + result = temp; + } else { + uhash_put(gGenderInfoCache, uprv_strdup(key), (void*) result, &status); + if (U_FAILURE(status)) { + return nullptr; + } + } + } + return result; +} + +const GenderInfo* GenderInfo::loadInstance(const Locale& locale, UErrorCode& status) { + LocalUResourceBundlePointer rb( + ures_openDirect(nullptr, "genderList", &status)); + if (U_FAILURE(status)) { + return nullptr; + } + LocalUResourceBundlePointer locRes(ures_getByKey(rb.getAlias(), "genderList", nullptr, &status)); + if (U_FAILURE(status)) { + return nullptr; + } + int32_t resLen = 0; + const char* curLocaleName = locale.getName(); + UErrorCode key_status = U_ZERO_ERROR; + const char16_t* s = ures_getStringByKey(locRes.getAlias(), curLocaleName, &resLen, &key_status); + if (s == nullptr) { + key_status = U_ZERO_ERROR; + char parentLocaleName[ULOC_FULLNAME_CAPACITY]; + uprv_strcpy(parentLocaleName, curLocaleName); + while (s == nullptr && uloc_getParent(parentLocaleName, parentLocaleName, ULOC_FULLNAME_CAPACITY, &key_status) > 0) { + key_status = U_ZERO_ERROR; + resLen = 0; + s = ures_getStringByKey(locRes.getAlias(), parentLocaleName, &resLen, &key_status); + key_status = U_ZERO_ERROR; + } + } + if (s == nullptr) { + return &gObjs[NEUTRAL]; + } + char type_str[256] = ""; + u_UCharsToChars(s, type_str, resLen + 1); + if (uprv_strcmp(type_str, gNeutralStr) == 0) { + return &gObjs[NEUTRAL]; + } + if (uprv_strcmp(type_str, gMixedNeutralStr) == 0) { + return &gObjs[MIXED_NEUTRAL]; + } + if (uprv_strcmp(type_str, gMailTaintsStr) == 0) { + return &gObjs[MALE_TAINTS]; + } + return &gObjs[NEUTRAL]; +} + +UGender GenderInfo::getListGender(const UGender* genders, int32_t length, UErrorCode& status) const { + if (U_FAILURE(status)) { + return UGENDER_OTHER; + } + if (length == 0) { + return UGENDER_OTHER; + } + if (length == 1) { + return genders[0]; + } + UBool has_female = false; + UBool has_male = false; + switch (_style) { + case NEUTRAL: + return UGENDER_OTHER; + case MIXED_NEUTRAL: + for (int32_t i = 0; i < length; ++i) { + switch (genders[i]) { + case UGENDER_OTHER: + return UGENDER_OTHER; + break; + case UGENDER_FEMALE: + if (has_male) { + return UGENDER_OTHER; + } + has_female = true; + break; + case UGENDER_MALE: + if (has_female) { + return UGENDER_OTHER; + } + has_male = true; + break; + default: + break; + } + } + return has_male ? UGENDER_MALE : UGENDER_FEMALE; + break; + case MALE_TAINTS: + for (int32_t i = 0; i < length; ++i) { + if (genders[i] != UGENDER_FEMALE) { + return UGENDER_MALE; + } + } + return UGENDER_FEMALE; + break; + default: + return UGENDER_OTHER; + break; + } +} + +const GenderInfo* GenderInfo::getNeutralInstance() { + return &gObjs[NEUTRAL]; +} + +const GenderInfo* GenderInfo::getMixedNeutralInstance() { + return &gObjs[MIXED_NEUTRAL]; +} + +const GenderInfo* GenderInfo::getMaleTaintsInstance() { + return &gObjs[MALE_TAINTS]; +} + +U_NAMESPACE_END + +U_CAPI const UGenderInfo* U_EXPORT2 +ugender_getInstance(const char* locale, UErrorCode* status) { + return (const UGenderInfo*) icu::GenderInfo::getInstance(locale, *status); +} + +U_CAPI UGender U_EXPORT2 +ugender_getListGender(const UGenderInfo* genderInfo, const UGender* genders, int32_t size, UErrorCode* status) { + return ((const icu::GenderInfo *)genderInfo)->getListGender(genders, size, *status); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/gregocal.cpp b/intl/icu/source/i18n/gregocal.cpp new file mode 100644 index 0000000000..5fd71d496c --- /dev/null +++ b/intl/icu/source/i18n/gregocal.cpp @@ -0,0 +1,1307 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File GREGOCAL.CPP +* +* Modification History: +* +* Date Name Description +* 02/05/97 clhuang Creation. +* 03/28/97 aliu Made highly questionable fix to computeFields to +* handle DST correctly. +* 04/22/97 aliu Cleaned up code drastically. Added monthLength(). +* Finished unimplemented parts of computeTime() for +* week-based date determination. Removed quetionable +* fix and wrote correct fix for computeFields() and +* daylight time handling. Rewrote inDaylightTime() +* and computeFields() to handle sensitive Daylight to +* Standard time transitions correctly. +* 05/08/97 aliu Added code review changes. Fixed isLeapYear() to +* not cutover. +* 08/12/97 aliu Added equivalentTo. Misc other fixes. Updated +* add() from Java source. +* 07/28/98 stephen Sync up with JDK 1.2 +* 09/14/98 stephen Changed type of kOneDay, kOneWeek to double. +* Fixed bug in roll() +* 10/15/99 aliu Fixed j31, incorrect WEEK_OF_YEAR computation. +* 10/15/99 aliu Fixed j32, cannot set date to Feb 29 2000 AD. +* {JDK bug 4210209 4209272} +* 11/15/99 weiv Added YEAR_WOY and DOW_LOCAL computation +* to timeToFields method, updated kMinValues, kMaxValues & kLeastMaxValues +* 12/09/99 aliu Fixed j81, calculation errors and roll bugs +* in year of cutover. +* 01/24/2000 aliu Revised computeJulianDay for YEAR YEAR_WOY WOY. +******************************************************************************** +*/ + +#include "unicode/utypes.h" +#include + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/gregocal.h" +#include "gregoimp.h" +#include "umutex.h" +#include "uassert.h" + +// ***************************************************************************** +// class GregorianCalendar +// ***************************************************************************** + +/** +* Note that the Julian date used here is not a true Julian date, since +* it is measured from midnight, not noon. This value is the Julian +* day number of January 1, 1970 (Gregorian calendar) at noon UTC. [LIU] +*/ + +static const int16_t kNumDays[] += {0,31,59,90,120,151,181,212,243,273,304,334}; // 0-based, for day-in-year +static const int16_t kLeapNumDays[] += {0,31,60,91,121,152,182,213,244,274,305,335}; // 0-based, for day-in-year +static const int8_t kMonthLength[] += {31,28,31,30,31,30,31,31,30,31,30,31}; // 0-based +static const int8_t kLeapMonthLength[] += {31,29,31,30,31,30,31,31,30,31,30,31}; // 0-based + +// setTimeInMillis() limits the Julian day range to +/-7F000000. +// This would seem to limit the year range to: +// ms=+183882168921600000 jd=7f000000 December 20, 5828963 AD +// ms=-184303902528000000 jd=81000000 September 20, 5838270 BC +// HOWEVER, CalendarRegressionTest/Test4167060 shows that the actual +// range limit on the year field is smaller (~ +/-140000). [alan 3.0] + +static const int32_t kGregorianCalendarLimits[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 1, 1}, // ERA + { 1, 1, 140742, 144683}, // YEAR + { 0, 0, 11, 11}, // MONTH + { 1, 1, 52, 53}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 28, 31}, // DAY_OF_MONTH + { 1, 1, 365, 366}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 4, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -140742, -140742, 140742, 144683}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -140742, -140742, 140742, 144683}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 11, 11}, // ORDINAL_MONTH +}; + +/* +* 
+*                            Greatest       Least 
+* Field name        Minimum   Minimum     Maximum     Maximum
+* ----------        -------   -------     -------     -------
+* ERA                     0         0           1           1
+* YEAR                    1         1      140742      144683
+* MONTH                   0         0          11          11
+* WEEK_OF_YEAR            1         1          52          53
+* WEEK_OF_MONTH           0         0           4           6
+* DAY_OF_MONTH            1         1          28          31
+* DAY_OF_YEAR             1         1         365         366
+* DAY_OF_WEEK             1         1           7           7
+* DAY_OF_WEEK_IN_MONTH   -1        -1           4           5
+* AM_PM                   0         0           1           1
+* HOUR                    0         0          11          11
+* HOUR_OF_DAY             0         0          23          23
+* MINUTE                  0         0          59          59
+* SECOND                  0         0          59          59
+* MILLISECOND             0         0         999         999
+* ZONE_OFFSET           -12*      -12*         12*         12*
+* DST_OFFSET              0         0           1*          1*
+* YEAR_WOY                1         1      140742      144683
+* DOW_LOCAL               1         1           7           7
+*
+* (*) In units of one-hour +*/ + +#if defined( U_DEBUG_CALSVC ) || defined (U_DEBUG_CAL) +#include +#endif + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(GregorianCalendar) + +// 00:00:00 UTC, October 15, 1582, expressed in ms from the epoch. +// Note that only Italy and other Catholic countries actually +// observed this cutover. Most other countries followed in +// the next few centuries, some as late as 1928. [LIU] +// in Java, -12219292800000L +//const UDate GregorianCalendar::kPapalCutover = -12219292800000L; +static const uint32_t kCutoverJulianDay = 2299161; +static const UDate kPapalCutover = (2299161.0 - kEpochStartAsJulianDay) * U_MILLIS_PER_DAY; +//static const UDate kPapalCutoverJulian = (2299161.0 - kEpochStartAsJulianDay); + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(UErrorCode& status) +: Calendar(status), +fGregorianCutover(kPapalCutover), +fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), +fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(TimeZone* zone, UErrorCode& status) +: Calendar(zone, Locale::getDefault(), status), +fGregorianCutover(kPapalCutover), +fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), +fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(const TimeZone& zone, UErrorCode& status) +: Calendar(zone, Locale::getDefault(), status), +fGregorianCutover(kPapalCutover), +fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), +fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(const Locale& aLocale, UErrorCode& status) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, status), +fGregorianCutover(kPapalCutover), +fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), +fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(TimeZone* zone, const Locale& aLocale, + UErrorCode& status) + : Calendar(zone, aLocale, status), + fGregorianCutover(kPapalCutover), + fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), + fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(const TimeZone& zone, const Locale& aLocale, + UErrorCode& status) + : Calendar(zone, aLocale, status), + fGregorianCutover(kPapalCutover), + fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), + fIsGregorian(true), fInvertGregorian(false) +{ + setTimeInMillis(getNow(), status); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(int32_t year, int32_t month, int32_t date, + UErrorCode& status) + : Calendar(TimeZone::createDefault(), Locale::getDefault(), status), + fGregorianCutover(kPapalCutover), + fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), + fIsGregorian(true), fInvertGregorian(false) +{ + set(UCAL_ERA, AD); + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(int32_t year, int32_t month, int32_t date, + int32_t hour, int32_t minute, UErrorCode& status) + : Calendar(TimeZone::createDefault(), Locale::getDefault(), status), + fGregorianCutover(kPapalCutover), + fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), + fIsGregorian(true), fInvertGregorian(false) +{ + set(UCAL_ERA, AD); + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); + set(UCAL_HOUR_OF_DAY, hour); + set(UCAL_MINUTE, minute); +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(int32_t year, int32_t month, int32_t date, + int32_t hour, int32_t minute, int32_t second, + UErrorCode& status) + : Calendar(TimeZone::createDefault(), Locale::getDefault(), status), + fGregorianCutover(kPapalCutover), + fCutoverJulianDay(kCutoverJulianDay), fNormalizedGregorianCutover(fGregorianCutover), fGregorianCutoverYear(1582), + fIsGregorian(true), fInvertGregorian(false) +{ + set(UCAL_ERA, AD); + set(UCAL_YEAR, year); + set(UCAL_MONTH, month); + set(UCAL_DATE, date); + set(UCAL_HOUR_OF_DAY, hour); + set(UCAL_MINUTE, minute); + set(UCAL_SECOND, second); +} + +// ------------------------------------- + +GregorianCalendar::~GregorianCalendar() +{ +} + +// ------------------------------------- + +GregorianCalendar::GregorianCalendar(const GregorianCalendar &source) +: Calendar(source), +fGregorianCutover(source.fGregorianCutover), +fCutoverJulianDay(source.fCutoverJulianDay), fNormalizedGregorianCutover(source.fNormalizedGregorianCutover), fGregorianCutoverYear(source.fGregorianCutoverYear), +fIsGregorian(source.fIsGregorian), fInvertGregorian(source.fInvertGregorian) +{ +} + +// ------------------------------------- + +GregorianCalendar* GregorianCalendar::clone() const +{ + return new GregorianCalendar(*this); +} + +// ------------------------------------- + +GregorianCalendar & +GregorianCalendar::operator=(const GregorianCalendar &right) +{ + if (this != &right) + { + Calendar::operator=(right); + fGregorianCutover = right.fGregorianCutover; + fNormalizedGregorianCutover = right.fNormalizedGregorianCutover; + fGregorianCutoverYear = right.fGregorianCutoverYear; + fCutoverJulianDay = right.fCutoverJulianDay; + } + return *this; +} + +// ------------------------------------- + +UBool GregorianCalendar::isEquivalentTo(const Calendar& other) const +{ + // Calendar override. + return Calendar::isEquivalentTo(other) && + fGregorianCutover == ((GregorianCalendar*)&other)->fGregorianCutover; +} + +// ------------------------------------- + +void +GregorianCalendar::setGregorianChange(UDate date, UErrorCode& status) +{ + if (U_FAILURE(status)) + return; + + // Precompute two internal variables which we use to do the actual + // cutover computations. These are the normalized cutover, which is the + // midnight at or before the cutover, and the cutover year. The + // normalized cutover is in pure date milliseconds; it contains no time + // of day or timezone component, and it used to compare against other + // pure date values. + double cutoverDay = ClockMath::floorDivide(date, (double)kOneDay); + + // Handle the rare case of numeric overflow where the user specifies a time + // outside of INT32_MIN .. INT32_MAX number of days. + + if (cutoverDay <= INT32_MIN) { + cutoverDay = INT32_MIN; + fGregorianCutover = fNormalizedGregorianCutover = cutoverDay * kOneDay; + } else if (cutoverDay >= INT32_MAX) { + cutoverDay = INT32_MAX; + fGregorianCutover = fNormalizedGregorianCutover = cutoverDay * kOneDay; + } else { + fNormalizedGregorianCutover = cutoverDay * kOneDay; + fGregorianCutover = date; + } + + // Normalize the year so BC values are represented as 0 and negative + // values. + GregorianCalendar *cal = new GregorianCalendar(getTimeZone(), status); + /* test for nullptr */ + if (cal == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if(U_FAILURE(status)) + return; + cal->setTime(date, status); + fGregorianCutoverYear = cal->get(UCAL_YEAR, status); + if (cal->get(UCAL_ERA, status) == BC) + fGregorianCutoverYear = 1 - fGregorianCutoverYear; + fCutoverJulianDay = (int32_t)cutoverDay; + delete cal; +} + + +void GregorianCalendar::handleComputeFields(int32_t julianDay, UErrorCode& status) { + int32_t eyear, month, dayOfMonth, dayOfYear, unusedRemainder; + + + if(U_FAILURE(status)) { + return; + } + +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: jd%d- (greg's %d)- [cut=%d]\n", + __FILE__, __LINE__, julianDay, getGregorianDayOfYear(), fCutoverJulianDay); +#endif + + + if (julianDay >= fCutoverJulianDay) { + month = getGregorianMonth(); + dayOfMonth = getGregorianDayOfMonth(); + dayOfYear = getGregorianDayOfYear(); + eyear = getGregorianYear(); + } else { + // The Julian epoch day (not the same as Julian Day) + // is zero on Saturday December 30, 0 (Gregorian). + int32_t julianEpochDay = julianDay - (kJan1_1JulianDay - 2); + eyear = (int32_t) ClockMath::floorDivide((4.0*julianEpochDay) + 1464.0, (int32_t) 1461, &unusedRemainder); + + // Compute the Julian calendar day number for January 1, eyear + int32_t january1 = 365*(eyear-1) + ClockMath::floorDivide(eyear-1, (int32_t)4); + dayOfYear = (julianEpochDay - january1); // 0-based + + // Julian leap years occurred historically every 4 years starting + // with 8 AD. Before 8 AD the spacing is irregular; every 3 years + // from 45 BC to 9 BC, and then none until 8 AD. However, we don't + // implement this historical detail; instead, we implement the + // computationally cleaner proleptic calendar, which assumes + // consistent 4-year cycles throughout time. + UBool isLeap = ((eyear&0x3) == 0); // equiv. to (eyear%4 == 0) + + // Common Julian/Gregorian calculation + int32_t correction = 0; + int32_t march1 = isLeap ? 60 : 59; // zero-based DOY for March 1 + if (dayOfYear >= march1) { + correction = isLeap ? 1 : 2; + } + month = (12 * (dayOfYear + correction) + 6) / 367; // zero-based month + dayOfMonth = dayOfYear - (isLeap?kLeapNumDays[month]:kNumDays[month]) + 1; // one-based DOM + ++dayOfYear; +#if defined (U_DEBUG_CAL) + // fprintf(stderr, "%d - %d[%d] + 1\n", dayOfYear, isLeap?kLeapNumDays[month]:kNumDays[month], month ); + // fprintf(stderr, "%s:%d: greg's HCF %d -> %d/%d/%d not %d/%d/%d\n", + // __FILE__, __LINE__,julianDay, + // eyear,month,dayOfMonth, + // getGregorianYear(), getGregorianMonth(), getGregorianDayOfMonth() ); + fprintf(stderr, "%s:%d: doy %d (greg's %d)- [cut=%d]\n", + __FILE__, __LINE__, dayOfYear, getGregorianDayOfYear(), fCutoverJulianDay); +#endif + + } + + // [j81] if we are after the cutover in its year, shift the day of the year + if((eyear == fGregorianCutoverYear) && (julianDay >= fCutoverJulianDay)) { + //from handleComputeMonthStart + int32_t gregShift = Grego::gregorianShift(eyear); +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: gregorian shift %d ::: doy%d => %d [cut=%d]\n", + __FILE__, __LINE__,gregShift, dayOfYear, dayOfYear+gregShift, fCutoverJulianDay); +#endif + dayOfYear += gregShift; + } + + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); + internalSet(UCAL_EXTENDED_YEAR, eyear); + int32_t era = AD; + if (eyear < 1) { + era = BC; + eyear = 1 - eyear; + } + internalSet(UCAL_ERA, era); + internalSet(UCAL_YEAR, eyear); +} + + +// ------------------------------------- + +UDate +GregorianCalendar::getGregorianChange() const +{ + return fGregorianCutover; +} + +// ------------------------------------- + +UBool +GregorianCalendar::isLeapYear(int32_t year) const +{ + // MSVC complains bitterly if we try to use Grego::isLeapYear here + // NOTE: year&0x3 == year%4 + return (year >= fGregorianCutoverYear ? + (((year&0x3) == 0) && ((year%100 != 0) || (year%400 == 0))) : // Gregorian + ((year&0x3) == 0)); // Julian +} + +// ------------------------------------- + +int32_t GregorianCalendar::handleComputeJulianDay(UCalendarDateFields bestField) +{ + fInvertGregorian = false; + + int32_t jd = Calendar::handleComputeJulianDay(bestField); + + if((bestField == UCAL_WEEK_OF_YEAR) && // if we are doing WOY calculations, we are counting relative to Jan 1 *julian* + (internalGet(UCAL_EXTENDED_YEAR)==fGregorianCutoverYear) && + jd >= fCutoverJulianDay) { + fInvertGregorian = true; // So that the Julian Jan 1 will be used in handleComputeMonthStart + return Calendar::handleComputeJulianDay(bestField); + } + + + // The following check handles portions of the cutover year BEFORE the + // cutover itself happens. + //if ((fIsGregorian==true) != (jd >= fCutoverJulianDay)) { /* cutoverJulianDay)) { */ + if ((fIsGregorian) != (jd >= fCutoverJulianDay)) { /* cutoverJulianDay)) { */ +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: jd [invert] %d\n", + __FILE__, __LINE__, jd); +#endif + fInvertGregorian = true; + jd = Calendar::handleComputeJulianDay(bestField); +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: fIsGregorian %s, fInvertGregorian %s - ", + __FILE__, __LINE__,fIsGregorian?"T":"F", fInvertGregorian?"T":"F"); + fprintf(stderr, " jd NOW %d\n", + jd); +#endif + } else { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: jd [==] %d - %sfIsGregorian %sfInvertGregorian, %d\n", + __FILE__, __LINE__, jd, fIsGregorian?"T":"F", fInvertGregorian?"T":"F", bestField); +#endif + } + + if(fIsGregorian && (internalGet(UCAL_EXTENDED_YEAR) == fGregorianCutoverYear)) { + int32_t gregShift = Grego::gregorianShift(internalGet(UCAL_EXTENDED_YEAR)); + if (bestField == UCAL_DAY_OF_YEAR) { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: [DOY%d] gregorian shift of JD %d += %d\n", + __FILE__, __LINE__, fFields[bestField],jd, gregShift); +#endif + jd -= gregShift; + } else if ( bestField == UCAL_WEEK_OF_MONTH ) { + int32_t weekShift = 14; +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: [WOY/WOM] gregorian week shift of %d += %d\n", + __FILE__, __LINE__, jd, weekShift); +#endif + jd += weekShift; // shift by weeks for week based fields. + } + } + + return jd; +} + +int32_t GregorianCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, + + UBool /* useMonth */) const +{ + GregorianCalendar *nonConstThis = (GregorianCalendar*)this; // cast away const + + // If the month is out of range, adjust it into range, and + // modify the extended year value accordingly. + if (month < 0 || month > 11) { + eyear += ClockMath::floorDivide(month, 12, &month); + } + + UBool isLeap = eyear%4 == 0; + int64_t y = (int64_t)eyear-1; + int64_t julianDay = 365*y + ClockMath::floorDivide(y, (int64_t)4) + (kJan1_1JulianDay - 3); + + nonConstThis->fIsGregorian = (eyear >= fGregorianCutoverYear); +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: (hcms%d/%d) fIsGregorian %s, fInvertGregorian %s\n", + __FILE__, __LINE__, eyear,month, fIsGregorian?"T":"F", fInvertGregorian?"T":"F"); +#endif + if (fInvertGregorian) { + nonConstThis->fIsGregorian = !fIsGregorian; + } + if (fIsGregorian) { + isLeap = isLeap && ((eyear%100 != 0) || (eyear%400 == 0)); + // Add 2 because Gregorian calendar starts 2 days after + // Julian calendar + int32_t gregShift = Grego::gregorianShift(eyear); +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: (hcms%d/%d) gregorian shift of %d += %d\n", + __FILE__, __LINE__, eyear, month, julianDay, gregShift); +#endif + julianDay += gregShift; + } + + // At this point julianDay indicates the day BEFORE the first + // day of January 1, of either the Julian or Gregorian + // calendar. + + if (month != 0) { + julianDay += isLeap?kLeapNumDays[month]:kNumDays[month]; + } + + return static_cast(julianDay); +} + +int32_t GregorianCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const +{ + // If the month is out of range, adjust it into range, and + // modify the extended year value accordingly. + if (month < 0 || month > 11) { + extendedYear += ClockMath::floorDivide(month, 12, &month); + } + + return isLeapYear(extendedYear) ? kLeapMonthLength[month] : kMonthLength[month]; +} + +int32_t GregorianCalendar::handleGetYearLength(int32_t eyear) const { + return isLeapYear(eyear) ? 366 : 365; +} + + +int32_t +GregorianCalendar::monthLength(int32_t month) const +{ + int32_t year = internalGet(UCAL_EXTENDED_YEAR); + return handleGetMonthLength(year, month); +} + +// ------------------------------------- + +int32_t +GregorianCalendar::monthLength(int32_t month, int32_t year) const +{ + return isLeapYear(year) ? kLeapMonthLength[month] : kMonthLength[month]; +} + +// ------------------------------------- + +int32_t +GregorianCalendar::yearLength(int32_t year) const +{ + return isLeapYear(year) ? 366 : 365; +} + +// ------------------------------------- + +int32_t +GregorianCalendar::yearLength() const +{ + return isLeapYear(internalGet(UCAL_YEAR)) ? 366 : 365; +} + +// ------------------------------------- + +/** +* After adjustments such as add(MONTH), add(YEAR), we don't want the +* month to jump around. E.g., we don't want Jan 31 + 1 month to go to Mar +* 3, we want it to go to Feb 28. Adjustments which might run into this +* problem call this method to retain the proper month. +*/ +void +GregorianCalendar::pinDayOfMonth() +{ + int32_t monthLen = monthLength(internalGetMonth()); + int32_t dom = internalGet(UCAL_DATE); + if(dom > monthLen) + set(UCAL_DATE, monthLen); +} + +// ------------------------------------- + + +UBool +GregorianCalendar::validateFields() const +{ + for (int32_t field = 0; field < UCAL_FIELD_COUNT; field++) { + // Ignore DATE and DAY_OF_YEAR which are handled below + if (field != UCAL_DATE && + field != UCAL_DAY_OF_YEAR && + isSet((UCalendarDateFields)field) && + ! boundsCheck(internalGet((UCalendarDateFields)field), (UCalendarDateFields)field)) + return false; + } + + // Values differ in Least-Maximum and Maximum should be handled + // specially. + if (isSet(UCAL_DATE)) { + int32_t date = internalGet(UCAL_DATE); + if (date < getMinimum(UCAL_DATE) || + date > monthLength(internalGetMonth())) { + return false; + } + } + + if (isSet(UCAL_DAY_OF_YEAR)) { + int32_t days = internalGet(UCAL_DAY_OF_YEAR); + if (days < 1 || days > yearLength()) { + return false; + } + } + + // Handle DAY_OF_WEEK_IN_MONTH, which must not have the value zero. + // We've checked against minimum and maximum above already. + if (isSet(UCAL_DAY_OF_WEEK_IN_MONTH) && + 0 == internalGet(UCAL_DAY_OF_WEEK_IN_MONTH)) { + return false; + } + + return true; +} + +// ------------------------------------- + +UBool +GregorianCalendar::boundsCheck(int32_t value, UCalendarDateFields field) const +{ + return value >= getMinimum(field) && value <= getMaximum(field); +} + +// ------------------------------------- + +UDate +GregorianCalendar::getEpochDay(UErrorCode& status) +{ + complete(status); + // Divide by 1000 (convert to seconds) in order to prevent overflow when + // dealing with UDate(Long.MIN_VALUE) and UDate(Long.MAX_VALUE). + double wallSec = internalGetTime()/1000 + (internalGet(UCAL_ZONE_OFFSET) + internalGet(UCAL_DST_OFFSET))/1000; + + return ClockMath::floorDivide(wallSec, kOneDay/1000.0); +} + +// ------------------------------------- + + +// ------------------------------------- + +/** +* Compute the julian day number of the day BEFORE the first day of +* January 1, year 1 of the given calendar. If julianDay == 0, it +* specifies (Jan. 1, 1) - 1, in whatever calendar we are using (Julian +* or Gregorian). +*/ +double GregorianCalendar::computeJulianDayOfYear(UBool isGregorian, + int32_t year, UBool& isLeap) +{ + isLeap = year%4 == 0; + int32_t y = year - 1; + double julianDay = 365.0*y + ClockMath::floorDivide(y, 4) + (kJan1_1JulianDay - 3); + + if (isGregorian) { + isLeap = isLeap && ((year%100 != 0) || (year%400 == 0)); + // Add 2 because Gregorian calendar starts 2 days after Julian calendar + julianDay += Grego::gregorianShift(year); + } + + return julianDay; +} + +// /** +// * Compute the day of week, relative to the first day of week, from +// * 0..6, of the current DOW_LOCAL or DAY_OF_WEEK fields. This is +// * equivalent to get(DOW_LOCAL) - 1. +// */ +// int32_t GregorianCalendar::computeRelativeDOW() const { +// int32_t relDow = 0; +// if (fStamp[UCAL_DOW_LOCAL] > fStamp[UCAL_DAY_OF_WEEK]) { +// relDow = internalGet(UCAL_DOW_LOCAL) - 1; // 1-based +// } else if (fStamp[UCAL_DAY_OF_WEEK] != kUnset) { +// relDow = internalGet(UCAL_DAY_OF_WEEK) - getFirstDayOfWeek(); +// if (relDow < 0) relDow += 7; +// } +// return relDow; +// } + +// /** +// * Compute the day of week, relative to the first day of week, +// * from 0..6 of the given julian day. +// */ +// int32_t GregorianCalendar::computeRelativeDOW(double julianDay) const { +// int32_t relDow = julianDayToDayOfWeek(julianDay) - getFirstDayOfWeek(); +// if (relDow < 0) { +// relDow += 7; +// } +// return relDow; +// } + +// /** +// * Compute the DOY using the WEEK_OF_YEAR field and the julian day +// * of the day BEFORE January 1 of a year (a return value from +// * computeJulianDayOfYear). +// */ +// int32_t GregorianCalendar::computeDOYfromWOY(double julianDayOfYear) const { +// // Compute DOY from day of week plus week of year + +// // Find the day of the week for the first of this year. This +// // is zero-based, with 0 being the locale-specific first day of +// // the week. Add 1 to get first day of year. +// int32_t fdy = computeRelativeDOW(julianDayOfYear + 1); + +// return +// // Compute doy of first (relative) DOW of WOY 1 +// (((7 - fdy) < getMinimalDaysInFirstWeek()) +// ? (8 - fdy) : (1 - fdy)) + +// // Adjust for the week number. +// + (7 * (internalGet(UCAL_WEEK_OF_YEAR) - 1)) + +// // Adjust for the DOW +// + computeRelativeDOW(); +// } + +// ------------------------------------- + +double +GregorianCalendar::millisToJulianDay(UDate millis) +{ + return (double)kEpochStartAsJulianDay + ClockMath::floorDivide(millis, (double)kOneDay); +} + +// ------------------------------------- + +UDate +GregorianCalendar::julianDayToMillis(double julian) +{ + return (UDate) ((julian - kEpochStartAsJulianDay) * (double) kOneDay); +} + +// ------------------------------------- + +int32_t +GregorianCalendar::aggregateStamp(int32_t stamp_a, int32_t stamp_b) +{ + return (((stamp_a != kUnset && stamp_b != kUnset) + ? uprv_max(stamp_a, stamp_b) + : (int32_t)kUnset)); +} + +// ------------------------------------- + +/** +* Roll a field by a signed amount. +* Note: This will be made public later. [LIU] +*/ + +void +GregorianCalendar::roll(EDateFields field, int32_t amount, UErrorCode& status) { + roll((UCalendarDateFields) field, amount, status); +} + +void +GregorianCalendar::roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) UPRV_NO_SANITIZE_UNDEFINED { + if((amount == 0) || U_FAILURE(status)) { + return; + } + + // J81 processing. (gregorian cutover) + UBool inCutoverMonth = false; + int32_t cMonthLen=0; // 'c' for cutover; in days + int32_t cDayOfMonth=0; // no discontinuity: [0, cMonthLen) + double cMonthStart=0.0; // in ms + + // Common code - see if we're in the cutover month of the cutover year + if(get(UCAL_EXTENDED_YEAR, status) == fGregorianCutoverYear) { + switch (field) { + case UCAL_DAY_OF_MONTH: + case UCAL_WEEK_OF_MONTH: + { + int32_t max = monthLength(internalGetMonth()); + UDate t = internalGetTime(); + // We subtract 1 from the DAY_OF_MONTH to make it zero-based, and an + // additional 10 if we are after the cutover. Thus the monthStart + // value will be correct iff we actually are in the cutover month. + cDayOfMonth = internalGet(UCAL_DAY_OF_MONTH) - ((t >= fGregorianCutover) ? 10 : 0); + cMonthStart = t - ((cDayOfMonth - 1) * kOneDay); + // A month containing the cutover is 10 days shorter. + if ((cMonthStart < fGregorianCutover) && + (cMonthStart + (cMonthLen=(max-10))*kOneDay >= fGregorianCutover)) { + inCutoverMonth = true; + } + } + break; + default: + ; + } + } + + switch (field) { + case UCAL_WEEK_OF_YEAR: { + // Unlike WEEK_OF_MONTH, WEEK_OF_YEAR never shifts the day of the + // week. Also, rolling the week of the year can have seemingly + // strange effects simply because the year of the week of year + // may be different from the calendar year. For example, the + // date Dec 28, 1997 is the first day of week 1 of 1998 (if + // weeks start on Sunday and the minimal days in first week is + // <= 3). + int32_t woy = get(UCAL_WEEK_OF_YEAR, status); + // Get the ISO year, which matches the week of year. This + // may be one year before or after the calendar year. + int32_t isoYear = get(UCAL_YEAR_WOY, status); + int32_t isoDoy = internalGet(UCAL_DAY_OF_YEAR); + if (internalGetMonth() == UCAL_JANUARY) { + if (woy >= 52) { + isoDoy += handleGetYearLength(isoYear); + } + } else { + if (woy == 1) { + isoDoy -= handleGetYearLength(isoYear - 1); + } + } + woy += amount; + // Do fast checks to avoid unnecessary computation: + if (woy < 1 || woy > 52) { + // Determine the last week of the ISO year. + // We do this using the standard formula we use + // everywhere in this file. If we can see that the + // days at the end of the year are going to fall into + // week 1 of the next year, we drop the last week by + // subtracting 7 from the last day of the year. + int32_t lastDoy = handleGetYearLength(isoYear); + int32_t lastRelDow = (lastDoy - isoDoy + internalGet(UCAL_DAY_OF_WEEK) - + getFirstDayOfWeek()) % 7; + if (lastRelDow < 0) lastRelDow += 7; + if ((6 - lastRelDow) >= getMinimalDaysInFirstWeek()) lastDoy -= 7; + int32_t lastWoy = weekNumber(lastDoy, lastRelDow + 1); + woy = ((woy + lastWoy - 1) % lastWoy) + 1; + } + set(UCAL_WEEK_OF_YEAR, woy); + set(UCAL_YEAR_WOY,isoYear); + return; + } + + case UCAL_DAY_OF_MONTH: + if( !inCutoverMonth ) { + Calendar::roll(field, amount, status); + return; + } else { + // [j81] 1582 special case for DOM + // The default computation works except when the current month + // contains the Gregorian cutover. We handle this special case + // here. [j81 - aliu] + double monthLen = cMonthLen * kOneDay; + double msIntoMonth = uprv_fmod(internalGetTime() - cMonthStart + + amount * kOneDay, monthLen); + if (msIntoMonth < 0) { + msIntoMonth += monthLen; + } +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: roll DOM %d -> %.0lf ms \n", + __FILE__, __LINE__,amount, cMonthLen, cMonthStart+msIntoMonth); +#endif + setTimeInMillis(cMonthStart + msIntoMonth, status); + return; + } + + case UCAL_WEEK_OF_MONTH: + if( !inCutoverMonth ) { + Calendar::roll(field, amount, status); + return; + } else { +#if defined (U_DEBUG_CAL) + fprintf(stderr, "%s:%d: roll WOM %d ??????????????????? \n", + __FILE__, __LINE__,amount); +#endif + // NOTE: following copied from the old + // GregorianCalendar::roll( WEEK_OF_MONTH ) code + + // This is tricky, because during the roll we may have to shift + // to a different day of the week. For example: + + // s m t w r f s + // 1 2 3 4 5 + // 6 7 8 9 10 11 12 + + // When rolling from the 6th or 7th back one week, we go to the + // 1st (assuming that the first partial week counts). The same + // thing happens at the end of the month. + + // The other tricky thing is that we have to figure out whether + // the first partial week actually counts or not, based on the + // minimal first days in the week. And we have to use the + // correct first day of the week to delineate the week + // boundaries. + + // Here's our algorithm. First, we find the real boundaries of + // the month. Then we discard the first partial week if it + // doesn't count in this locale. Then we fill in the ends with + // phantom days, so that the first partial week and the last + // partial week are full weeks. We then have a nice square + // block of weeks. We do the usual rolling within this block, + // as is done elsewhere in this method. If we wind up on one of + // the phantom days that we added, we recognize this and pin to + // the first or the last day of the month. Easy, eh? + + // Another wrinkle: To fix jitterbug 81, we have to make all this + // work in the oddball month containing the Gregorian cutover. + // This month is 10 days shorter than usual, and also contains + // a discontinuity in the days; e.g., the default cutover month + // is Oct 1582, and goes from day of month 4 to day of month 15. + + // Normalize the DAY_OF_WEEK so that 0 is the first day of the week + // in this locale. We have dow in 0..6. + int32_t dow = internalGet(UCAL_DAY_OF_WEEK) - getFirstDayOfWeek(); + if (dow < 0) + dow += 7; + + // Find the day of month, compensating for cutover discontinuity. + int32_t dom = cDayOfMonth; + + // Find the day of the week (normalized for locale) for the first + // of the month. + int32_t fdm = (dow - dom + 1) % 7; + if (fdm < 0) + fdm += 7; + + // Get the first day of the first full week of the month, + // including phantom days, if any. Figure out if the first week + // counts or not; if it counts, then fill in phantom days. If + // not, advance to the first real full week (skip the partial week). + int32_t start; + if ((7 - fdm) < getMinimalDaysInFirstWeek()) + start = 8 - fdm; // Skip the first partial week + else + start = 1 - fdm; // This may be zero or negative + + // Get the day of the week (normalized for locale) for the last + // day of the month. + int32_t monthLen = cMonthLen; + int32_t ldm = (monthLen - dom + dow) % 7; + // We know monthLen >= DAY_OF_MONTH so we skip the += 7 step here. + + // Get the limit day for the blocked-off rectangular month; that + // is, the day which is one past the last day of the month, + // after the month has already been filled in with phantom days + // to fill out the last week. This day has a normalized DOW of 0. + int32_t limit = monthLen + 7 - ldm; + + // Now roll between start and (limit - 1). + int32_t gap = limit - start; + int32_t newDom = (dom + amount*7 - start) % gap; + if (newDom < 0) + newDom += gap; + newDom += start; + + // Finally, pin to the real start and end of the month. + if (newDom < 1) + newDom = 1; + if (newDom > monthLen) + newDom = monthLen; + + // Set the DAY_OF_MONTH. We rely on the fact that this field + // takes precedence over everything else (since all other fields + // are also set at this point). If this fact changes (if the + // disambiguation algorithm changes) then we will have to unset + // the appropriate fields here so that DAY_OF_MONTH is attended + // to. + + // If we are in the cutover month, manipulate ms directly. Don't do + // this in general because it doesn't work across DST boundaries + // (details, details). This takes care of the discontinuity. + setTimeInMillis(cMonthStart + (newDom-1)*kOneDay, status); + return; + } + + default: + Calendar::roll(field, amount, status); + return; + } +} + +// ------------------------------------- + + +/** +* Return the minimum value that this field could have, given the current date. +* For the Gregorian calendar, this is the same as getMinimum() and getGreatestMinimum(). +* @param field the time field. +* @return the minimum value that this field could have, given the current date. +* @deprecated ICU 2.6. Use getActualMinimum(UCalendarDateFields field) instead. +*/ +int32_t GregorianCalendar::getActualMinimum(EDateFields field) const +{ + return getMinimum((UCalendarDateFields)field); +} + +int32_t GregorianCalendar::getActualMinimum(EDateFields field, UErrorCode& /* status */) const +{ + return getMinimum((UCalendarDateFields)field); +} + +/** +* Return the minimum value that this field could have, given the current date. +* For the Gregorian calendar, this is the same as getMinimum() and getGreatestMinimum(). +* @param field the time field. +* @return the minimum value that this field could have, given the current date. +* @draft ICU 2.6. +*/ +int32_t GregorianCalendar::getActualMinimum(UCalendarDateFields field, UErrorCode& /* status */) const +{ + return getMinimum(field); +} + + +// ------------------------------------ + +/** +* Old year limits were least max 292269054, max 292278994. +*/ + +/** +* @stable ICU 2.0 +*/ +int32_t GregorianCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return kGregorianCalendarLimits[field][limitType]; +} + +/** +* Return the maximum value that this field could have, given the current date. +* For example, with the date "Feb 3, 1997" and the DAY_OF_MONTH field, the actual +* maximum would be 28; for "Feb 3, 1996" it s 29. Similarly for a Hebrew calendar, +* for some years the actual maximum for MONTH is 12, and for others 13. +* @stable ICU 2.0 +*/ +int32_t GregorianCalendar::getActualMaximum(UCalendarDateFields field, UErrorCode& status) const +{ + /* It is a known limitation that the code here (and in getActualMinimum) + * won't behave properly at the extreme limits of GregorianCalendar's + * representable range (except for the code that handles the YEAR + * field). That's because the ends of the representable range are at + * odd spots in the year. For calendars with the default Gregorian + * cutover, these limits are Sun Dec 02 16:47:04 GMT 292269055 BC to Sun + * Aug 17 07:12:55 GMT 292278994 AD, somewhat different for non-GMT + * zones. As a result, if the calendar is set to Aug 1 292278994 AD, + * the actual maximum of DAY_OF_MONTH is 17, not 30. If the date is Mar + * 31 in that year, the actual maximum month might be Jul, whereas is + * the date is Mar 15, the actual maximum might be Aug -- depending on + * the precise semantics that are desired. Similar considerations + * affect all fields. Nonetheless, this effect is sufficiently arcane + * that we permit it, rather than complicating the code to handle such + * intricacies. - liu 8/20/98 + + * UPDATE: No longer true, since we have pulled in the limit values on + * the year. - Liu 11/6/00 */ + + switch (field) { + + case UCAL_YEAR: + /* The year computation is no different, in principle, from the + * others, however, the range of possible maxima is large. In + * addition, the way we know we've exceeded the range is different. + * For these reasons, we use the special case code below to handle + * this field. + * + * The actual maxima for YEAR depend on the type of calendar: + * + * Gregorian = May 17, 292275056 BC - Aug 17, 292278994 AD + * Julian = Dec 2, 292269055 BC - Jan 3, 292272993 AD + * Hybrid = Dec 2, 292269055 BC - Aug 17, 292278994 AD + * + * We know we've exceeded the maximum when either the month, date, + * time, or era changes in response to setting the year. We don't + * check for month, date, and time here because the year and era are + * sufficient to detect an invalid year setting. NOTE: If code is + * added to check the month and date in the future for some reason, + * Feb 29 must be allowed to shift to Mar 1 when setting the year. + */ + { + if(U_FAILURE(status)) return 0; + Calendar *cal = clone(); + if(!cal) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + cal->setLenient(true); + + int32_t era = cal->get(UCAL_ERA, status); + UDate d = cal->getTime(status); + + /* Perform a binary search, with the invariant that lowGood is a + * valid year, and highBad is an out of range year. + */ + int32_t lowGood = kGregorianCalendarLimits[UCAL_YEAR][1]; + int32_t highBad = kGregorianCalendarLimits[UCAL_YEAR][2]+1; + while ((lowGood + 1) < highBad) { + int32_t y = (lowGood + highBad) / 2; + cal->set(UCAL_YEAR, y); + if (cal->get(UCAL_YEAR, status) == y && cal->get(UCAL_ERA, status) == era) { + lowGood = y; + } else { + highBad = y; + cal->setTime(d, status); // Restore original fields + } + } + + delete cal; + return lowGood; + } + + default: + return Calendar::getActualMaximum(field,status); + } +} + + +int32_t GregorianCalendar::handleGetExtendedYear() { + // the year to return + int32_t year = kEpochYear; + + // year field to use + int32_t yearField = UCAL_EXTENDED_YEAR; + + // There are three separate fields which could be used to + // derive the proper year. Use the one most recently set. + if (fStamp[yearField] < fStamp[UCAL_YEAR]) + yearField = UCAL_YEAR; + if (fStamp[yearField] < fStamp[UCAL_YEAR_WOY]) + yearField = UCAL_YEAR_WOY; + + // based on the "best" year field, get the year + switch(yearField) { + case UCAL_EXTENDED_YEAR: + year = internalGet(UCAL_EXTENDED_YEAR, kEpochYear); + break; + + case UCAL_YEAR: + { + // The year defaults to the epoch start, the era to AD + int32_t era = internalGet(UCAL_ERA, AD); + if (era == BC) { + year = 1 - internalGet(UCAL_YEAR, 1); // Convert to extended year + } else { + year = internalGet(UCAL_YEAR, kEpochYear); + } + } + break; + + case UCAL_YEAR_WOY: + year = handleGetExtendedYearFromWeekFields(internalGet(UCAL_YEAR_WOY), internalGet(UCAL_WEEK_OF_YEAR)); +#if defined (U_DEBUG_CAL) + // if(internalGet(UCAL_YEAR_WOY) != year) { + fprintf(stderr, "%s:%d: hGEYFWF[%d,%d] -> %d\n", + __FILE__, __LINE__,internalGet(UCAL_YEAR_WOY),internalGet(UCAL_WEEK_OF_YEAR),year); + //} +#endif + break; + + default: + year = kEpochYear; + } + return year; +} + +int32_t GregorianCalendar::handleGetExtendedYearFromWeekFields(int32_t yearWoy, int32_t woy) +{ + // convert year to extended form + int32_t era = internalGet(UCAL_ERA, AD); + if(era == BC) { + yearWoy = 1 - yearWoy; + } + return Calendar::handleGetExtendedYearFromWeekFields(yearWoy, woy); +} + + +// ------------------------------------- + +/** +* Return the ERA. We need a special method for this because the +* default ERA is AD, but a zero (unset) ERA is BC. +*/ +int32_t +GregorianCalendar::internalGetEra() const { + return isSet(UCAL_ERA) ? internalGet(UCAL_ERA) : (int32_t)AD; +} + +const char * +GregorianCalendar::getType() const { + //static const char kGregorianType = "gregorian"; + + return "gregorian"; +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + + +UBool GregorianCalendar::haveDefaultCentury() const +{ + return true; +} + +static void U_CALLCONV +initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + GregorianCalendar calendar(status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate GregorianCalendar::defaultCenturyStart() const { + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t GregorianCalendar::defaultCenturyStartYear() const { + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/gregoimp.cpp b/intl/icu/source/i18n/gregoimp.cpp new file mode 100644 index 0000000000..31b5aeed83 --- /dev/null +++ b/intl/icu/source/i18n/gregoimp.cpp @@ -0,0 +1,176 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (c) 2003-2008, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Author: Alan Liu + * Created: September 2 2003 + * Since: ICU 2.8 + ********************************************************************** + */ + +#include "gregoimp.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/ucal.h" +#include "uresimp.h" +#include "cstring.h" +#include "uassert.h" + +U_NAMESPACE_BEGIN + +int32_t ClockMath::floorDivide(int32_t numerator, int32_t denominator) { + return (numerator >= 0) ? + numerator / denominator : ((numerator + 1) / denominator) - 1; +} + +int64_t ClockMath::floorDivide(int64_t numerator, int64_t denominator) { + return (numerator >= 0) ? + numerator / denominator : ((numerator + 1) / denominator) - 1; +} + +int32_t ClockMath::floorDivide(double numerator, int32_t denominator, + int32_t* remainder) { + // For an integer n and representable ⌊x/n⌋, ⌊RN(x/n)⌋=⌊x/n⌋, where RN is + // rounding to nearest. + double quotient = uprv_floor(numerator / denominator); + if (remainder != nullptr) { + // For doubles x and n, where n is an integer and ⌊x+n⌋ < 2³¹, the + // expression `(int32_t) (x + n)` evaluated with rounding to nearest + // differs from ⌊x+n⌋ if 0 < ⌈x⌉−x ≪ x+n, as `x + n` is rounded up to + // n+⌈x⌉ = ⌊x+n⌋ + 1. Rewriting it as ⌊x⌋+n makes the addition exact. + *remainder = (int32_t) (uprv_floor(numerator) - (quotient * denominator)); + } + return (int32_t) quotient; +} + +double ClockMath::floorDivide(double dividend, double divisor, + double* remainder) { + // Only designed to work for positive divisors + U_ASSERT(divisor > 0); + double quotient = floorDivide(dividend, divisor); + double r = dividend - (quotient * divisor); + // N.B. For certain large dividends, on certain platforms, there + // is a bug such that the quotient is off by one. If you doubt + // this to be true, set a breakpoint below and run cintltst. + if (r < 0 || r >= divisor) { + // E.g. 6.7317038241449352e+022 / 86400000.0 is wrong on my + // machine (too high by one). 4.1792057231752762e+024 / + // 86400000.0 is wrong the other way (too low). + double q = quotient; + quotient += (r < 0) ? -1 : +1; + if (q == quotient) { + // For quotients > ~2^53, we won't be able to add or + // subtract one, since the LSB of the mantissa will be > + // 2^0; that is, the exponent (base 2) will be larger than + // the length, in bits, of the mantissa. In that case, we + // can't give a correct answer, so we set the remainder to + // zero. This has the desired effect of making extreme + // values give back an approximate answer rather than + // crashing. For example, UDate values above a ~10^25 + // might all have a time of midnight. + r = 0; + } else { + r = dividend - (quotient * divisor); + } + } + U_ASSERT(0 <= r && r < divisor); + if (remainder != nullptr) { + *remainder = r; + } + return quotient; +} + +const int32_t JULIAN_1_CE = 1721426; // January 1, 1 CE Gregorian +const int32_t JULIAN_1970_CE = 2440588; // January 1, 1970 CE Gregorian + +const int16_t Grego::DAYS_BEFORE[24] = + {0,31,59,90,120,151,181,212,243,273,304,334, + 0,31,60,91,121,152,182,213,244,274,305,335}; + +const int8_t Grego::MONTH_LENGTH[24] = + {31,28,31,30,31,30,31,31,30,31,30,31, + 31,29,31,30,31,30,31,31,30,31,30,31}; + +double Grego::fieldsToDay(int32_t year, int32_t month, int32_t dom) { + + int32_t y = year - 1; + + double julian = 365 * y + ClockMath::floorDivide(y, 4) + (JULIAN_1_CE - 3) + // Julian cal + ClockMath::floorDivide(y, 400) - ClockMath::floorDivide(y, 100) + 2 + // => Gregorian cal + DAYS_BEFORE[month + (isLeapYear(year) ? 12 : 0)] + dom; // => month/dom + + return julian - JULIAN_1970_CE; // JD => epoch day +} + +void Grego::dayToFields(double day, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow, int32_t& doy) { + + // Convert from 1970 CE epoch to 1 CE epoch (Gregorian calendar) + day += JULIAN_1970_CE - JULIAN_1_CE; + + // Convert from the day number to the multiple radix + // representation. We use 400-year, 100-year, and 4-year cycles. + // For example, the 4-year cycle has 4 years + 1 leap day; giving + // 1461 == 365*4 + 1 days. + int32_t n400 = ClockMath::floorDivide(day, 146097, &doy); // 400-year cycle length + int32_t n100 = ClockMath::floorDivide(doy, 36524, &doy); // 100-year cycle length + int32_t n4 = ClockMath::floorDivide(doy, 1461, &doy); // 4-year cycle length + int32_t n1 = ClockMath::floorDivide(doy, 365, &doy); + year = 400*n400 + 100*n100 + 4*n4 + n1; + if (n100 == 4 || n1 == 4) { + doy = 365; // Dec 31 at end of 4- or 400-year cycle + } else { + ++year; + } + + UBool isLeap = isLeapYear(year); + + // Gregorian day zero is a Monday. + dow = (int32_t) uprv_fmod(day + 1, 7); + dow += (dow < 0) ? (UCAL_SUNDAY + 7) : UCAL_SUNDAY; + + // Common Julian/Gregorian calculation + int32_t correction = 0; + int32_t march1 = isLeap ? 60 : 59; // zero-based DOY for March 1 + if (doy >= march1) { + correction = isLeap ? 1 : 2; + } + month = (12 * (doy + correction) + 6) / 367; // zero-based month + dom = doy - DAYS_BEFORE[month + (isLeap ? 12 : 0)] + 1; // one-based DOM + doy++; // one-based doy +} + +void Grego::timeToFields(UDate time, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow, int32_t& doy, int32_t& mid) { + double millisInDay; + double day = ClockMath::floorDivide((double)time, (double)U_MILLIS_PER_DAY, &millisInDay); + mid = (int32_t)millisInDay; + dayToFields(day, year, month, dom, dow, doy); +} + +int32_t Grego::dayOfWeek(double day) { + int32_t dow; + ClockMath::floorDivide(day + int{UCAL_THURSDAY}, 7, &dow); + return (dow == 0) ? UCAL_SATURDAY : dow; +} + +int32_t Grego::dayOfWeekInMonth(int32_t year, int32_t month, int32_t dom) { + int32_t weekInMonth = (dom + 6)/7; + if (weekInMonth == 4) { + if (dom + 7 > monthLength(year, month)) { + weekInMonth = -1; + } + } else if (weekInMonth == 5) { + weekInMonth = -1; + } + return weekInMonth; +} + +U_NAMESPACE_END + +#endif +//eof diff --git a/intl/icu/source/i18n/gregoimp.h b/intl/icu/source/i18n/gregoimp.h new file mode 100644 index 0000000000..d65d6a4f88 --- /dev/null +++ b/intl/icu/source/i18n/gregoimp.h @@ -0,0 +1,312 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (c) 2003-2008, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Author: Alan Liu + * Created: September 2 2003 + * Since: ICU 2.8 + ********************************************************************** + */ + +#ifndef GREGOIMP_H +#define GREGOIMP_H +#include "unicode/utypes.h" +#if !UCONFIG_NO_FORMATTING + +#include "unicode/ures.h" +#include "unicode/locid.h" +#include "putilimp.h" + +U_NAMESPACE_BEGIN + +/** + * A utility class providing mathematical functions used by time zone + * and calendar code. Do not instantiate. Formerly just named 'Math'. + * @internal + */ +class ClockMath { + public: + /** + * Divide two integers, returning the floor of the quotient. + * Unlike the built-in division, this is mathematically + * well-behaved. E.g., -1/4 => 0 but + * floorDivide(-1,4) => -1. + * @param numerator the numerator + * @param denominator a divisor which must be != 0 + * @return the floor of the quotient + */ + static int32_t floorDivide(int32_t numerator, int32_t denominator); + + /** + * Divide two integers, returning the floor of the quotient. + * Unlike the built-in division, this is mathematically + * well-behaved. E.g., -1/4 => 0 but + * floorDivide(-1,4) => -1. + * @param numerator the numerator + * @param denominator a divisor which must be != 0 + * @return the floor of the quotient + */ + static int64_t floorDivide(int64_t numerator, int64_t denominator); + + /** + * Divide two numbers, returning the floor of the quotient. + * Unlike the built-in division, this is mathematically + * well-behaved. E.g., -1/4 => 0 but + * floorDivide(-1,4) => -1. + * @param numerator the numerator + * @param denominator a divisor which must be != 0 + * @return the floor of the quotient + */ + static inline double floorDivide(double numerator, double denominator); + + /** + * Divide two numbers, returning the floor of the quotient and + * the modulus remainder. Unlike the built-in division, this is + * mathematically well-behaved. E.g., -1/4 => 0 and + * -1%4 => -1, but floorDivide(-1,4) => + * -1 with remainder => 3. NOTE: If numerator is + * too large, the returned quotient may overflow. + * @param numerator the numerator + * @param denominator a divisor which must be != 0 + * @param remainder output parameter to receive the + * remainder. Unlike numerator % denominator, this + * will always be non-negative, in the half-open range [0, + * |denominator|). + * @return the floor of the quotient + */ + static int32_t floorDivide(double numerator, int32_t denominator, + int32_t* remainder); + + /** + * For a positive divisor, return the quotient and remainder + * such that dividend = quotient*divisor + remainder and + * 0 <= remainder < divisor. + * + * Works around edge-case bugs. Handles pathological input + * (dividend >> divisor) reasonably. + * + * Calling with a divisor <= 0 is disallowed. + */ + static double floorDivide(double dividend, double divisor, + double* remainder); +}; + +// Useful millisecond constants +#define kOneDay (1.0 * U_MILLIS_PER_DAY) // 86,400,000 +#define kOneHour (60*60*1000) +#define kOneMinute 60000 +#define kOneSecond 1000 +#define kOneMillisecond 1 +#define kOneWeek (7.0 * kOneDay) // 604,800,000 + +// Epoch constants +#define kJan1_1JulianDay 1721426 // January 1, year 1 (Gregorian) + +#define kEpochStartAsJulianDay 2440588 // January 1, 1970 (Gregorian) + +#define kEpochYear 1970 + + +#define kEarliestViableMillis -185331720384000000.0 // minimum representable by julian day -1e17 + +#define kLatestViableMillis 185753453990400000.0 // max representable by julian day +1e17 + +/** + * The minimum supported Julian day. This value is equivalent to + * MIN_MILLIS. + */ +#define MIN_JULIAN (-0x7F000000) + +/** + * The minimum supported epoch milliseconds. This value is equivalent + * to MIN_JULIAN. + */ +#define MIN_MILLIS ((MIN_JULIAN - kEpochStartAsJulianDay) * kOneDay) + +/** + * The maximum supported Julian day. This value is equivalent to + * MAX_MILLIS. + */ +#define MAX_JULIAN (+0x7F000000) + +/** + * The maximum supported epoch milliseconds. This value is equivalent + * to MAX_JULIAN. + */ +#define MAX_MILLIS ((MAX_JULIAN - kEpochStartAsJulianDay) * kOneDay) + +/** + * A utility class providing proleptic Gregorian calendar functions + * used by time zone and calendar code. Do not instantiate. + * + * Note: Unlike GregorianCalendar, all computations performed by this + * class occur in the pure proleptic GregorianCalendar. + */ +class Grego { + public: + /** + * Return true if the given year is a leap year. + * @param year Gregorian year, with 0 == 1 BCE, -1 == 2 BCE, etc. + * @return true if the year is a leap year + */ + static inline UBool isLeapYear(int32_t year); + + /** + * Return the number of days in the given month. + * @param year Gregorian year, with 0 == 1 BCE, -1 == 2 BCE, etc. + * @param month 0-based month, with 0==Jan + * @return the number of days in the given month + */ + static inline int8_t monthLength(int32_t year, int32_t month); + + /** + * Return the length of a previous month of the Gregorian calendar. + * @param y the extended year + * @param m the 0-based month number + * @return the number of days in the month previous to the given month + */ + static inline int8_t previousMonthLength(int y, int m); + + /** + * Convert a year, month, and day-of-month, given in the proleptic + * Gregorian calendar, to 1970 epoch days. + * @param year Gregorian year, with 0 == 1 BCE, -1 == 2 BCE, etc. + * @param month 0-based month, with 0==Jan + * @param dom 1-based day of month + * @return the day number, with day 0 == Jan 1 1970 + */ + static double fieldsToDay(int32_t year, int32_t month, int32_t dom); + + /** + * Convert a 1970-epoch day number to proleptic Gregorian year, + * month, day-of-month, and day-of-week. + * @param day 1970-epoch day (integral value) + * @param year output parameter to receive year + * @param month output parameter to receive month (0-based, 0==Jan) + * @param dom output parameter to receive day-of-month (1-based) + * @param dow output parameter to receive day-of-week (1-based, 1==Sun) + * @param doy output parameter to receive day-of-year (1-based) + */ + static void dayToFields(double day, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow, int32_t& doy); + + /** + * Convert a 1970-epoch day number to proleptic Gregorian year, + * month, day-of-month, and day-of-week. + * @param day 1970-epoch day (integral value) + * @param year output parameter to receive year + * @param month output parameter to receive month (0-based, 0==Jan) + * @param dom output parameter to receive day-of-month (1-based) + * @param dow output parameter to receive day-of-week (1-based, 1==Sun) + */ + static inline void dayToFields(double day, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow); + + /** + * Convert a 1970-epoch milliseconds to proleptic Gregorian year, + * month, day-of-month, and day-of-week, day of year and millis-in-day. + * @param time 1970-epoch milliseconds + * @param year output parameter to receive year + * @param month output parameter to receive month (0-based, 0==Jan) + * @param dom output parameter to receive day-of-month (1-based) + * @param dow output parameter to receive day-of-week (1-based, 1==Sun) + * @param doy output parameter to receive day-of-year (1-based) + * @param mid output parameter to receive millis-in-day + */ + static void timeToFields(UDate time, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow, int32_t& doy, int32_t& mid); + + /** + * Return the day of week on the 1970-epoch day + * @param day the 1970-epoch day (integral value) + * @return the day of week + */ + static int32_t dayOfWeek(double day); + + /** + * Returns the ordinal number for the specified day of week within the month. + * The valid return value is 1, 2, 3, 4 or -1. + * @param year Gregorian year, with 0 == 1 BCE, -1 == 2 BCE, etc. + * @param month 0-based month, with 0==Jan + * @param dom 1-based day of month + * @return The ordinal number for the specified day of week within the month + */ + static int32_t dayOfWeekInMonth(int32_t year, int32_t month, int32_t dom); + + /** + * Converts Julian day to time as milliseconds. + * @param julian the given Julian day number. + * @return time as milliseconds. + * @internal + */ + static inline double julianDayToMillis(int32_t julian); + + /** + * Converts time as milliseconds to Julian day. + * @param millis the given milliseconds. + * @return the Julian day number. + * @internal + */ + static inline int32_t millisToJulianDay(double millis); + + /** + * Calculates the Gregorian day shift value for an extended year. + * @param eyear Extended year + * @returns number of days to ADD to Julian in order to convert from J->G + */ + static inline int32_t gregorianShift(int32_t eyear); + + private: + static const int16_t DAYS_BEFORE[24]; + static const int8_t MONTH_LENGTH[24]; +}; + +inline double ClockMath::floorDivide(double numerator, double denominator) { + return uprv_floor(numerator / denominator); +} + +inline UBool Grego::isLeapYear(int32_t year) { + // year&0x3 == year%4 + return ((year&0x3) == 0) && ((year%100 != 0) || (year%400 == 0)); +} + +inline int8_t +Grego::monthLength(int32_t year, int32_t month) { + return MONTH_LENGTH[month + (isLeapYear(year) ? 12 : 0)]; +} + +inline int8_t +Grego::previousMonthLength(int y, int m) { + return (m > 0) ? monthLength(y, m-1) : 31; +} + +inline void Grego::dayToFields(double day, int32_t& year, int32_t& month, + int32_t& dom, int32_t& dow) { + int32_t doy_unused; + dayToFields(day,year,month,dom,dow,doy_unused); +} + +inline double Grego::julianDayToMillis(int32_t julian) +{ + return (julian - kEpochStartAsJulianDay) * kOneDay; +} + +inline int32_t Grego::millisToJulianDay(double millis) { + return (int32_t) (kEpochStartAsJulianDay + ClockMath::floorDivide(millis, (double)kOneDay)); +} + +inline int32_t Grego::gregorianShift(int32_t eyear) { + int64_t y = (int64_t)eyear-1; + int32_t gregShift = static_cast(ClockMath::floorDivide(y, (int64_t)400) - ClockMath::floorDivide(y, (int64_t)100) + 2); + return gregShift; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING +#endif // GREGOIMP_H + +//eof diff --git a/intl/icu/source/i18n/hebrwcal.cpp b/intl/icu/source/i18n/hebrwcal.cpp new file mode 100644 index 0000000000..efd0d8fd4a --- /dev/null +++ b/intl/icu/source/i18n/hebrwcal.cpp @@ -0,0 +1,790 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2003-2016, International Business Machines Corporation +* and others. All Rights Reserved. +****************************************************************************** +* +* File HEBRWCAL.CPP +* +* Modification History: +* +* Date Name Description +* 12/03/2003 srl ported from java HebrewCalendar +***************************************************************************** +*/ + +#include "hebrwcal.h" + +#if !UCONFIG_NO_FORMATTING + +#include "cmemory.h" +#include "cstring.h" +#include "umutex.h" +#include +#include "gregoimp.h" // ClockMath +#include "astro.h" // CalendarCache +#include "uhash.h" +#include "ucln_in.h" + +// Hebrew Calendar implementation + +/** +* The absolute date, in milliseconds since 1/1/1970 AD, Gregorian, +* of the start of the Hebrew calendar. In order to keep this calendar's +* time of day in sync with that of the Gregorian calendar, we use +* midnight, rather than sunset the day before. +*/ +//static const double EPOCH_MILLIS = -180799862400000.; // 1/1/1 HY + +static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 0, 0}, // ERA + { -5000000, -5000000, 5000000, 5000000}, // YEAR + { 0, 0, 12, 12}, // MONTH + { 1, 1, 51, 56}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 29, 30}, // DAY_OF_MONTH + { 1, 1, 353, 385}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 5, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 11, 12}, // ORDINAL_MONTH +}; + +/** +* The lengths of the Hebrew months. This is complicated, because there +* are three different types of years, or six if you count leap years. +* Due to the rules for postponing the start of the year to avoid having +* certain holidays fall on the sabbath, the year can end up being three +* different lengths, called "deficient", "normal", and "complete". +*/ +static const int8_t MONTH_LENGTH[][3] = { + // Deficient Normal Complete + { 30, 30, 30 }, //Tishri + { 29, 29, 30 }, //Heshvan + { 29, 30, 30 }, //Kislev + { 29, 29, 29 }, //Tevet + { 30, 30, 30 }, //Shevat + { 30, 30, 30 }, //Adar I (leap years only) + { 29, 29, 29 }, //Adar + { 30, 30, 30 }, //Nisan + { 29, 29, 29 }, //Iyar + { 30, 30, 30 }, //Sivan + { 29, 29, 29 }, //Tammuz + { 30, 30, 30 }, //Av + { 29, 29, 29 }, //Elul +}; + +/** +* The cumulative # of days to the end of each month in a non-leap year +* Although this can be calculated from the MONTH_LENGTH table, +* keeping it around separately makes some calculations a lot faster +*/ + +static const int16_t MONTH_START[][3] = { + // Deficient Normal Complete + { 0, 0, 0 }, // (placeholder) + { 30, 30, 30 }, // Tishri + { 59, 59, 60 }, // Heshvan + { 88, 89, 90 }, // Kislev + { 117, 118, 119 }, // Tevet + { 147, 148, 149 }, // Shevat + { 147, 148, 149 }, // (Adar I placeholder) + { 176, 177, 178 }, // Adar + { 206, 207, 208 }, // Nisan + { 235, 236, 237 }, // Iyar + { 265, 266, 267 }, // Sivan + { 294, 295, 296 }, // Tammuz + { 324, 325, 326 }, // Av + { 353, 354, 355 }, // Elul +}; + +/** +* The cumulative # of days to the end of each month in a leap year +*/ +static const int16_t LEAP_MONTH_START[][3] = { + // Deficient Normal Complete + { 0, 0, 0 }, // (placeholder) + { 30, 30, 30 }, // Tishri + { 59, 59, 60 }, // Heshvan + { 88, 89, 90 }, // Kislev + { 117, 118, 119 }, // Tevet + { 147, 148, 149 }, // Shevat + { 177, 178, 179 }, // Adar I + { 206, 207, 208 }, // Adar II + { 236, 237, 238 }, // Nisan + { 265, 266, 267 }, // Iyar + { 295, 296, 297 }, // Sivan + { 324, 325, 326 }, // Tammuz + { 354, 355, 356 }, // Av + { 383, 384, 385 }, // Elul +}; + +static icu::CalendarCache *gCache = nullptr; + +U_CDECL_BEGIN +static UBool calendar_hebrew_cleanup() { + delete gCache; + gCache = nullptr; + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +/** +* Constructs a default HebrewCalendar using the current time +* in the default time zone with the default locale. +* @internal +*/ +HebrewCalendar::HebrewCalendar(const Locale& aLocale, UErrorCode& success) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success) + +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + + +HebrewCalendar::~HebrewCalendar() { +} + +const char *HebrewCalendar::getType() const { + return "hebrew"; +} + +HebrewCalendar* HebrewCalendar::clone() const { + return new HebrewCalendar(*this); +} + +HebrewCalendar::HebrewCalendar(const HebrewCalendar& other) : Calendar(other) { +} + + +//------------------------------------------------------------------------- +// Rolling and adding functions overridden from Calendar +// +// These methods call through to the default implementation in IBMCalendar +// for most of the fields and only handle the unusual ones themselves. +//------------------------------------------------------------------------- + +/** +* Add a signed amount to a specified field, using this calendar's rules. +* For example, to add three days to the current date, you can call +* add(Calendar.DATE, 3). +*

+* When adding to certain fields, the values of other fields may conflict and +* need to be changed. For example, when adding one to the {@link #MONTH MONTH} field +* for the date "30 Av 5758", the {@link #DAY_OF_MONTH DAY_OF_MONTH} field +* must be adjusted so that the result is "29 Elul 5758" rather than the invalid +* "30 Elul 5758". +*

+* This method is able to add to +* all fields except for {@link #ERA ERA}, {@link #DST_OFFSET DST_OFFSET}, +* and {@link #ZONE_OFFSET ZONE_OFFSET}. +*

+* Note: You should always use {@link #roll roll} and add rather +* than attempting to perform arithmetic operations directly on the fields +* of a HebrewCalendar. Since the {@link #MONTH MONTH} field behaves +* discontinuously in non-leap years, simple arithmetic can give invalid results. +*

+* @param field the time field. +* @param amount the amount to add to the field. +* +* @exception IllegalArgumentException if the field is invalid or refers +* to a field that cannot be handled by this method. +* @internal +*/ +void HebrewCalendar::add(UCalendarDateFields field, int32_t amount, UErrorCode& status) +{ + if(U_FAILURE(status)) { + return; + } + switch (field) { + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + { + // We can't just do a set(MONTH, get(MONTH) + amount). The + // reason is ADAR_1. Suppose amount is +2 and we land in + // ADAR_1 -- then we have to bump to ADAR_2 aka ADAR. But + // if amount is -2 and we land in ADAR_1, then we have to + // bump the other way -- down to SHEVAT. - Alan 11/00 + int32_t month = get(UCAL_MONTH, status); + int32_t year = get(UCAL_YEAR, status); + UBool acrossAdar1; + if (amount > 0) { + acrossAdar1 = (month < ADAR_1); // started before ADAR_1? + month += amount; + for (;;) { + if (acrossAdar1 && month>=ADAR_1 && !isLeapYear(year)) { + ++month; + } + if (month <= ELUL) { + break; + } + month -= ELUL+1; + ++year; + acrossAdar1 = true; + } + } else { + acrossAdar1 = (month > ADAR_1); // started after ADAR_1? + month += amount; + for (;;) { + if (acrossAdar1 && month<=ADAR_1 && !isLeapYear(year)) { + --month; + } + if (month >= 0) { + break; + } + month += ELUL+1; + --year; + acrossAdar1 = true; + } + } + set(UCAL_MONTH, month); + set(UCAL_YEAR, year); + pinField(UCAL_DAY_OF_MONTH, status); + break; + } + + default: + Calendar::add(field, amount, status); + break; + } +} + +/** +* @deprecated ICU 2.6 use UCalendarDateFields instead of EDateFields +*/ +void HebrewCalendar::add(EDateFields field, int32_t amount, UErrorCode& status) +{ + add((UCalendarDateFields)field, amount, status); +} + +/** +* Rolls (up/down) a specified amount time on the given field. For +* example, to roll the current date up by three days, you can call +* roll(Calendar.DATE, 3). If the +* field is rolled past its maximum allowable value, it will "wrap" back +* to its minimum and continue rolling. +* For example, calling roll(Calendar.DATE, 10) +* on a Hebrew calendar set to "25 Av 5758" will result in the date "5 Av 5758". +*

+* When rolling certain fields, the values of other fields may conflict and +* need to be changed. For example, when rolling the {@link #MONTH MONTH} field +* upward by one for the date "30 Av 5758", the {@link #DAY_OF_MONTH DAY_OF_MONTH} field +* must be adjusted so that the result is "29 Elul 5758" rather than the invalid +* "30 Elul". +*

+* This method is able to roll +* all fields except for {@link #ERA ERA}, {@link #DST_OFFSET DST_OFFSET}, +* and {@link #ZONE_OFFSET ZONE_OFFSET}. Subclasses may, of course, add support for +* additional fields in their overrides of roll. +*

+* Note: You should always use roll and {@link #add add} rather +* than attempting to perform arithmetic operations directly on the fields +* of a HebrewCalendar. Since the {@link #MONTH MONTH} field behaves +* discontinuously in non-leap years, simple arithmetic can give invalid results. +*

+* @param field the time field. +* @param amount the amount by which the field should be rolled. +* +* @exception IllegalArgumentException if the field is invalid or refers +* to a field that cannot be handled by this method. +* @internal +*/ +void HebrewCalendar::roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) +{ + if(U_FAILURE(status)) { + return; + } + switch (field) { + case UCAL_MONTH: + case UCAL_ORDINAL_MONTH: + { + int32_t month = get(UCAL_MONTH, status); + int32_t year = get(UCAL_YEAR, status); + + UBool leapYear = isLeapYear(year); + int32_t yearLength = monthsInYear(year); + int32_t newMonth = month + (amount % yearLength); + // + // If it's not a leap year and we're rolling past the missing month + // of ADAR_1, we need to roll an extra month to make up for it. + // + if (!leapYear) { + if (amount > 0 && month < ADAR_1 && newMonth >= ADAR_1) { + newMonth++; + } else if (amount < 0 && month > ADAR_1 && newMonth <= ADAR_1) { + newMonth--; + } + } + set(UCAL_MONTH, (newMonth + 13) % 13); + pinField(UCAL_DAY_OF_MONTH, status); + return; + } + default: + Calendar::roll(field, amount, status); + } +} + +void HebrewCalendar::roll(EDateFields field, int32_t amount, UErrorCode& status) { + roll((UCalendarDateFields)field, amount, status); +} + +//------------------------------------------------------------------------- +// Support methods +//------------------------------------------------------------------------- + +// Hebrew date calculations are performed in terms of days, hours, and +// "parts" (or halakim), which are 1/1080 of an hour, or 3 1/3 seconds. +static const int32_t HOUR_PARTS = 1080; +static const int32_t DAY_PARTS = 24*HOUR_PARTS; + +// An approximate value for the length of a lunar month. +// It is used to calculate the approximate year and month of a given +// absolute date. +static const int32_t MONTH_DAYS = 29; +static const int32_t MONTH_FRACT = 12*HOUR_PARTS + 793; +static const int32_t MONTH_PARTS = MONTH_DAYS*DAY_PARTS + MONTH_FRACT; + +// The time of the new moon (in parts) on 1 Tishri, year 1 (the epoch) +// counting from noon on the day before. BAHARAD is an abbreviation of +// Bet (Monday), Hey (5 hours from sunset), Resh-Daled (204). +static const int32_t BAHARAD = 11*HOUR_PARTS + 204; + +/** +* Finds the day # of the first day in the given Hebrew year. +* To do this, we want to calculate the time of the Tishri 1 new moon +* in that year. +*

+* The algorithm here is similar to ones described in a number of +* references, including: +*

+*/ +int32_t HebrewCalendar::startOfYear(int32_t year, UErrorCode &status) +{ + ucln_i18n_registerCleanup(UCLN_I18N_HEBREW_CALENDAR, calendar_hebrew_cleanup); + int32_t day = CalendarCache::get(&gCache, year, status); + + if (day == 0) { + // # of months before year + int32_t months = (int32_t)ClockMath::floorDivide((235 * (int64_t)year - 234), (int64_t)19); + + int64_t frac = (int64_t)months * MONTH_FRACT + BAHARAD; // Fractional part of day # + day = months * 29 + (int32_t)(frac / DAY_PARTS); // Whole # part of calculation + frac = frac % DAY_PARTS; // Time of day + + int32_t wd = (day % 7); // Day of week (0 == Monday) + + if (wd == 2 || wd == 4 || wd == 6) { + // If the 1st is on Sun, Wed, or Fri, postpone to the next day + day += 1; + wd = (day % 7); + } + if (wd == 1 && frac > 15*HOUR_PARTS+204 && !isLeapYear(year) ) { + // If the new moon falls after 3:11:20am (15h204p from the previous noon) + // on a Tuesday and it is not a leap year, postpone by 2 days. + // This prevents 356-day years. + day += 2; + } + else if (wd == 0 && frac > 21*HOUR_PARTS+589 && isLeapYear(year-1) ) { + // If the new moon falls after 9:32:43 1/3am (21h589p from yesterday noon) + // on a Monday and *last* year was a leap year, postpone by 1 day. + // Prevents 382-day years. + day += 1; + } + CalendarCache::put(&gCache, year, day, status); + } + return day; +} + +/** +* Find the day of the week for a given day +* +* @param day The # of days since the start of the Hebrew calendar, +* 1-based (i.e. 1/1/1 AM is day 1). +*/ +int32_t HebrewCalendar::absoluteDayToDayOfWeek(int32_t day) +{ + // We know that 1/1/1 AM is a Monday, which makes the math easy... + return (day % 7) + 1; +} + +/** +* Returns the the type of a given year. +* 0 "Deficient" year with 353 or 383 days +* 1 "Normal" year with 354 or 384 days +* 2 "Complete" year with 355 or 385 days +*/ +int32_t HebrewCalendar::yearType(int32_t year) const +{ + int32_t yearLength = handleGetYearLength(year); + + if (yearLength > 380) { + yearLength -= 30; // Subtract length of leap month. + } + + int type = 0; + + switch (yearLength) { + case 353: + type = 0; break; + case 354: + type = 1; break; + case 355: + type = 2; break; + default: + //throw new RuntimeException("Illegal year length " + yearLength + " in year " + year); + type = 1; + } + return type; +} + +/** +* Determine whether a given Hebrew year is a leap year +* +* The rule here is that if (year % 19) == 0, 3, 6, 8, 11, 14, or 17. +* The formula below performs the same test, believe it or not. +*/ +UBool HebrewCalendar::isLeapYear(int32_t year) { + //return (year * 12 + 17) % 19 >= 12; + int32_t x = (year*12 + 17) % 19; + return x >= ((x < 0) ? -7 : 12); +} + +int32_t HebrewCalendar::monthsInYear(int32_t year) { + return isLeapYear(year) ? 13 : 12; +} + +//------------------------------------------------------------------------- +// Calendar framework +//------------------------------------------------------------------------- + +/** +* @internal +*/ +int32_t HebrewCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return LIMITS[field][limitType]; +} + +/** +* Returns the length of the given month in the given year +* @internal +*/ +int32_t HebrewCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + // Resolve out-of-range months. This is necessary in order to + // obtain the correct year. We correct to + // a 12- or 13-month year (add/subtract 12 or 13, depending + // on the year) but since we _always_ number from 0..12, and + // the leap year determines whether or not month 5 (Adar 1) + // is present, we allow 0..12 in any given year. + while (month < 0) { + month += monthsInYear(--extendedYear); + } + // Careful: allow 0..12 in all years + while (month > 12) { + month -= monthsInYear(extendedYear++); + } + + switch (month) { + case HESHVAN: + case KISLEV: + // These two month lengths can vary + return MONTH_LENGTH[month][yearType(extendedYear)]; + + default: + // The rest are a fixed length + return MONTH_LENGTH[month][0]; + } +} + +/** +* Returns the number of days in the given Hebrew year +* @internal +*/ +int32_t HebrewCalendar::handleGetYearLength(int32_t eyear) const { + UErrorCode status = U_ZERO_ERROR; + return startOfYear(eyear+1, status) - startOfYear(eyear, status); +} + +void HebrewCalendar::validateField(UCalendarDateFields field, UErrorCode &status) { + if ((field == UCAL_MONTH || field == UCAL_ORDINAL_MONTH) + && !isLeapYear(handleGetExtendedYear()) && internalGetMonth() == ADAR_1) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + Calendar::validateField(field, status); +} +//------------------------------------------------------------------------- +// Functions for converting from milliseconds to field values +//------------------------------------------------------------------------- + +/** +* Subclasses may override this method to compute several fields +* specific to each calendar system. These are: +* +*
  • ERA +*
  • YEAR +*
  • MONTH +*
  • DAY_OF_MONTH +*
  • DAY_OF_YEAR +*
  • EXTENDED_YEAR
+* +* Subclasses can refer to the DAY_OF_WEEK and DOW_LOCAL fields, +* which will be set when this method is called. Subclasses can +* also call the getGregorianXxx() methods to obtain Gregorian +* calendar equivalents for the given Julian day. +* +*

In addition, subclasses should compute any subclass-specific +* fields, that is, fields from BASE_FIELD_COUNT to +* getFieldCount() - 1. +* @internal +*/ +void HebrewCalendar::handleComputeFields(int32_t julianDay, UErrorCode &status) { + int32_t d = julianDay - 347997; + double m = ClockMath::floorDivide((d * (double)DAY_PARTS), (double) MONTH_PARTS); // Months (approx) + int32_t year = (int32_t)(ClockMath::floorDivide((19. * m + 234.), 235.) + 1.); // Years (approx) + int32_t ys = startOfYear(year, status); // 1st day of year + int32_t dayOfYear = (d - ys); + + // Because of the postponement rules, it's possible to guess wrong. Fix it. + while (dayOfYear < 1) { + year--; + ys = startOfYear(year, status); + dayOfYear = (d - ys); + } + + // Now figure out which month we're in, and the date within that month + int32_t type = yearType(year); + UBool isLeap = isLeapYear(year); + + int32_t month = 0; + int32_t momax = UPRV_LENGTHOF(MONTH_START); + while (month < momax && dayOfYear > ( isLeap ? LEAP_MONTH_START[month][type] : MONTH_START[month][type] ) ) { + month++; + } + if (month >= momax || month<=0) { + // TODO: I found dayOfYear could be out of range when + // a large value is set to julianDay. I patched startOfYear + // to reduce the chace, but it could be still reproduced either + // by startOfYear or other places. For now, we check + // the month is in valid range to avoid out of array index + // access problem here. However, we need to carefully review + // the calendar implementation to check the extreme limit of + // each calendar field and the code works well for any values + // in the valid value range. -yoshito + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + month--; + int dayOfMonth = dayOfYear - (isLeap ? LEAP_MONTH_START[month][type] : MONTH_START[month][type]); + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_EXTENDED_YEAR, year); + int32_t ordinal_month = month; + if (!isLeap && ordinal_month > ADAR_1) { + ordinal_month--; + } + internalSet(UCAL_ORDINAL_MONTH, ordinal_month); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); +} + +//------------------------------------------------------------------------- +// Functions for converting from field values to milliseconds +//------------------------------------------------------------------------- + +/** +* @internal +*/ +int32_t HebrewCalendar::handleGetExtendedYear() { + int32_t year; + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + year = internalGet(UCAL_YEAR, 1); // Default to year 1 + } + return year; +} + +/** +* Return JD of start of given month/year. +* @internal +*/ +int32_t HebrewCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, UBool /*useMonth*/) const { + UErrorCode status = U_ZERO_ERROR; + // Resolve out-of-range months. This is necessary in order to + // obtain the correct year. We correct to + // a 12- or 13-month year (add/subtract 12 or 13, depending + // on the year) but since we _always_ number from 0..12, and + // the leap year determines whether or not month 5 (Adar 1) + // is present, we allow 0..12 in any given year. + while (month < 0) { + month += monthsInYear(--eyear); + } + // Careful: allow 0..12 in all years + while (month > 12) { + month -= monthsInYear(eyear++); + } + + int32_t day = startOfYear(eyear, status); + + if(U_FAILURE(status)) { + return 0; + } + + if (month != 0) { + if (isLeapYear(eyear)) { + day += LEAP_MONTH_START[month][yearType(eyear)]; + } else { + day += MONTH_START[month][yearType(eyear)]; + } + } + + return (int) (day + 347997); +} + +constexpr uint32_t kHebrewRelatedYearDiff = -3760; + +int32_t HebrewCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kHebrewRelatedYearDiff; +} + +void HebrewCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kHebrewRelatedYearDiff); +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + +UBool HebrewCalendar::haveDefaultCentury() const +{ + return true; +} + +static void U_CALLCONV initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + HebrewCalendar calendar(Locale("@calendar=hebrew"),status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + + +UDate HebrewCalendar::defaultCenturyStart() const { + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t HebrewCalendar::defaultCenturyStartYear() const { + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + +bool HebrewCalendar::inTemporalLeapYear(UErrorCode& status) const { + if (U_FAILURE(status)) return false; + int32_t eyear = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) return false; + return isLeapYear(eyear); +} + +static const char * const gTemporalMonthCodesForHebrew[] = { + "M01", "M02", "M03", "M04", "M05", "M05L", "M06", + "M07", "M08", "M09", "M10", "M11", "M12", nullptr +}; + +const char* HebrewCalendar::getTemporalMonthCode(UErrorCode& status) const { + int32_t month = get(UCAL_MONTH, status); + if (U_FAILURE(status)) return nullptr; + return gTemporalMonthCodesForHebrew[month]; +} + +void HebrewCalendar::setTemporalMonthCode(const char* code, UErrorCode& status ) +{ + if (U_FAILURE(status)) return; + int32_t len = static_cast(uprv_strlen(code)); + if (len == 3 || len == 4) { + for (int m = 0; gTemporalMonthCodesForHebrew[m] != nullptr; m++) { + if (uprv_strcmp(code, gTemporalMonthCodesForHebrew[m]) == 0) { + set(UCAL_MONTH, m); + return; + } + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; +} + +int32_t HebrewCalendar::internalGetMonth() const { + if (resolveFields(kMonthPrecedence) == UCAL_ORDINAL_MONTH) { + int32_t ordinalMonth = internalGet(UCAL_ORDINAL_MONTH); + HebrewCalendar *nonConstThis = (HebrewCalendar*)this; // cast away const + + int32_t year = nonConstThis->handleGetExtendedYear(); + return ordinalMonth + ((isLeapYear(year) && (ordinalMonth > ADAR_1)) ? 1: 0); + } + return Calendar::internalGetMonth(); +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(HebrewCalendar) + +U_NAMESPACE_END + +#endif // UCONFIG_NO_FORMATTING + diff --git a/intl/icu/source/i18n/hebrwcal.h b/intl/icu/source/i18n/hebrwcal.h new file mode 100644 index 0000000000..829a642211 --- /dev/null +++ b/intl/icu/source/i18n/hebrwcal.h @@ -0,0 +1,492 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2003-2013, International Business Machines Corporation +* and others. All Rights Reserved. +****************************************************************************** +* +* File HEBRWCAL.H +* +* Modification History: +* +* Date Name Description +* 05/13/2003 srl copied from gregocal.h +* 11/26/2003 srl copied from buddhcal.h +****************************************************************************** +*/ + +#ifndef HEBRWCAL_H +#define HEBRWCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/gregocal.h" + +U_NAMESPACE_BEGIN + +/** + * HebrewCalendar is a subclass of Calendar + * that that implements the traditional Hebrew calendar. + * This is the civil calendar in Israel and the liturgical calendar + * of the Jewish faith worldwide. + *

+ * The Hebrew calendar is lunisolar and thus has a number of interesting + * properties that distinguish it from the Gregorian. Months start + * on the day of (an arithmetic approximation of) each new moon. Since the + * solar year (approximately 365.24 days) is not an even multiple of + * the lunar month (approximately 29.53 days) an extra "leap month" is + * inserted in 7 out of every 19 years. To make matters even more + * interesting, the start of a year can be delayed by up to three days + * in order to prevent certain holidays from falling on the Sabbath and + * to prevent certain illegal year lengths. Finally, the lengths of certain + * months can vary depending on the number of days in the year. + *

+ * The leap month is known as "Adar 1" and is inserted between the + * months of Shevat and Adar in leap years. Since the leap month does + * not come at the end of the year, calculations involving + * month numbers are particularly complex. Users of this class should + * make sure to use the {@link #roll roll} and {@link #add add} methods + * rather than attempting to perform date arithmetic by manipulating + * the fields directly. + *

+ * Note: In the traditional Hebrew calendar, days start at sunset. + * However, in order to keep the time fields in this class + * synchronized with those of the other calendars and with local clock time, + * we treat days and months as beginning at midnight, + * roughly 6 hours after the corresponding sunset. + *

+ * If you are interested in more information on the rules behind the Hebrew + * calendar, see one of the following references: + *

+ *

+ * @see com.ibm.icu.util.GregorianCalendar + * + * @author Laura Werner + * @author Alan Liu + * @author Steven R. Loomis + *

+ * @internal + */ +class U_I18N_API HebrewCalendar : public Calendar { +public: + /** + * Useful constants for HebrewCalendar. + * @internal + */ + enum Month { + /** + * Constant for Tishri, the 1st month of the Hebrew year. + */ + TISHRI, + /** + * Constant for Heshvan, the 2nd month of the Hebrew year. + */ + HESHVAN, + /** + * Constant for Kislev, the 3rd month of the Hebrew year. + */ + KISLEV, + + /** + * Constant for Tevet, the 4th month of the Hebrew year. + */ + TEVET, + + /** + * Constant for Shevat, the 5th month of the Hebrew year. + */ + SHEVAT, + + /** + * Constant for Adar I, the 6th month of the Hebrew year + * (present in leap years only). In non-leap years, the calendar + * jumps from Shevat (5th month) to Adar (7th month). + */ + ADAR_1, + + /** + * Constant for the Adar, the 7th month of the Hebrew year. + */ + ADAR, + + /** + * Constant for Nisan, the 8th month of the Hebrew year. + */ + NISAN, + + /** + * Constant for Iyar, the 9th month of the Hebrew year. + */ + IYAR, + + /** + * Constant for Sivan, the 10th month of the Hebrew year. + */ + SIVAN, + + /** + * Constant for Tammuz, the 11th month of the Hebrew year. + */ + TAMUZ, + + /** + * Constant for Av, the 12th month of the Hebrew year. + */ + AV, + + /** + * Constant for Elul, the 13th month of the Hebrew year. + */ + ELUL + }; + + /** + * Constructs a HebrewCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of HebrewCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + HebrewCalendar(const Locale& aLocale, UErrorCode& success); + + + /** + * Destructor + * @internal + */ + virtual ~HebrewCalendar(); + + /** + * Copy constructor + * @param source the object to be copied. + * @internal + */ + HebrewCalendar(const HebrewCalendar& source); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual HebrewCalendar* clone() const override; + +public: + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "hebrew". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + + // Calendar API + public: + /** + * (Overrides Calendar) UDate Arithmetic function. Adds the specified (signed) amount + * of time to the given time field, based on the calendar's rules. For more + * information, see the documentation for Calendar::add(). + * + * @param field The time field. + * @param amount The amount of date or time to be added to the field. + * @param status Output param set to success/failure code on exit. If any value + * previously set in the time field is invalid, this will be set to + * an error status. + */ + virtual void add(UCalendarDateFields field, int32_t amount, UErrorCode& status) override; + /** + * @deprecated ICU 2.6 use UCalendarDateFields instead of EDateFields + */ + virtual void add(EDateFields field, int32_t amount, UErrorCode& status) override; + + + /** + * (Overrides Calendar) Rolls up or down by the given amount in the specified field. + * For more information, see the documentation for Calendar::roll(). + * + * @param field The time field. + * @param amount Indicates amount to roll. + * @param status Output param set to success/failure code on exit. If any value + * previously set in the time field is invalid, this will be set to + * an error status. + * @internal + */ + virtual void roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) override; + + /** + * (Overrides Calendar) Rolls up or down by the given amount in the specified field. + * For more information, see the documentation for Calendar::roll(). + * + * @param field The time field. + * @param amount Indicates amount to roll. + * @param status Output param set to success/failure code on exit. If any value + * previously set in the time field is invalid, this will be set to + * an error status. + * @deprecated ICU 2.6. Use roll(UCalendarDateFields field, int32_t amount, UErrorCode& status) instead. +` */ + virtual void roll(EDateFields field, int32_t amount, UErrorCode& status) override; + + /** + * @internal + */ + static UBool isLeapYear(int32_t year) ; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + protected: + + /** + * Subclass API for defining limits of different types. + * Subclasses must implement this method to return limits for the + * following fields: + * + * 

UCAL_ERA
+     * UCAL_YEAR
+     * UCAL_MONTH
+     * UCAL_WEEK_OF_YEAR
+     * UCAL_WEEK_OF_MONTH
+     * UCAL_DATE (DAY_OF_MONTH on Java)
+     * UCAL_DAY_OF_YEAR
+     * UCAL_DAY_OF_WEEK_IN_MONTH
+     * UCAL_YEAR_WOY
+     * UCAL_EXTENDED_YEAR
+ * + * @param field one of the above field numbers + * @param limitType one of MINIMUM, GREATEST_MINIMUM, + * LEAST_MAXIMUM, or MAXIMUM + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Return the number of days in the given month of the given extended + * year of this calendar system. Subclasses should override this + * method if they can provide a more correct or more efficient + * implementation than the default implementation in Calendar. + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given extended year of this + * calendar system. Subclasses should override this method if they can + * provide a more correct or more efficient implementation than the + * default implementation in Calendar. + * @stable ICU 2.0 + */ + virtual int32_t handleGetYearLength(int32_t eyear) const override; + /** + * Subclasses may override this method to compute several fields + * specific to each calendar system. These are: + * + *
  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + *

The GregorianCalendar implementation implements + * a calendar with the specified Julian/Gregorian cutover date. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + /** + * Return the extended year defined by the current fields. This will + * use the UCAL_EXTENDED_YEAR field or the UCAL_YEAR and supra-year fields (such + * as UCAL_ERA) specific to the calendar system, depending on which set of + * fields is newer. + * @return the extended year + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + /** + * Return the Julian day number of day before the first day of the + * given month in the given extended year. Subclasses should override + * this method to implement their calendar system. + * @param eyear the extended year + * @param month the zero-based month, or 0 if useMonth is false + * @param useMonth if false, compute the day before the first day of + * the given year, otherwise, compute the day before the first day of + * the given month + * @param return the Julian day number of the day before the first + * day of the given month and year + * @internal + */ + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, + UBool useMonth) const override; + + + /** + * Validate a single field of this calendar. + * Overrides Calendar::validateField(int) to provide + * special handling for month validation for Hebrew calendar. + * @internal + */ + virtual void validateField(UCalendarDateFields field, UErrorCode &status) override; + + protected: + /** + * Returns true because the Hebrew Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + + public: + /** + * Returns true if the date is in a leap year. + * + * @param status ICU Error Code + * @return True if the date in the fields is in a Temporal proposal + * defined leap year. False otherwise. + */ + virtual bool inTemporalLeapYear(UErrorCode& status) const override; + + /** + * Gets The Temporal monthCode value corresponding to the month for the date. + * The value is a string identifier that starts with the literal grapheme + * "M" followed by two graphemes representing the zero-padded month number + * of the current month in a normal (non-leap) year and suffixed by an + * optional literal grapheme "L" if this is a leap month in a lunisolar + * calendar. For the Hebrew calendar, the values are "M01" .. "M12" for + * non-leap year, and "M01" .. "M05", "M05L", "M06" .. "M12" for leap year. + * + * @param status ICU Error Code + * @return One of 13 possible strings in {"M01".. "M05", "M05L", + * "M06" .. "M12"}. + * @draft ICU 73 + */ + virtual const char* getTemporalMonthCode(UErrorCode& status) const override; + + /** + * Sets The Temporal monthCode which is a string identifier that starts + * with the literal grapheme "M" followed by two graphemes representing + * the zero-padded month number of the current month in a normal + * (non-leap) year and suffixed by an optional literal grapheme "L" if this + * is a leap month in a lunisolar calendar. For Hebrew calendar, the values + * are "M01" .. "M12" for non-leap years, and "M01" .. "M05", "M05L", "M06" + * .. "M12" for leap year. + * + * @param temporalMonth The value to be set for temporal monthCode. + * @param status ICU Error Code + * + * @draft ICU 73 + */ + virtual void setTemporalMonthCode(const char* code, UErrorCode& status ) override; + + protected: + virtual int32_t internalGetMonth() const override; + + private: // Calendar-specific implementation + /** + * Finds the day # of the first day in the given Hebrew year. + * To do this, we want to calculate the time of the Tishri 1 new moon + * in that year. + *

+ * The algorithm here is similar to ones described in a number of + * references, including: + *

+ * @param year extended year + * @return day number (JD) + * @internal + */ + static int32_t startOfYear(int32_t year, UErrorCode& status); + + static int32_t absoluteDayToDayOfWeek(int32_t day) ; + + /** + * @internal + */ + int32_t yearType(int32_t year) const; + + /** + * @internal + */ + static int32_t monthsInYear(int32_t year) ; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif +//eof + diff --git a/intl/icu/source/i18n/i18n.rc b/intl/icu/source/i18n/i18n.rc new file mode 100644 index 0000000000..c31ef3ad5f --- /dev/null +++ b/intl/icu/source/i18n/i18n.rc @@ -0,0 +1,110 @@ +// Do not edit with Microsoft Developer Studio Resource Editor. +// It will permanently substitute version numbers that are intended to be +// picked up by the pre-processor during each build. +// Copyright (C) 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// Copyright (c) 2001-2010 International Business Machines +// Corporation and others. All Rights Reserved. +// +#include "../common/msvcres.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#include +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// + +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL +#pragma code_page(1252) + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "../common/msvcres.h\0" +END + +2 TEXTINCLUDE +BEGIN + "#include \0" +END + +3 TEXTINCLUDE +BEGIN + "\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// +#define STR(s) #s +#define CommaVersionString(a, b, c, d) STR(a) ", " STR(b) ", " STR(c) ", " STR(d) "\0" + +VS_VERSION_INFO VERSIONINFO + FILEVERSION U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, U_ICU_VERSION_BUILDLEVEL_NUM + PRODUCTVERSION U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, U_ICU_VERSION_BUILDLEVEL_NUM + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "00000000" + BEGIN + VALUE "Comments", ICU_WEBSITE "\0" + VALUE "CompanyName", ICU_COMPANY "\0" + VALUE "FileDescription", ICU_PRODUCT_PREFIX " I18N DLL\0" + VALUE "FileVersion", CommaVersionString(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, U_ICU_VERSION_BUILDLEVEL_NUM) + VALUE "LegalCopyright", U_COPYRIGHT_STRING "\0" +#ifdef _DEBUG + VALUE "OriginalFilename", "icuin" U_ICU_VERSION_SHORT "d.dll\0" +#else + VALUE "OriginalFilename", "icuin" U_ICU_VERSION_SHORT ".dll\0" +#endif + VALUE "PrivateBuild", "\0" + VALUE "ProductName", ICU_PRODUCT "\0" + VALUE "ProductVersion", CommaVersionString(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM, U_ICU_VERSION_PATCHLEVEL_NUM, U_ICU_VERSION_BUILDLEVEL_NUM) + VALUE "SpecialBuild", "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x000, 0000 + END +END + +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// + + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED + diff --git a/intl/icu/source/i18n/i18n.vcxproj b/intl/icu/source/i18n/i18n.vcxproj new file mode 100644 index 0000000000..73ab2d4651 --- /dev/null +++ b/intl/icu/source/i18n/i18n.vcxproj @@ -0,0 +1,514 @@ + + + + {0178B127-6269-407D-B112-93877BB62776} + + + DynamicLibrary + false + MultiByte + + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + .\$(Platform)\$(Configuration)\ + .\$(Platform)\$(Configuration)\ + + .\x86\$(Configuration)\ + .\x86\$(Configuration)\ + + true + false + + + + + $(OutDir)\icuin.tlb + + + U_ATTRIBUTE_DEPRECATED=;U_I18N_IMPLEMENTATION;%(PreprocessorDefinitions) + false + Level3 + ..\..\include;..\common;%(AdditionalIncludeDirectories) + Default + ProgramDatabase + $(OutDir)/icuin.pch + $(OutDir)/ + $(OutDir)/ + $(OutDir)/icuin.pdb + + + ../common;%(AdditionalIncludeDirectories) + + + .\..\..\$(IcuLibOutputDir);%(AdditionalLibraryDirectories) + + + + + + true + MultiThreadedDebugDLL + + + icuucd.lib;%(AdditionalDependencies) + ..\..\$(IcuBinOutputDir)\icuin$(IcuMajorVersion)d.dll + .\..\..\$(IcuLibOutputDir)\icuind.pdb + ..\..\$(IcuLibOutputDir)\icuind.lib + + + + + + MultiThreadedDLL + true + + + icuuc.lib;%(AdditionalDependencies) + ..\..\$(IcuBinOutputDir)\icuin$(IcuMajorVersion).dll + .\..\..\$(IcuLibOutputDir)\icuin.pdb + ..\..\$(IcuLibOutputDir)\icuin.lib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/intl/icu/source/i18n/i18n.vcxproj.filters b/intl/icu/source/i18n/i18n.vcxproj.filters new file mode 100644 index 0000000000..e35edb5c1b --- /dev/null +++ b/intl/icu/source/i18n/i18n.vcxproj.filters @@ -0,0 +1,1292 @@ + + + + + {68f85997-0019-471f-b155-5eed0137f082} + + + {8152306c-460d-49f3-961a-c500eda24e9d} + + + {15349ca9-e31d-4f6b-a4c4-f73892fa5aa6} + + + {faac495b-a9d6-47ad-ae47-a56c30c60b3a} + + + {e750fa6c-e471-4db0-92f9-81c84d84b5ba} + + + {59edb5a3-26d1-4c91-af50-4aa35e6e9730} + + + {7b4a0782-fad3-44cd-b3b4-e75b0356d600} + + + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + misc + + + regex + + + regex + + + regex + + + regex + + + regex + + + regex + + + regex + + + regex + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + spoof + + + spoof + + + spoof + + + spoof + + + spoof + + + collation + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + misc + + + regex + + + regex + + + regex + + + regex + + + regex + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + transforms + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + charset detect + + + spoof + + + spoof + + + spoof + + + formatting + + + formatting + + + formatting + + + formatting + + + formatting + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + collation + + + formatting + + + + + misc + + + diff --git a/intl/icu/source/i18n/i18n_uwp.vcxproj b/intl/icu/source/i18n/i18n_uwp.vcxproj new file mode 100644 index 0000000000..9afcb5ddb9 --- /dev/null +++ b/intl/icu/source/i18n/i18n_uwp.vcxproj @@ -0,0 +1,746 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Debug + ARM + + + Debug + ARM64 + + + Release + Win32 + + + Release + x64 + + + Release + ARM + + + Release + ARM64 + + + + {6786C051-383B-47E0-9E82-B8B994E06A25} + DynamicLibrary + en-US + + + DynamicLibrary + false + MultiByte + + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + .\$(Platform)\$(Configuration)UWP\ + .\$(Platform)\$(Configuration)UWP\ + + .\x86\$(Configuration)UWP\ + .\x86\$(Configuration)UWP\ + + true + false + + + + true + true + + + ..\..\include;..\common;%(AdditionalIncludeDirectories) + + + U_ATTRIBUTE_DEPRECATED=;_CRT_SECURE_NO_DEPRECATE;U_I18N_IMPLEMENTATION;U_PLATFORM_USES_ONLY_WIN32_API=1;%(PreprocessorDefinitions) + true + + + true + false + true + Level3 + true + Default + NotUsing + false + /utf-8 %(AdditionalOptions) + ProgramDatabase + + + 0x0409 + ../common;%(AdditionalIncludeDirectories) + + + true + false + true + + + true + + + + + NDEBUG;%(PreprocessorDefinitions) + + + NDEBUG;%(PreprocessorDefinitions) + MultiThreadedDLL + + + NDEBUG;%(PreprocessorDefinitions) + + + true + + + + + _DEBUG;%(PreprocessorDefinitions) + + + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + Disabled + EnableFastChecks + true + + + _DEBUG;%(PreprocessorDefinitions) + + + true + + + + + Win32 + + + WIN32;%(PreprocessorDefinitions) + + + + + X64 + + + WIN64;WIN32;%(PreprocessorDefinitions) + + + MachineX64 + + + + + ARM + + + ARM;WIN32;%(PreprocessorDefinitions) + + + MachineARM + + + + + ARM64 + + + ARM64;WIN32;%(PreprocessorDefinitions) + + + MachineARM64 + + + + + .\..\..\lib32uwp\icuin.tlb + + + .\x86\ReleaseUWP/i18n.pch + .\x86\ReleaseUWP/ + .\x86\ReleaseUWP/ + .\x86\ReleaseUWP/ + + + ..\..\bin32uwp\icuin$(IcuMajorVersion).dll + .\..\..\lib32uwp\icuin.pdb + ..\..\lib32uwp\icuin.lib + ..\..\lib32uwp\icuuc.lib;%(AdditionalDependencies) + + + + + .\..\..\lib32uwp\icuind.tlb + + + .\x86\DebugUWP/i18n.pch + .\x86\DebugUWP/ + .\x86\DebugUWP/ + .\x86\DebugUWP/ + + + ..\..\bin32uwp\icuin$(IcuMajorVersion)d.dll + .\..\..\lib32uwp\icuind.pdb + ..\..\lib32uwp\icuind.lib + ..\..\lib32uwp\icuucd.lib;%(AdditionalDependencies) + + + + + .\..\..\lib64uwp\icuin.tlb + + + .\x64\ReleaseUWP/i18n.pch + .\x64\ReleaseUWP/ + .\x64\ReleaseUWP/ + .\x64\ReleaseUWP/ + + + ..\..\bin64uwp\icuin$(IcuMajorVersion).dll + .\..\..\lib64uwp\icuin.pdb + ..\..\lib64uwp\icuin.lib + ..\..\lib64uwp\icuuc.lib;%(AdditionalDependencies) + + + + + .\..\..\lib64uwp\icuind.tlb + + + .\x64\DebugUWP/i18n.pch + .\x64\DebugUWP/ + .\x64\DebugUWP/ + .\x64\DebugUWP/ + + + ..\..\bin64uwp\icuin$(IcuMajorVersion)d.dll + .\..\..\lib64uwp\icuind.pdb + ..\..\lib64uwp\icuind.lib + ..\..\lib64uwp\icuucd.lib;%(AdditionalDependencies) + + + + + .\..\..\libARMuwp\icuin.tlb + + + .\ARM\ReleaseUWP/i18n.pch + .\ARM\ReleaseUWP/ + .\ARM\ReleaseUWP/ + .\ARM\ReleaseUWP/ + + + ..\..\binARMuwp\icuin$(IcuMajorVersion).dll + .\..\..\libARMuwp\icuin.pdb + ..\..\libARMuwp\icuin.lib + ..\..\libARMuwp\icuuc.lib;%(AdditionalDependencies) + + + + + .\..\..\libARMuwp\icuind.tlb + + + .\ARM\DebugUWP/i18n.pch + .\ARM\DebugUWP/ + .\ARM\DebugUWP/ + .\ARM\DebugUWP/ + + + ..\..\binARMuwp\icuin$(IcuMajorVersion)d.dll + .\..\..\libARMuwp\icuind.pdb + ..\..\libARMuwp\icuind.lib + ..\..\libARMuwp\icuucd.lib;%(AdditionalDependencies) + + + + + .\..\..\libARM64uwp\icuin.tlb + + + .\ARM64\ReleaseUWP/i18n.pch + .\ARM64\ReleaseUWP/ + .\ARM64\ReleaseUWP/ + .\ARM64\ReleaseUWP/ + + + ..\..\binARM64uwp\icuin$(IcuMajorVersion).dll + .\..\..\libARM64uwp\icuin.pdb + ..\..\libARM64uwp\icuin.lib + ..\..\libARM64uwp\icuuc.lib;%(AdditionalDependencies) + + + + + .\..\..\libARM64uwp\icuind.tlb + + + .\ARM64\DebugUWP/i18n.pch + .\ARM64\DebugUWP/ + .\ARM64\DebugUWP/ + .\ARM64\DebugUWP/ + + + ..\..\binARM64uwp\icuin$(IcuMajorVersion)d.dll + .\..\..\libARM64uwp\icuind.pdb + ..\..\libARM64uwp\icuind.lib + ..\..\libARM64uwp\icuucd.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/intl/icu/source/i18n/indiancal.cpp b/intl/icu/source/i18n/indiancal.cpp new file mode 100644 index 0000000000..29c2749f48 --- /dev/null +++ b/intl/icu/source/i18n/indiancal.cpp @@ -0,0 +1,379 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + * Copyright (C) 2003-2014, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File INDIANCAL.CPP + ***************************************************************************** + */ + +#include "indiancal.h" +#include +#if !UCONFIG_NO_FORMATTING + +#include "mutex.h" +#include +#include "gregoimp.h" // Math +#include "uhash.h" + +// Debugging +#ifdef U_DEBUG_INDIANCAL +#include +#include + +#endif + +U_NAMESPACE_BEGIN + +// Implementation of the IndianCalendar class + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + + +IndianCalendar* IndianCalendar::clone() const { + return new IndianCalendar(*this); +} + +IndianCalendar::IndianCalendar(const Locale& aLocale, UErrorCode& success) + : Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +IndianCalendar::IndianCalendar(const IndianCalendar& other) : Calendar(other) { +} + +IndianCalendar::~IndianCalendar() +{ +} +const char *IndianCalendar::getType() const { + return "indian"; +} + +static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 0, 0}, // ERA + { -5000000, -5000000, 5000000, 5000000}, // YEAR + { 0, 0, 11, 11}, // MONTH + { 1, 1, 52, 53}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 30, 31}, // DAY_OF_MONTH + { 1, 1, 365, 366}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 5, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 11, 11}, // ORDINAL_MONTH +}; + +static const int32_t INDIAN_ERA_START = 78; +static const int32_t INDIAN_YEAR_START = 80; + +int32_t IndianCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return LIMITS[field][limitType]; +} + +/* + * Determine whether the given gregorian year is a Leap year + */ +static UBool isGregorianLeap(int32_t year) +{ + return Grego::isLeapYear(year); +} + +//---------------------------------------------------------------------- +// Calendar framework +//---------------------------------------------------------------------- + +/* + * Return the length (in days) of the given month. + * + * @param eyear The year in Saka Era + * @param month The month(0-based) in Indian calendar + */ +int32_t IndianCalendar::handleGetMonthLength(int32_t eyear, int32_t month) const { + if (month < 0 || month > 11) { + eyear += ClockMath::floorDivide(month, 12, &month); + } + + if (isGregorianLeap(eyear + INDIAN_ERA_START) && month == 0) { + return 31; + } + + if (month >= 1 && month <= 5) { + return 31; + } + + return 30; +} + +/* + * Return the number of days in the given Indian year + * + * @param eyear The year in Saka Era. + */ +int32_t IndianCalendar::handleGetYearLength(int32_t eyear) const { + return isGregorianLeap(eyear + INDIAN_ERA_START) ? 366 : 365; +} +/* + * Returns the Julian Day corresponding to gregorian date + * + * @param year The Gregorian year + * @param month The month in Gregorian Year, 0 based. + * @param date The date in Gregorian day in month + */ +static double gregorianToJD(int32_t year, int32_t month, int32_t date) { + return Grego::fieldsToDay(year, month, date) + kEpochStartAsJulianDay - 0.5; +} + +/* + * Returns the Gregorian Date corresponding to a given Julian Day + * Month is 0 based. + * @param jd The Julian Day + */ +static int32_t* jdToGregorian(double jd, int32_t gregorianDate[3]) { + int32_t gdow; + Grego::dayToFields(jd - kEpochStartAsJulianDay, + gregorianDate[0], gregorianDate[1], gregorianDate[2], gdow); + return gregorianDate; +} + + +//------------------------------------------------------------------------- +// Functions for converting from field values to milliseconds.... +//------------------------------------------------------------------------- +static double IndianToJD(int32_t year, int32_t month, int32_t date) { + int32_t leapMonth, gyear, m; + double start, jd; + + gyear = year + INDIAN_ERA_START; + + + if(isGregorianLeap(gyear)) { + leapMonth = 31; + start = gregorianToJD(gyear, 2 /* The third month in 0 based month */, 21); + } + else { + leapMonth = 30; + start = gregorianToJD(gyear, 2 /* The third month in 0 based month */, 22); + } + + if (month == 1) { + jd = start + (date - 1); + } else { + jd = start + leapMonth; + m = month - 2; + + //m = Math.min(m, 5); + if (m > 5) { + m = 5; + } + + jd += m * 31; + + if (month >= 8) { + m = month - 7; + jd += m * 30; + } + jd += date - 1; + } + + return jd; +} + +/* + * Return JD of start of given month/year of Indian Calendar + * @param eyear The year in Indian Calendar measured from Saka Era (78 AD). + * @param month The month in Indian calendar + */ +int32_t IndianCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, UBool /* useMonth */ ) const { + + //month is 0 based; converting it to 1-based + int32_t imonth; + + // If the month is out of range, adjust it into range, and adjust the extended year accordingly + if (month < 0 || month > 11) { + eyear += (int32_t)ClockMath::floorDivide(month, 12, &month); + } + + if(month == 12){ + imonth = 1; + } else { + imonth = month + 1; + } + + double jd = IndianToJD(eyear ,imonth, 1); + + return (int32_t)jd; +} + +//------------------------------------------------------------------------- +// Functions for converting from milliseconds to field values +//------------------------------------------------------------------------- + +int32_t IndianCalendar::handleGetExtendedYear() { + int32_t year; + + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + year = internalGet(UCAL_YEAR, 1); // Default to year 1 + } + + return year; +} + +/* + * Override Calendar to compute several fields specific to the Indian + * calendar system. These are: + * + *
  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + */ +void IndianCalendar::handleComputeFields(int32_t julianDay, UErrorCode& /* status */) { + double jdAtStartOfGregYear; + int32_t leapMonth, IndianYear, yday, IndianMonth, IndianDayOfMonth, mday; + int32_t gregorianYear; // Stores gregorian date corresponding to Julian day; + int32_t gd[3]; + + gregorianYear = jdToGregorian(julianDay, gd)[0]; // Gregorian date for Julian day + IndianYear = gregorianYear - INDIAN_ERA_START; // Year in Saka era + jdAtStartOfGregYear = gregorianToJD(gregorianYear, 0, 1); // JD at start of Gregorian year + yday = (int32_t)(julianDay - jdAtStartOfGregYear); // Day number in Gregorian year (starting from 0) + + if (yday < INDIAN_YEAR_START) { + // Day is at the end of the preceding Saka year + IndianYear -= 1; + leapMonth = isGregorianLeap(gregorianYear - 1) ? 31 : 30; // Days in leapMonth this year, previous Gregorian year + yday += leapMonth + (31 * 5) + (30 * 3) + 10; + } else { + leapMonth = isGregorianLeap(gregorianYear) ? 31 : 30; // Days in leapMonth this year + yday -= INDIAN_YEAR_START; + } + + if (yday < leapMonth) { + IndianMonth = 0; + IndianDayOfMonth = yday + 1; + } else { + mday = yday - leapMonth; + if (mday < (31 * 5)) { + IndianMonth = (int32_t)uprv_floor(mday / 31) + 1; + IndianDayOfMonth = (mday % 31) + 1; + } else { + mday -= 31 * 5; + IndianMonth = (int32_t)uprv_floor(mday / 30) + 6; + IndianDayOfMonth = (mday % 30) + 1; + } + } + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_EXTENDED_YEAR, IndianYear); + internalSet(UCAL_YEAR, IndianYear); + internalSet(UCAL_MONTH, IndianMonth); + internalSet(UCAL_ORDINAL_MONTH, IndianMonth); + internalSet(UCAL_DAY_OF_MONTH, IndianDayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, yday + 1); // yday is 0-based +} + +constexpr uint32_t kIndianRelatedYearDiff = 79; + +int32_t IndianCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kIndianRelatedYearDiff; +} + +void IndianCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kIndianRelatedYearDiff); +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + + +UBool IndianCalendar::haveDefaultCentury() const +{ + return true; +} + +static void U_CALLCONV +initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + + IndianCalendar calendar ( Locale ( "@calendar=Indian" ), status); + if ( U_SUCCESS ( status ) ) { + calendar.setTime ( Calendar::getNow(), status ); + calendar.add ( UCAL_YEAR, -80, status ); + + UDate newStart = calendar.getTime ( status ); + int32_t newYear = calendar.get ( UCAL_YEAR, status ); + + gSystemDefaultCenturyStart = newStart; + gSystemDefaultCenturyStartYear = newYear; + } + // We have no recourse upon failure. +} + + +UDate +IndianCalendar::defaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t +IndianCalendar::defaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IndianCalendar) + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/indiancal.h b/intl/icu/source/i18n/indiancal.h new file mode 100644 index 0000000000..5ef9113a85 --- /dev/null +++ b/intl/icu/source/i18n/indiancal.h @@ -0,0 +1,333 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ***************************************************************************** + * Copyright (C) 2003-2008, International Business Machines Corporation + * and others. All Rights Reserved. + ***************************************************************************** + * + * File INDIANCAL.H + ***************************************************************************** + */ + +#ifndef INDIANCAL_H +#define INDIANCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" + +U_NAMESPACE_BEGIN + +/** + * Concrete class which provides the Indian calendar. + *

+ * IndianCalendar is a subclass of Calendar + * that numbers years since the beginning of SAKA ERA. This is the civil calendar + * which is accepted by government of India as Indian National Calendar. + * The two calendars most widely used in India today are the Vikrama calendar + * followed in North India and the Shalivahana or Saka calendar which is followed + * in South India and Maharashtra. + + * A variant of the Shalivahana Calendar was reformed and standardized as the + * Indian National calendar in 1957. + *

+ * Some details of Indian National Calendar (to be implemented) : + * The Months + * Month Length Start date (Gregorian) + * ================================================= + * 1 Chaitra 30/31 March 22* + * 2 Vaisakha 31 April 21 + * 3 Jyaistha 31 May 22 + * 4 Asadha 31 June 22 + * 5 Sravana 31 July 23 + * 6 Bhadra 31 August 23 + * 7 Asvina 30 September 23 + * 8 Kartika 30 October 23 + * 9 Agrahayana 30 November 22 + * 10 Pausa 30 December 22 + * 11 Magha 30 January 21 + * 12 Phalguna 30 February 20 + + * In leap years, Chaitra has 31 days and starts on March 21 instead. + * The leap years of Gregorian calendar and Indian National Calendar are in synchornization. + * So When its a leap year in Gregorian calendar then Chaitra has 31 days. + * + * The Years + * Years are counted in the Saka Era, which starts its year 0 in 78AD (by gregorian calendar). + * So for eg. 9th June 2006 by Gregorian Calendar, is same as 19th of Jyaistha in 1928 of Saka + * era by Indian National Calendar. + *

+ * The Indian Calendar has only one allowable era: Saka Era. If the + * calendar is not in lenient mode (see setLenient), dates before + * 1/1/1 Saka Era are rejected with an IllegalArgumentException. + *

+ * @internal + */ + + +class U_I18N_API IndianCalendar : public Calendar { +public: + /** + * Useful constants for IndianCalendar. + * @internal + */ + enum EEras { + /** + * Constant for Chaitra, the 1st month of the Indian year. + */ + CHAITRA, + + /** + * Constant for Vaisakha, the 2nd month of the Indian year. + */ + VAISAKHA, + + /** + * Constant for Jyaistha, the 3rd month of the Indian year. + */ + JYAISTHA, + + /** + * Constant for Asadha, the 4th month of the Indian year. + */ + ASADHA, + + /** + * Constant for Sravana, the 5th month of the Indian year. + */ + SRAVANA, + + /** + * Constant for Bhadra the 6th month of the Indian year + */ + BHADRA, + + /** + * Constant for the Asvina, the 7th month of the Indian year. + */ + ASVINA, + + /** + * Constant for Kartika, the 8th month of the Indian year. + */ + KARTIKA, + + /** + * Constant for Agrahayana, the 9th month of the Indian year. + */ + AGRAHAYANA, + + /** + * Constant for Pausa, the 10th month of the Indian year. + */ + PAUSA, + + /** + * Constant for Magha, the 11th month of the Indian year. + */ + MAGHA, + + /** + * Constant for Phalguna, the 12th month of the Indian year. + */ + PHALGUNA + }; + + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs an IndianCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IndianCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @param beCivil Whether the calendar should be civil (default-true) or religious (false) + * @internal + */ + IndianCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IndianCalendar(const IndianCalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~IndianCalendar(); + + /** + * Determines whether this object uses the fixed-cycle Indian civil calendar + * or an approximation of the religious, astronomical calendar. + * + * @param beCivil CIVIL to use the civil calendar, + * ASTRONOMICAL to use the astronomical calendar. + * @internal + */ + //void setCivil(ECivil beCivil, UErrorCode &status); + + /** + * Returns true if this object is using the fixed-cycle civil + * calendar, or false if using the religious, astronomical + * calendar. + * @internal + */ + //UBool isCivil(); + + + // TODO: copy c'tor, etc + + // clone + virtual IndianCalendar* clone() const override; + + private: + /** + * Determine whether a year is the gregorian year a leap year + */ + //static UBool isGregorianLeap(int32_t year); + //---------------------------------------------------------------------- + // Calendar framework + //---------------------------------------------------------------------- + protected: + /** + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Return the length (in days) of the given month. + * + * @param year The year in Saka era + * @param year The month(0-based) in Indian year + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given Indian year + * @internal + */ + virtual int32_t handleGetYearLength(int32_t extendedYear) const override; + + //------------------------------------------------------------------------- + // Functions for converting from field values to milliseconds.... + //------------------------------------------------------------------------- + + // Return JD of start of given month/year + /** + * @internal + */ + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const override; + + //------------------------------------------------------------------------- + // Functions for converting from milliseconds to field values + //------------------------------------------------------------------------- + + /** + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Override Calendar to compute several fields specific to the Indian + * calendar system. These are: + * + *

  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "indian". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + +private: + IndianCalendar() = delete; // default constructor not implemented + + // Default century. +protected: + /** + * Returns true because the Indian Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; +}; + +U_NAMESPACE_END + +#endif +#endif + + + diff --git a/intl/icu/source/i18n/inputext.cpp b/intl/icu/source/i18n/inputext.cpp new file mode 100644 index 0000000000..bce9862c0f --- /dev/null +++ b/intl/icu/source/i18n/inputext.cpp @@ -0,0 +1,164 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2005-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "inputext.h" + +#include "cmemory.h" +#include "cstring.h" + +#include + +U_NAMESPACE_BEGIN + +#define BUFFER_SIZE 8192 + +#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type)) +#define DELETE_ARRAY(array) uprv_free((void *) (array)) + +InputText::InputText(UErrorCode &status) + : fInputBytes(NEW_ARRAY(uint8_t, BUFFER_SIZE)), // The text to be checked. Markup will have been + // removed if appropriate. + fByteStats(NEW_ARRAY(int16_t, 256)), // byte frequency statistics for the input text. + // Value is percent, not absolute. + fDeclaredEncoding(0), + fRawInput(0), + fRawLength(0) +{ + if (fInputBytes == nullptr || fByteStats == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } +} + +InputText::~InputText() +{ + DELETE_ARRAY(fDeclaredEncoding); + DELETE_ARRAY(fByteStats); + DELETE_ARRAY(fInputBytes); +} + +void InputText::setText(const char *in, int32_t len) +{ + fInputLen = 0; + fC1Bytes = false; + fRawInput = (const uint8_t *) in; + fRawLength = len == -1? (int32_t)uprv_strlen(in) : len; +} + +void InputText::setDeclaredEncoding(const char* encoding, int32_t len) +{ + if(encoding) { + if (len == -1) { + len = (int32_t)uprv_strlen(encoding); + } + + len += 1; // to make place for the \0 at the end. + uprv_free(fDeclaredEncoding); + fDeclaredEncoding = NEW_ARRAY(char, len); + uprv_strncpy(fDeclaredEncoding, encoding, len); + } +} + +UBool InputText::isSet() const +{ + return fRawInput != nullptr; +} + +/** +* MungeInput - after getting a set of raw input data to be analyzed, preprocess +* it by removing what appears to be html markup. +* +* @internal +*/ +void InputText::MungeInput(UBool fStripTags) { + int srci = 0; + int dsti = 0; + uint8_t b; + bool inMarkup = false; + int32_t openTags = 0; + int32_t badTags = 0; + + // + // html / xml markup stripping. + // quick and dirty, not 100% accurate, but hopefully good enough, statistically. + // discard everything within < brackets > + // Count how many total '<' and illegal (nested) '<' occur, so we can make some + // guess as to whether the input was actually marked up at all. + // TODO: Think about how this interacts with EBCDIC charsets that are detected. + if (fStripTags) { + for (srci = 0; srci < fRawLength && dsti < BUFFER_SIZE; srci += 1) { + b = fRawInput[srci]; + + if (b == (uint8_t)0x3C) { /* Check for the ASCII '<' */ + if (inMarkup) { + badTags += 1; + } + + inMarkup = true; + openTags += 1; + } + + if (! inMarkup) { + fInputBytes[dsti++] = b; + } + + if (b == (uint8_t)0x3E) { /* Check for the ASCII '>' */ + inMarkup = false; + } + } + + fInputLen = dsti; + } + + // + // If it looks like this input wasn't marked up, or if it looks like it's + // essentially nothing but markup abandon the markup stripping. + // Detection will have to work on the unstripped input. + // + if (openTags<5 || openTags/5 < badTags || + (fInputLen < 100 && fRawLength>600)) + { + int32_t limit = fRawLength; + + if (limit > BUFFER_SIZE) { + limit = BUFFER_SIZE; + } + + for (srci=0; srci +#include "gregoimp.h" // Math +#include "astro.h" // CalendarAstronomer +#include "uhash.h" +#include "ucln_in.h" +#include "uassert.h" + +static const UDate HIJRA_MILLIS = -42521587200000.0; // 7/16/622 AD 00:00 + +// Debugging +#ifdef U_DEBUG_ISLAMCAL +# include +# include +static void debug_islamcal_loc(const char *f, int32_t l) +{ + fprintf(stderr, "%s:%d: ", f, l); +} + +static void debug_islamcal_msg(const char *pat, ...) +{ + va_list ap; + va_start(ap, pat); + vfprintf(stderr, pat, ap); + fflush(stderr); +} +// must use double parens, i.e.: U_DEBUG_ISLAMCAL_MSG(("four is: %d",4)); +#define U_DEBUG_ISLAMCAL_MSG(x) {debug_islamcal_loc(__FILE__,__LINE__);debug_islamcal_msg x;} +#else +#define U_DEBUG_ISLAMCAL_MSG(x) +#endif + + +// --- The cache -- +// cache of months +static icu::CalendarCache *gMonthCache = nullptr; +static icu::CalendarAstronomer *gIslamicCalendarAstro = nullptr; + +U_CDECL_BEGIN +static UBool calendar_islamic_cleanup() { + if (gMonthCache) { + delete gMonthCache; + gMonthCache = nullptr; + } + if (gIslamicCalendarAstro) { + delete gIslamicCalendarAstro; + gIslamicCalendarAstro = nullptr; + } + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +// Implementation of the IslamicCalendar class + +/** + * Friday EPOC + */ +static const int32_t CIVIL_EPOC = 1948440; // CE 622 July 16 Friday (Julian calendar) / CE 622 July 19 (Gregorian calendar) + +/** + * Thursday EPOC + */ +static const int32_t ASTRONOMICAL_EPOC = 1948439; // CE 622 July 15 Thursday (Julian calendar) + + +static const int32_t UMALQURA_YEAR_START = 1300; +static const int32_t UMALQURA_YEAR_END = 1600; + +static const int UMALQURA_MONTHLENGTH[] = { + //* 1300 -1302 */ "1010 1010 1010", "1101 0101 0100", "1110 1100 1001", + 0x0AAA, 0x0D54, 0x0EC9, + //* 1303 -1307 */ "0110 1101 0100", "0110 1110 1010", "0011 0110 1100", "1010 1010 1101", "0101 0101 0101", + 0x06D4, 0x06EA, 0x036C, 0x0AAD, 0x0555, + //* 1308 -1312 */ "0110 1010 1001", "0111 1001 0010", "1011 1010 1001", "0101 1101 0100", "1010 1101 1010", + 0x06A9, 0x0792, 0x0BA9, 0x05D4, 0x0ADA, + //* 1313 -1317 */ "0101 0101 1100", "1101 0010 1101", "0110 1001 0101", "0111 0100 1010", "1011 0101 0100", + 0x055C, 0x0D2D, 0x0695, 0x074A, 0x0B54, + //* 1318 -1322 */ "1011 0110 1010", "0101 1010 1101", "0100 1010 1110", "1010 0100 1111", "0101 0001 0111", + 0x0B6A, 0x05AD, 0x04AE, 0x0A4F, 0x0517, + //* 1323 -1327 */ "0110 1000 1011", "0110 1010 0101", "1010 1101 0101", "0010 1101 0110", "1001 0101 1011", + 0x068B, 0x06A5, 0x0AD5, 0x02D6, 0x095B, + //* 1328 -1332 */ "0100 1001 1101", "1010 0100 1101", "1101 0010 0110", "1101 1001 0101", "0101 1010 1100", + 0x049D, 0x0A4D, 0x0D26, 0x0D95, 0x05AC, + //* 1333 -1337 */ "1001 1011 0110", "0010 1011 1010", "1010 0101 1011", "0101 0010 1011", "1010 1001 0101", + 0x09B6, 0x02BA, 0x0A5B, 0x052B, 0x0A95, + //* 1338 -1342 */ "0110 1100 1010", "1010 1110 1001", "0010 1111 0100", "1001 0111 0110", "0010 1011 0110", + 0x06CA, 0x0AE9, 0x02F4, 0x0976, 0x02B6, + //* 1343 -1347 */ "1001 0101 0110", "1010 1100 1010", "1011 1010 0100", "1011 1101 0010", "0101 1101 1001", + 0x0956, 0x0ACA, 0x0BA4, 0x0BD2, 0x05D9, + //* 1348 -1352 */ "0010 1101 1100", "1001 0110 1101", "0101 0100 1101", "1010 1010 0101", "1011 0101 0010", + 0x02DC, 0x096D, 0x054D, 0x0AA5, 0x0B52, + //* 1353 -1357 */ "1011 1010 0101", "0101 1011 0100", "1001 1011 0110", "0101 0101 0111", "0010 1001 0111", + 0x0BA5, 0x05B4, 0x09B6, 0x0557, 0x0297, + //* 1358 -1362 */ "0101 0100 1011", "0110 1010 0011", "0111 0101 0010", "1011 0110 0101", "0101 0110 1010", + 0x054B, 0x06A3, 0x0752, 0x0B65, 0x056A, + //* 1363 -1367 */ "1010 1010 1011", "0101 0010 1011", "1100 1001 0101", "1101 0100 1010", "1101 1010 0101", + 0x0AAB, 0x052B, 0x0C95, 0x0D4A, 0x0DA5, + //* 1368 -1372 */ "0101 1100 1010", "1010 1101 0110", "1001 0101 0111", "0100 1010 1011", "1001 0100 1011", + 0x05CA, 0x0AD6, 0x0957, 0x04AB, 0x094B, + //* 1373 -1377 */ "1010 1010 0101", "1011 0101 0010", "1011 0110 1010", "0101 0111 0101", "0010 0111 0110", + 0x0AA5, 0x0B52, 0x0B6A, 0x0575, 0x0276, + //* 1378 -1382 */ "1000 1011 0111", "0100 0101 1011", "0101 0101 0101", "0101 1010 1001", "0101 1011 0100", + 0x08B7, 0x045B, 0x0555, 0x05A9, 0x05B4, + //* 1383 -1387 */ "1001 1101 1010", "0100 1101 1101", "0010 0110 1110", "1001 0011 0110", "1010 1010 1010", + 0x09DA, 0x04DD, 0x026E, 0x0936, 0x0AAA, + //* 1388 -1392 */ "1101 0101 0100", "1101 1011 0010", "0101 1101 0101", "0010 1101 1010", "1001 0101 1011", + 0x0D54, 0x0DB2, 0x05D5, 0x02DA, 0x095B, + //* 1393 -1397 */ "0100 1010 1011", "1010 0101 0101", "1011 0100 1001", "1011 0110 0100", "1011 0111 0001", + 0x04AB, 0x0A55, 0x0B49, 0x0B64, 0x0B71, + //* 1398 -1402 */ "0101 1011 0100", "1010 1011 0101", "1010 0101 0101", "1101 0010 0101", "1110 1001 0010", + 0x05B4, 0x0AB5, 0x0A55, 0x0D25, 0x0E92, + //* 1403 -1407 */ "1110 1100 1001", "0110 1101 0100", "1010 1110 1001", "1001 0110 1011", "0100 1010 1011", + 0x0EC9, 0x06D4, 0x0AE9, 0x096B, 0x04AB, + //* 1408 -1412 */ "1010 1001 0011", "1101 0100 1001", "1101 1010 0100", "1101 1011 0010", "1010 1011 1001", + 0x0A93, 0x0D49, 0x0DA4, 0x0DB2, 0x0AB9, + //* 1413 -1417 */ "0100 1011 1010", "1010 0101 1011", "0101 0010 1011", "1010 1001 0101", "1011 0010 1010", + 0x04BA, 0x0A5B, 0x052B, 0x0A95, 0x0B2A, + //* 1418 -1422 */ "1011 0101 0101", "0101 0101 1100", "0100 1011 1101", "0010 0011 1101", "1001 0001 1101", + 0x0B55, 0x055C, 0x04BD, 0x023D, 0x091D, + //* 1423 -1427 */ "1010 1001 0101", "1011 0100 1010", "1011 0101 1010", "0101 0110 1101", "0010 1011 0110", + 0x0A95, 0x0B4A, 0x0B5A, 0x056D, 0x02B6, + //* 1428 -1432 */ "1001 0011 1011", "0100 1001 1011", "0110 0101 0101", "0110 1010 1001", "0111 0101 0100", + 0x093B, 0x049B, 0x0655, 0x06A9, 0x0754, + //* 1433 -1437 */ "1011 0110 1010", "0101 0110 1100", "1010 1010 1101", "0101 0101 0101", "1011 0010 1001", + 0x0B6A, 0x056C, 0x0AAD, 0x0555, 0x0B29, + //* 1438 -1442 */ "1011 1001 0010", "1011 1010 1001", "0101 1101 0100", "1010 1101 1010", "0101 0101 1010", + 0x0B92, 0x0BA9, 0x05D4, 0x0ADA, 0x055A, + //* 1443 -1447 */ "1010 1010 1011", "0101 1001 0101", "0111 0100 1001", "0111 0110 0100", "1011 1010 1010", + 0x0AAB, 0x0595, 0x0749, 0x0764, 0x0BAA, + //* 1448 -1452 */ "0101 1011 0101", "0010 1011 0110", "1010 0101 0110", "1110 0100 1101", "1011 0010 0101", + 0x05B5, 0x02B6, 0x0A56, 0x0E4D, 0x0B25, + //* 1453 -1457 */ "1011 0101 0010", "1011 0110 1010", "0101 1010 1101", "0010 1010 1110", "1001 0010 1111", + 0x0B52, 0x0B6A, 0x05AD, 0x02AE, 0x092F, + //* 1458 -1462 */ "0100 1001 0111", "0110 0100 1011", "0110 1010 0101", "0110 1010 1100", "1010 1101 0110", + 0x0497, 0x064B, 0x06A5, 0x06AC, 0x0AD6, + //* 1463 -1467 */ "0101 0101 1101", "0100 1001 1101", "1010 0100 1101", "1101 0001 0110", "1101 1001 0101", + 0x055D, 0x049D, 0x0A4D, 0x0D16, 0x0D95, + //* 1468 -1472 */ "0101 1010 1010", "0101 1011 0101", "0010 1101 1010", "1001 0101 1011", "0100 1010 1101", + 0x05AA, 0x05B5, 0x02DA, 0x095B, 0x04AD, + //* 1473 -1477 */ "0101 1001 0101", "0110 1100 1010", "0110 1110 0100", "1010 1110 1010", "0100 1111 0101", + 0x0595, 0x06CA, 0x06E4, 0x0AEA, 0x04F5, + //* 1478 -1482 */ "0010 1011 0110", "1001 0101 0110", "1010 1010 1010", "1011 0101 0100", "1011 1101 0010", + 0x02B6, 0x0956, 0x0AAA, 0x0B54, 0x0BD2, + //* 1483 -1487 */ "0101 1101 1001", "0010 1110 1010", "1001 0110 1101", "0100 1010 1101", "1010 1001 0101", + 0x05D9, 0x02EA, 0x096D, 0x04AD, 0x0A95, + //* 1488 -1492 */ "1011 0100 1010", "1011 1010 0101", "0101 1011 0010", "1001 1011 0101", "0100 1101 0110", + 0x0B4A, 0x0BA5, 0x05B2, 0x09B5, 0x04D6, + //* 1493 -1497 */ "1010 1001 0111", "0101 0100 0111", "0110 1001 0011", "0111 0100 1001", "1011 0101 0101", + 0x0A97, 0x0547, 0x0693, 0x0749, 0x0B55, + //* 1498 -1508 */ "0101 0110 1010", "1010 0110 1011", "0101 0010 1011", "1010 1000 1011", "1101 0100 0110", "1101 1010 0011", "0101 1100 1010", "1010 1101 0110", "0100 1101 1011", "0010 0110 1011", "1001 0100 1011", + 0x056A, 0x0A6B, 0x052B, 0x0A8B, 0x0D46, 0x0DA3, 0x05CA, 0x0AD6, 0x04DB, 0x026B, 0x094B, + //* 1509 -1519 */ "1010 1010 0101", "1011 0101 0010", "1011 0110 1001", "0101 0111 0101", "0001 0111 0110", "1000 1011 0111", "0010 0101 1011", "0101 0010 1011", "0101 0110 0101", "0101 1011 0100", "1001 1101 1010", + 0x0AA5, 0x0B52, 0x0B69, 0x0575, 0x0176, 0x08B7, 0x025B, 0x052B, 0x0565, 0x05B4, 0x09DA, + //* 1520 -1530 */ "0100 1110 1101", "0001 0110 1101", "1000 1011 0110", "1010 1010 0110", "1101 0101 0010", "1101 1010 1001", "0101 1101 0100", "1010 1101 1010", "1001 0101 1011", "0100 1010 1011", "0110 0101 0011", + 0x04ED, 0x016D, 0x08B6, 0x0AA6, 0x0D52, 0x0DA9, 0x05D4, 0x0ADA, 0x095B, 0x04AB, 0x0653, + //* 1531 -1541 */ "0111 0010 1001", "0111 0110 0010", "1011 1010 1001", "0101 1011 0010", "1010 1011 0101", "0101 0101 0101", "1011 0010 0101", "1101 1001 0010", "1110 1100 1001", "0110 1101 0010", "1010 1110 1001", + 0x0729, 0x0762, 0x0BA9, 0x05B2, 0x0AB5, 0x0555, 0x0B25, 0x0D92, 0x0EC9, 0x06D2, 0x0AE9, + //* 1542 -1552 */ "0101 0110 1011", "0100 1010 1011", "1010 0101 0101", "1101 0010 1001", "1101 0101 0100", "1101 1010 1010", "1001 1011 0101", "0100 1011 1010", "1010 0011 1011", "0100 1001 1011", "1010 0100 1101", + 0x056B, 0x04AB, 0x0A55, 0x0D29, 0x0D54, 0x0DAA, 0x09B5, 0x04BA, 0x0A3B, 0x049B, 0x0A4D, + //* 1553 -1563 */ "1010 1010 1010", "1010 1101 0101", "0010 1101 1010", "1001 0101 1101", "0100 0101 1110", "1010 0010 1110", "1100 1001 1010", "1101 0101 0101", "0110 1011 0010", "0110 1011 1001", "0100 1011 1010", + 0x0AAA, 0x0AD5, 0x02DA, 0x095D, 0x045E, 0x0A2E, 0x0C9A, 0x0D55, 0x06B2, 0x06B9, 0x04BA, + //* 1564 -1574 */ "1010 0101 1101", "0101 0010 1101", "1010 1001 0101", "1011 0101 0010", "1011 1010 1000", "1011 1011 0100", "0101 1011 1001", "0010 1101 1010", "1001 0101 1010", "1011 0100 1010", "1101 1010 0100", + 0x0A5D, 0x052D, 0x0A95, 0x0B52, 0x0BA8, 0x0BB4, 0x05B9, 0x02DA, 0x095A, 0x0B4A, 0x0DA4, + //* 1575 -1585 */ "1110 1101 0001", "0110 1110 1000", "1011 0110 1010", "0101 0110 1101", "0101 0011 0101", "0110 1001 0101", "1101 0100 1010", "1101 1010 1000", "1101 1101 0100", "0110 1101 1010", "0101 0101 1011", + 0x0ED1, 0x06E8, 0x0B6A, 0x056D, 0x0535, 0x0695, 0x0D4A, 0x0DA8, 0x0DD4, 0x06DA, 0x055B, + //* 1586 -1596 */ "0010 1001 1101", "0110 0010 1011", "1011 0001 0101", "1011 0100 1010", "1011 1001 0101", "0101 1010 1010", "1010 1010 1110", "1001 0010 1110", "1100 1000 1111", "0101 0010 0111", "0110 1001 0101", + 0x029D, 0x062B, 0x0B15, 0x0B4A, 0x0B95, 0x05AA, 0x0AAE, 0x092E, 0x0C8F, 0x0527, 0x0695, + //* 1597 -1600 */ "0110 1010 1010", "1010 1101 0110", "0101 0101 1101", "0010 1001 1101", }; + 0x06AA, 0x0AD6, 0x055D, 0x029D +}; + +int32_t getUmalqura_MonthLength(int32_t y, int32_t m) { + int32_t mask = (int32_t) (0x01 << (11 - m)); // set mask for bit corresponding to month + if((UMALQURA_MONTHLENGTH[y] & mask) == 0 ) + return 29; + else + return 30; + +} + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +const char *IslamicCalendar::getType() const { + return "islamic"; +} + +IslamicCalendar* IslamicCalendar::clone() const { + return new IslamicCalendar(*this); +} + +IslamicCalendar::IslamicCalendar(const Locale& aLocale, UErrorCode& success) +: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +IslamicCalendar::~IslamicCalendar() +{ +} +//------------------------------------------------------------------------- +// Minimum / Maximum access functions +//------------------------------------------------------------------------- + +// Note: Current IslamicCalendar implementation does not work +// well with negative years. + +// TODO: In some cases the current ICU Islamic calendar implementation shows +// a month as having 31 days. Since date parsing now uses range checks based +// on the table below, we need to change the range for last day of month to +// include 31 as a workaround until the implementation is fixed. +static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 0, 0}, // ERA + { 1, 1, 5000000, 5000000}, // YEAR + { 0, 0, 11, 11}, // MONTH + { 1, 1, 50, 51}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 29, 31}, // DAY_OF_MONTH - 31 to workaround for cal implementation bug, should be 30 + { 1, 1, 354, 355}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { -1, -1, 5, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { 1, 1, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { 1, 1, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 11, 11}, // ORDINAL_MONTH +}; + +/** +* @draft ICU 2.4 +*/ +int32_t IslamicCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return LIMITS[field][limitType]; +} + +//------------------------------------------------------------------------- +// Assorted calculation utilities +// + +// we could compress this down more if we need to +static const int8_t umAlQuraYrStartEstimateFix[] = { + 0, 0, -1, 0, -1, 0, 0, 0, 0, 0, // 1300.. + -1, 0, 0, 0, 0, 0, 0, 0, -1, 0, // 1310.. + 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, // 1320.. + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // 1330.. + 0, 0, 1, 0, 0, -1, -1, 0, 0, 0, // 1340.. + 1, 0, 0, -1, 0, 0, 0, 1, 1, 0, // 1350.. + 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, // 1360.. + 0, 1, 1, 0, 0, -1, 0, 1, 0, 1, // 1370.. + 1, 0, 0, -1, 0, 1, 0, 0, 0, -1, // 1380.. + 0, 1, 0, 1, 0, 0, 0, -1, 0, 0, // 1390.. + 0, 0, -1, -1, 0, -1, 0, 1, 0, 0, // 1400.. + 0, -1, 0, 0, 0, 1, 0, 0, 0, 0, // 1410.. + 0, 1, 0, 0, -1, -1, 0, 0, 0, 1, // 1420.. + 0, 0, -1, -1, 0, -1, 0, 0, -1, -1, // 1430.. + 0, -1, 0, -1, 0, 0, -1, -1, 0, 0, // 1440.. + 0, 0, 0, 0, -1, 0, 1, 0, 1, 1, // 1450.. + 0, 0, -1, 0, 1, 0, 0, 0, 0, 0, // 1460.. + 1, 0, 1, 0, 0, 0, -1, 0, 1, 0, // 1470.. + 0, -1, -1, 0, 0, 0, 1, 0, 0, 0, // 1480.. + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, // 1490.. + 1, 0, 0, -1, 0, 0, 0, 1, 1, 0, // 1500.. + 0, -1, 0, 1, 0, 1, 1, 0, 0, 0, // 1510.. + 0, 1, 0, 0, 0, -1, 0, 0, 0, 1, // 1520.. + 0, 0, 0, -1, 0, 0, 0, 0, 0, -1, // 1530.. + 0, -1, 0, 1, 0, 0, 0, -1, 0, 1, // 1540.. + 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, // 1550.. + -1, 0, 0, 0, 0, 1, 0, 0, 0, -1, // 1560.. + 0, 0, 0, 0, -1, -1, 0, -1, 0, 1, // 1570.. + 0, 0, -1, -1, 0, 0, 1, 1, 0, 0, // 1580.. + -1, 0, 0, 0, 0, 1, 0, 0, 0, 0, // 1590.. + 1 // 1600 +}; + +/** +* Determine whether a year is a leap year in the Islamic civil calendar +*/ +UBool IslamicCalendar::civilLeapYear(int32_t year) +{ + return (14 + 11 * year) % 30 < 11; +} + +/** +* Return the day # on which the given year starts. Days are counted +* from the Hijri epoch, origin 0. +*/ +int32_t IslamicCalendar::yearStart(int32_t year) const{ + return trueMonthStart(12*(year-1)); +} + +/** +* Return the day # on which the given month starts. Days are counted +* from the Hijri epoch, origin 0. +* +* @param year The hijri year +* @param month The hijri month, 0-based (assumed to be in range 0..11) +*/ +int32_t IslamicCalendar::monthStart(int32_t year, int32_t month) const { + return trueMonthStart(12*(year-1) + month); +} + +/** +* Find the day number on which a particular month of the true/lunar +* Islamic calendar starts. +* +* @param month The month in question, origin 0 from the Hijri epoch +* +* @return The day number on which the given month starts. +*/ +int32_t IslamicCalendar::trueMonthStart(int32_t month) const +{ + UErrorCode status = U_ZERO_ERROR; + int32_t start = CalendarCache::get(&gMonthCache, month, status); + + if (start==0) { + // Make a guess at when the month started, using the average length + UDate origin = HIJRA_MILLIS + + uprv_floor(month * CalendarAstronomer::SYNODIC_MONTH) * kOneDay; + + // moonAge will fail due to memory allocation error + double age = moonAge(origin, status); + if (U_FAILURE(status)) { + goto trueMonthStartEnd; + } + + if (age >= 0) { + // The month has already started + do { + origin -= kOneDay; + age = moonAge(origin, status); + if (U_FAILURE(status)) { + goto trueMonthStartEnd; + } + } while (age >= 0); + } + else { + // Preceding month has not ended yet. + do { + origin += kOneDay; + age = moonAge(origin, status); + if (U_FAILURE(status)) { + goto trueMonthStartEnd; + } + } while (age < 0); + } + start = (int32_t)(ClockMath::floorDivide( + (int64_t)((int64_t)origin - HIJRA_MILLIS), (int64_t)kOneDay) + 1); + CalendarCache::put(&gMonthCache, month, start, status); + } +trueMonthStartEnd : + if(U_FAILURE(status)) { + start = 0; + } + return start; +} + +/** +* Return the "age" of the moon at the given time; this is the difference +* in ecliptic latitude between the moon and the sun. This method simply +* calls CalendarAstronomer.moonAge, converts to degrees, +* and adjusts the result to be in the range [-180, 180]. +* +* @param time The time at which the moon's age is desired, +* in millis since 1/1/1970. +*/ +double IslamicCalendar::moonAge(UDate time, UErrorCode &status) +{ + double age = 0; + + static UMutex astroLock; // pod bay door lock + umtx_lock(&astroLock); + if(gIslamicCalendarAstro == nullptr) { + gIslamicCalendarAstro = new CalendarAstronomer(); + if (gIslamicCalendarAstro == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return age; + } + ucln_i18n_registerCleanup(UCLN_I18N_ISLAMIC_CALENDAR, calendar_islamic_cleanup); + } + gIslamicCalendarAstro->setTime(time); + age = gIslamicCalendarAstro->getMoonAge(); + umtx_unlock(&astroLock); + + // Convert to degrees and normalize... + age = age * 180 / CalendarAstronomer::PI; + if (age > 180) { + age = age - 360; + } + + return age; +} + +//---------------------------------------------------------------------- +// Calendar framework +//---------------------------------------------------------------------- + +/** +* Return the length (in days) of the given month. +* +* @param year The hijri year +* @param year The hijri month, 0-based +* @draft ICU 2.4 +*/ +int32_t IslamicCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + month = 12*(extendedYear-1) + month; + return trueMonthStart(month+1) - trueMonthStart(month) ; +} + +/** +* Return the number of days in the given Islamic year +* @draft ICU 2.4 +*/ +int32_t IslamicCalendar::handleGetYearLength(int32_t extendedYear) const { + int32_t month = 12*(extendedYear-1); + return (trueMonthStart(month + 12) - trueMonthStart(month)); +} + +//------------------------------------------------------------------------- +// Functions for converting from field values to milliseconds.... +//------------------------------------------------------------------------- + +// Return JD of start of given month/year +// Calendar says: +// Get the Julian day of the day BEFORE the start of this year. +// If useMonth is true, get the day before the start of the month. +// Hence the -1 +/** +* @draft ICU 2.4 +*/ +int32_t IslamicCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, UBool /* useMonth */) const { + // This may be called by Calendar::handleComputeJulianDay with months out of the range + // 0..11. Need to handle that here since monthStart requires months in the range 0.11. + if (month > 11) { + eyear += (month / 12); + month %= 12; + } else if (month < 0) { + month++; + eyear += (month / 12) - 1; + month = (month % 12) + 11; + } + return monthStart(eyear, month) + getEpoc() - 1; +} + +//------------------------------------------------------------------------- +// Functions for converting from milliseconds to field values +//------------------------------------------------------------------------- + +/** +* @draft ICU 2.4 +*/ +int32_t IslamicCalendar::handleGetExtendedYear() { + int32_t year; + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + year = internalGet(UCAL_YEAR, 1); // Default to year 1 + } + return year; +} + +/** +* Override Calendar to compute several fields specific to the Islamic +* calendar system. These are: +* +*
  • ERA +*
  • YEAR +*
  • MONTH +*
  • DAY_OF_MONTH +*
  • DAY_OF_YEAR +*
  • EXTENDED_YEAR
+* +* The DAY_OF_WEEK and DOW_LOCAL fields are already set when this +* method is called. The getGregorianXxx() methods return Gregorian +* calendar equivalents for the given Julian day. +* @draft ICU 2.4 +*/ +void IslamicCalendar::handleComputeFields(int32_t julianDay, UErrorCode &status) { + if (U_FAILURE(status)) return; + int32_t days = julianDay - getEpoc(); + + // Guess at the number of elapsed full months since the epoch + int32_t month = (int32_t)uprv_floor((double)days / CalendarAstronomer::SYNODIC_MONTH); + + int32_t startDate = (int32_t)uprv_floor(month * CalendarAstronomer::SYNODIC_MONTH); + + double age = moonAge(internalGetTime(), status); + if (U_FAILURE(status)) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if ( days - startDate >= 25 && age > 0) { + // If we're near the end of the month, assume next month and search backwards + month++; + } + + // Find out the last time that the new moon was actually visible at this longitude + // This returns midnight the night that the moon was visible at sunset. + while ((startDate = trueMonthStart(month)) > days) { + // If it was after the date in question, back up a month and try again + month--; + } + + int32_t year = month >= 0 ? ((month / 12) + 1) : ((month + 1 ) / 12); + month = ((month % 12) + 12 ) % 12; + int32_t dayOfMonth = (days - monthStart(year, month)) + 1; + + // Now figure out the day of the year. + int32_t dayOfYear = (days - monthStart(year, 0)) + 1; + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_EXTENDED_YEAR, year); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); +} + +int32_t IslamicCalendar::getEpoc() const { + return CIVIL_EPOC; +} + +static int32_t gregoYearFromIslamicStart(int32_t year) { + // ad hoc conversion, improve under #10752 + // rough est for now, ok for grego 1846-2138, + // otherwise occasionally wrong (for 3% of years) + int cycle, offset, shift = 0; + if (year >= 1397) { + cycle = (year - 1397) / 67; + offset = (year - 1397) % 67; + shift = 2*cycle + ((offset >= 33)? 1: 0); + } else { + cycle = (year - 1396) / 67 - 1; + offset = -(year - 1396) % 67; + shift = 2*cycle + ((offset <= 33)? 1: 0); + } + return year + 579 - shift; +} + +int32_t IslamicCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return gregoYearFromIslamicStart(year); +} + +static int32_t firstIslamicStartYearFromGrego(int32_t year) { + // ad hoc conversion, improve under #10752 + // rough est for now, ok for grego 1846-2138, + // otherwise occasionally wrong (for 3% of years) + int cycle, offset, shift = 0; + if (year >= 1977) { + cycle = (year - 1977) / 65; + offset = (year - 1977) % 65; + shift = 2*cycle + ((offset >= 32)? 1: 0); + } else { + cycle = (year - 1976) / 65 - 1; + offset = -(year - 1976) % 65; + shift = 2*cycle + ((offset <= 32)? 1: 0); + } + return year - 579 + shift; +} + +void IslamicCalendar::setRelatedYear(int32_t year) +{ + set(UCAL_EXTENDED_YEAR, firstIslamicStartYearFromGrego(year)); +} + +/** + * The system maintains a static default century start date and Year. They are + * initialized the first time they are used. Once the system default century date + * and year are set, they do not change. + */ +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + + +UBool IslamicCalendar::haveDefaultCentury() const +{ + return true; +} + +UDate IslamicCalendar::defaultCenturyStart() const +{ + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t IslamicCalendar::defaultCenturyStartYear() const +{ + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + +bool +IslamicCalendar::inTemporalLeapYear(UErrorCode &status) const +{ + int32_t days = getActualMaximum(UCAL_DAY_OF_YEAR, status); + if (U_FAILURE(status)) return false; + return days == 355; +} + + +U_CFUNC void U_CALLCONV +IslamicCalendar::initializeSystemDefaultCentury() +{ + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + IslamicCalendar calendar(Locale("@calendar=islamic-civil"),status); + if (U_SUCCESS(status)) { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +/***************************************************************************** + * IslamicCivilCalendar + *****************************************************************************/ +IslamicCivilCalendar::IslamicCivilCalendar(const Locale& aLocale, UErrorCode& success) + : IslamicCalendar(aLocale, success) +{ +} + +IslamicCivilCalendar::~IslamicCivilCalendar() +{ +} + +const char *IslamicCivilCalendar::getType() const { + return "islamic-civil"; +} + +IslamicCivilCalendar* IslamicCivilCalendar::clone() const { + return new IslamicCivilCalendar(*this); +} + +/** +* Return the day # on which the given year starts. Days are counted +* from the Hijri epoch, origin 0. +*/ +int32_t IslamicCivilCalendar::yearStart(int32_t year) const{ + return static_cast( + (year-1)*354 + ClockMath::floorDivide((3+11*static_cast(year)), + static_cast(30))); +} + +/** +* Return the day # on which the given month starts. Days are counted +* from the Hijri epoch, origin 0. +* +* @param year The hijri year +* @param month The hijri month, 0-based (assumed to be in range 0..11) +*/ +int32_t IslamicCivilCalendar::monthStart(int32_t year, int32_t month) const { + // This does not handle months out of the range 0..11 + return static_cast( + uprv_ceil(29.5*month) + (year-1)*354 + + static_cast(ClockMath::floorDivide( + 3+11*static_cast(year), + static_cast(30)))); +} + +/** +* Return the length (in days) of the given month. +* +* @param year The hijri year +* @param year The hijri month, 0-based +* @draft ICU 2.4 +*/ +int32_t IslamicCivilCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + int32_t length = 29 + (month+1) % 2; + if (month == DHU_AL_HIJJAH && civilLeapYear(extendedYear)) { + length++; + } + return length; +} + +/** +* Return the number of days in the given Islamic year +* @draft ICU 2.4 +*/ +int32_t IslamicCivilCalendar::handleGetYearLength(int32_t extendedYear) const { + return 354 + (civilLeapYear(extendedYear) ? 1 : 0); +} + +/** +* Override Calendar to compute several fields specific to the Islamic +* calendar system. These are: +* +*
  • ERA +*
  • YEAR +*
  • MONTH +*
  • DAY_OF_MONTH +*
  • DAY_OF_YEAR +*
  • EXTENDED_YEAR
+* +* The DAY_OF_WEEK and DOW_LOCAL fields are already set when this +* method is called. The getGregorianXxx() methods return Gregorian +* calendar equivalents for the given Julian day. +* @draft ICU 2.4 +*/ +void IslamicCivilCalendar::handleComputeFields(int32_t julianDay, UErrorCode &status) { + if (U_FAILURE(status)) return; + int32_t days = julianDay - getEpoc(); + + // Use the civil calendar approximation, which is just arithmetic + int32_t year = static_cast( + ClockMath::floorDivide(30 * static_cast(days) + 10646, + static_cast(10631))); + int32_t month = static_cast( + uprv_ceil((days - 29 - yearStart(year)) / 29.5 )); + month = month<11?month:11; + + int32_t dayOfMonth = (days - monthStart(year, month)) + 1; + + // Now figure out the day of the year. + int32_t dayOfYear = (days - monthStart(year, 0)) + 1; + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_EXTENDED_YEAR, year); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); +} +/***************************************************************************** + * IslamicTBLACalendar + *****************************************************************************/ +IslamicTBLACalendar::IslamicTBLACalendar(const Locale& aLocale, UErrorCode& success) + : IslamicCivilCalendar(aLocale, success) +{ +} + +IslamicTBLACalendar::~IslamicTBLACalendar() +{ +} + +const char *IslamicTBLACalendar::getType() const { + return "islamic-tbla"; +} + +IslamicTBLACalendar* IslamicTBLACalendar::clone() const { + return new IslamicTBLACalendar(*this); +} + +int32_t IslamicTBLACalendar::getEpoc() const { + return ASTRONOMICAL_EPOC; +} + +/***************************************************************************** + * IslamicUmalquraCalendar + *****************************************************************************/ +IslamicUmalquraCalendar::IslamicUmalquraCalendar(const Locale& aLocale, UErrorCode& success) + : IslamicCalendar(aLocale, success) +{ +} + +IslamicUmalquraCalendar::~IslamicUmalquraCalendar() +{ +} + +const char *IslamicUmalquraCalendar::getType() const { + return "islamic-umalqura"; +} + +IslamicUmalquraCalendar* IslamicUmalquraCalendar::clone() const { + return new IslamicUmalquraCalendar(*this); +} + +/** +* Return the day # on which the given year starts. Days are counted +* from the Hijri epoch, origin 0. +*/ +int32_t IslamicUmalquraCalendar::yearStart(int32_t year) const { + if (year < UMALQURA_YEAR_START || year > UMALQURA_YEAR_END) { + return static_cast( + (year-1)*354 + ClockMath::floorDivide((3+11*static_cast(year)), + static_cast(30))); + } + year -= UMALQURA_YEAR_START; + // rounded least-squares fit of the dates previously calculated from UMALQURA_MONTHLENGTH iteration + int32_t yrStartLinearEstimate = static_cast( + (354.36720 * (double)year) + 460322.05 + 0.5); + // need a slight correction to some + return yrStartLinearEstimate + umAlQuraYrStartEstimateFix[year]; +} + +/** +* Return the day # on which the given month starts. Days are counted +* from the Hijri epoch, origin 0. +* +* @param year The hijri year +* @param month The hijri month, 0-based (assumed to be in range 0..11) +*/ +int32_t IslamicUmalquraCalendar::monthStart(int32_t year, int32_t month) const { + int32_t ms = yearStart(year); + for(int i=0; i< month; i++){ + ms+= handleGetMonthLength(year, i); + } + return ms; +} + +/** +* Return the length (in days) of the given month. +* +* @param year The hijri year +* @param year The hijri month, 0-based +*/ +int32_t IslamicUmalquraCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + int32_t length = 0; + if (extendedYearUMALQURA_YEAR_END) { + length = 29 + (month+1) % 2; + if (month == DHU_AL_HIJJAH && civilLeapYear(extendedYear)) { + length++; + } + return length; + } + return getUmalqura_MonthLength(extendedYear - UMALQURA_YEAR_START, month); +} + +/** +* Return the number of days in the given Islamic year +* @draft ICU 2.4 +*/ +int32_t IslamicUmalquraCalendar::handleGetYearLength(int32_t extendedYear) const { + if (extendedYearUMALQURA_YEAR_END) { + return 354 + (civilLeapYear(extendedYear) ? 1 : 0); + } + int len = 0; + for(int i=0; i<12; i++) { + len += handleGetMonthLength(extendedYear, i); + } + return len; +} + +/** +* Override Calendar to compute several fields specific to the Islamic +* calendar system. These are: +* +*
  • ERA +*
  • YEAR +*
  • MONTH +*
  • DAY_OF_MONTH +*
  • DAY_OF_YEAR +*
  • EXTENDED_YEAR
+* +* The DAY_OF_WEEK and DOW_LOCAL fields are already set when this +* method is called. The getGregorianXxx() methods return Gregorian +* calendar equivalents for the given Julian day. +* @draft ICU 2.4 +*/ +void IslamicUmalquraCalendar::handleComputeFields(int32_t julianDay, UErrorCode &status) { + if (U_FAILURE(status)) return; + int32_t year, month, dayOfMonth, dayOfYear; + int32_t days = julianDay - getEpoc(); + + int32_t umalquraStartdays = yearStart(UMALQURA_YEAR_START) ; + if (days < umalquraStartdays) { + //Use Civil calculation + year = (int32_t)ClockMath::floorDivide( + (30 * (int64_t)days + 10646) , (int64_t)10631.0 ); + month = (int32_t)uprv_ceil((days - 29 - yearStart(year)) / 29.5 ); + month = month < 11 ? month : 11; + } else { + int y =UMALQURA_YEAR_START-1, m =0; + long d = 1; + while (d > 0) { + y++; + d = days - yearStart(y) +1; + if (d == handleGetYearLength(y)) { + m=11; + break; + } + if (d < handleGetYearLength(y)){ + int monthLen = handleGetMonthLength(y, m); + m=0; + while(d > monthLen){ + d -= monthLen; + m++; + monthLen = handleGetMonthLength(y, m); + } + break; + } + } + year = y; + month = m; + } + + dayOfMonth = (days - monthStart(year, month)) + 1; + + // Now figure out the day of the year. + dayOfYear = (days - monthStart(year, 0)) + 1; + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_EXTENDED_YEAR, year); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); +} +/***************************************************************************** + * IslamicRGSACalendar + *****************************************************************************/ +IslamicRGSACalendar::IslamicRGSACalendar(const Locale& aLocale, UErrorCode& success) + : IslamicCalendar(aLocale, success) +{ +} + +IslamicRGSACalendar::~IslamicRGSACalendar() +{ +} + +const char *IslamicRGSACalendar::getType() const { + return "islamic-rgsa"; +} + +IslamicRGSACalendar* IslamicRGSACalendar::clone() const { + return new IslamicRGSACalendar(*this); +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IslamicCalendar) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IslamicCivilCalendar) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IslamicUmalquraCalendar) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IslamicTBLACalendar) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IslamicRGSACalendar) + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/islamcal.h b/intl/icu/source/i18n/islamcal.h new file mode 100644 index 0000000000..8469269bcc --- /dev/null +++ b/intl/icu/source/i18n/islamcal.h @@ -0,0 +1,763 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ******************************************************************************** + * Copyright (C) 2003-2013, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File ISLAMCAL.H + * + * Modification History: + * + * Date Name Description + * 10/14/2003 srl ported from java IslamicCalendar + ***************************************************************************** + */ + +#ifndef ISLAMCAL_H +#define ISLAMCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" + +U_NAMESPACE_BEGIN + +/** + * IslamicCalendar is a subclass of Calendar + * that implements the Islamic civil and religious calendars. It + * is used as the civil calendar in most of the Arab world and the + * liturgical calendar of the Islamic faith worldwide. This calendar + * is also known as the "Hijri" calendar, since it starts at the time + * of Mohammed's emigration (or "hijra") to Medinah on Thursday, + * July 15, 622 AD (Julian). + *

+ * The Islamic calendar is strictly lunar, and thus an Islamic year of twelve + * lunar months does not correspond to the solar year used by most other + * calendar systems, including the Gregorian. An Islamic year is, on average, + * about 354 days long, so each successive Islamic year starts about 11 days + * earlier in the corresponding Gregorian year. + *

+ * Each month of the calendar starts when the new moon's crescent is visible + * at sunset. However, in order to keep the time fields in this class + * synchronized with those of the other calendars and with local clock time, + * we treat days and months as beginning at midnight, + * roughly 6 hours after the corresponding sunset. + *

+ * There are two main variants of the Islamic calendar in existence. The first + * is the civil calendar, which uses a fixed cycle of alternating 29- + * and 30-day months, with a leap day added to the last month of 11 out of + * every 30 years. This calendar is easily calculated and thus predictable in + * advance, so it is used as the civil calendar in a number of Arab countries. + * This is the default behavior of a newly-created IslamicCalendar + * object. This calendar variant is implemented in the IslamicCivilCalendar + * class. + *

+ * The Islamic religious calendar, however, is based on the observation + * of the crescent moon. It is thus affected by the position at which the + * observations are made, seasonal variations in the time of sunset, the + * eccentricities of the moon's orbit, and even the weather at the observation + * site. This makes it impossible to calculate in advance, and it causes the + * start of a month in the religious calendar to differ from the civil calendar + * by up to three days. + *

+ * Using astronomical calculations for the position of the sun and moon, the + * moon's illumination, and other factors, it is possible to determine the start + * of a lunar month with a fairly high degree of certainty. However, these + * calculations are extremely complicated and thus slow, so most algorithms, + * including the one used here, are only approximations of the true astronomical + * calculations. At present, the approximations used in this class are fairly + * simplistic; they will be improved in later versions of the code. + *

+ * + * @see GregorianCalendar + * + * @author Laura Werner + * @author Alan Liu + * @author Steven R. Loomis + * @internal + */ +class U_I18N_API IslamicCalendar : public Calendar { + public: + //------------------------------------------------------------------------- + // Constants... + //------------------------------------------------------------------------- + /** + * Constants for the months + * @internal + */ + enum EMonths { + /** + * Constant for Muharram, the 1st month of the Islamic year. + * @internal + */ + MUHARRAM = 0, + + /** + * Constant for Safar, the 2nd month of the Islamic year. + * @internal + */ + SAFAR = 1, + + /** + * Constant for Rabi' al-awwal (or Rabi' I), the 3rd month of the Islamic year. + * @internal + */ + RABI_1 = 2, + + /** + * Constant for Rabi' al-thani or (Rabi' II), the 4th month of the Islamic year. + * @internal + */ + RABI_2 = 3, + + /** + * Constant for Jumada al-awwal or (Jumada I), the 5th month of the Islamic year. + * @internal + */ + JUMADA_1 = 4, + + /** + * Constant for Jumada al-thani or (Jumada II), the 6th month of the Islamic year. + * @internal + */ + JUMADA_2 = 5, + + /** + * Constant for Rajab, the 7th month of the Islamic year. + * @internal + */ + RAJAB = 6, + + /** + * Constant for Sha'ban, the 8th month of the Islamic year. + * @internal + */ + SHABAN = 7, + + /** + * Constant for Ramadan, the 9th month of the Islamic year. + * @internal + */ + RAMADAN = 8, + + /** + * Constant for Shawwal, the 10th month of the Islamic year. + * @internal + */ + SHAWWAL = 9, + + /** + * Constant for Dhu al-Qi'dah, the 11th month of the Islamic year. + * @internal + */ + DHU_AL_QIDAH = 10, + + /** + * Constant for Dhu al-Hijjah, the 12th month of the Islamic year. + * @internal + */ + DHU_AL_HIJJAH = 11, + + ISLAMIC_MONTH_MAX + }; + + + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs an IslamicCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IslamicCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + IslamicCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IslamicCalendar(const IslamicCalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~IslamicCalendar(); + + // clone + virtual IslamicCalendar* clone() const override; + + protected: + /** + * Determine whether a year is a leap year in the Islamic civil calendar + */ + static UBool civilLeapYear(int32_t year); + + /** + * Return the day # on which the given year starts. Days are counted + * from the Hijri epoch, origin 0. + */ + virtual int32_t yearStart(int32_t year) const; + + /** + * Return the day # on which the given month starts. Days are counted + * from the Hijri epoch, origin 0. + * + * @param year The hijri year + * @param year The hijri month, 0-based + */ + virtual int32_t monthStart(int32_t year, int32_t month) const; + + /** + * Find the day number on which a particular month of the true/lunar + * Islamic calendar starts. + * + * @param month The month in question, origin 0 from the Hijri epoch + * + * @return The day number on which the given month starts. + */ + int32_t trueMonthStart(int32_t month) const; + + private: + /** + * Return the "age" of the moon at the given time; this is the difference + * in ecliptic latitude between the moon and the sun. This method simply + * calls CalendarAstronomer.moonAge, converts to degrees, + * and adjusts the resultto be in the range [-180, 180]. + * + * @param time The time at which the moon's age is desired, + * in millis since 1/1/1970. + */ + static double moonAge(UDate time, UErrorCode &status); + + //---------------------------------------------------------------------- + // Calendar framework + //---------------------------------------------------------------------- + protected: + /** + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Return the length (in days) of the given month. + * + * @param year The hijri year + * @param year The hijri month, 0-based + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given Islamic year + * @internal + */ + virtual int32_t handleGetYearLength(int32_t extendedYear) const override; + + //------------------------------------------------------------------------- + // Functions for converting from field values to milliseconds.... + //------------------------------------------------------------------------- + + // Return JD of start of given month/year + /** + * @internal + */ + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const override; + + //------------------------------------------------------------------------- + // Functions for converting from milliseconds to field values + //------------------------------------------------------------------------- + + /** + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Override Calendar to compute several fields specific to the Islamic + * calendar system. These are: + * + *

  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + /** + * Return the epoc. + * @internal + */ + virtual int32_t getEpoc() const; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + /*U_I18N_API*/ static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "islamic". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + /** + * Returns true if the date is in a leap year. + * + * @param status ICU Error Code + * @return True if the date in the fields is in a Temporal proposal + * defined leap year. False otherwise. + */ + virtual bool inTemporalLeapYear(UErrorCode &status) const override; + + private: + IslamicCalendar() = delete; // default constructor not implemented + + // Default century. + protected: + /** + * Returns true because the Islamic Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + + private: + /** + * Initializes the 100-year window that dates with 2-digit years + * are considered to fall within so that its start date is 80 years + * before the current time. + */ + static void U_CALLCONV initializeSystemDefaultCentury(); +}; + +/* + * IslamicCivilCalendar is one of the two main variants of the Islamic calendar. + * The civil calendar, which uses a fixed cycle of alternating 29- + * and 30-day months, with a leap day added to the last month of 11 out of + * every 30 years. This calendar is easily calculated and thus predictable in + * advance, so it is used as the civil calendar in a number of Arab countries. + * This calendar is referring as "Islamic calendar, tabular (intercalary years + * [2,5,7,10,13,16,18,21,24,26,29]- civil epoch" in CLDR. + */ +class U_I18N_API IslamicCivilCalendar : public IslamicCalendar { + public: + /** + * Constructs an IslamicCivilCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IslamicCivilCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + IslamicCivilCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IslamicCivilCalendar(const IslamicCivilCalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~IslamicCivilCalendar(); + + // clone + virtual IslamicCivilCalendar* clone() const override; + + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "islamic-civil". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + protected: + /** + * Return the day # on which the given year starts. Days are counted + * from the Hijri epoch, origin 0. + * @internal + */ + virtual int32_t yearStart(int32_t year) const override; + + /** + * Return the day # on which the given month starts. Days are counted + * from the Hijri epoch, origin 0. + * + * @param year The hijri year + * @param year The hijri month, 0-based + * @internal + */ + virtual int32_t monthStart(int32_t year, int32_t month) const override; + + /** + * Return the length (in days) of the given month. + * + * @param year The hijri year + * @param year The hijri month, 0-based + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given Islamic year + * @internal + */ + virtual int32_t handleGetYearLength(int32_t extendedYear) const override; + + /** + * Override Calendar to compute several fields specific to the Islamic + * calendar system. These are: + * + *
  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; +}; + +/* + * IslamicTBLACalendar calendar. + * This is a subclass of IslamicCivilCalendar. The only differences in the + * calendar math is it uses different epoch. + * This calendar is referring as "Islamic calendar, tabular (intercalary years + * [2,5,7,10,13,16,18,21,24,26,29] - astronomical epoch" in CLDR. + */ +class U_I18N_API IslamicTBLACalendar : public IslamicCivilCalendar { + public: + /** + * Constructs an IslamicTBLACalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IslamicTBLACalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + IslamicTBLACalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IslamicTBLACalendar(const IslamicTBLACalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~IslamicTBLACalendar(); + + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "islamic-tbla". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + // clone + virtual IslamicTBLACalendar* clone() const override; + + protected: + /** + * Return the epoc. + * @internal + */ + virtual int32_t getEpoc() const override; +}; + +/* + * IslamicUmalquraCalendar + * This calendar is referred as "Islamic calendar, Umm al-Qura" in CLDR. + */ +class U_I18N_API IslamicUmalquraCalendar : public IslamicCalendar { + public: + /** + * Constructs an IslamicUmalquraCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IslamicUmalquraCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + IslamicUmalquraCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IslamicUmalquraCalendar(const IslamicUmalquraCalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~IslamicUmalquraCalendar(); + + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "islamic-umalqura". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + // clone + virtual IslamicUmalquraCalendar* clone() const override; + + protected: + /** + * Return the day # on which the given year starts. Days are counted + * from the Hijri epoch, origin 0. + * @internal + */ + virtual int32_t yearStart(int32_t year) const override; + + /** + * Return the day # on which the given month starts. Days are counted + * from the Hijri epoch, origin 0. + * + * @param year The hijri year + * @param year The hijri month, 0-based + * @internal + */ + virtual int32_t monthStart(int32_t year, int32_t month) const override; + + /** + * Return the length (in days) of the given month. + * + * @param year The hijri year + * @param year The hijri month, 0-based + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given Islamic year + * @internal + */ + virtual int32_t handleGetYearLength(int32_t extendedYear) const override; + + /** + * Override Calendar to compute several fields specific to the Islamic + * calendar system. These are: + * + *
  • ERA + *
  • YEAR + *
  • MONTH + *
  • DAY_OF_MONTH + *
  • DAY_OF_YEAR + *
  • EXTENDED_YEAR
+ * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; +}; + + +/* + * IslamicRGSACalendar + * Islamic calendar, Saudi Arabia sighting. Since the calendar depends on the + * sighting, it is impossible to implement by algorithm ahead of time. It is + * currently identical to IslamicCalendar except the getType will return + * "islamic-rgsa". + */ +class U_I18N_API IslamicRGSACalendar : public IslamicCalendar { + public: + /** + * Constructs an IslamicRGSACalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of IslamicRGSACalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + IslamicRGSACalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + IslamicRGSACalendar(const IslamicRGSACalendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~IslamicRGSACalendar(); + + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "islamic-rgsa". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + // clone + virtual IslamicRGSACalendar* clone() const override; +}; + +U_NAMESPACE_END + +#endif +#endif diff --git a/intl/icu/source/i18n/iso8601cal.cpp b/intl/icu/source/i18n/iso8601cal.cpp new file mode 100644 index 0000000000..1bc81fac15 --- /dev/null +++ b/intl/icu/source/i18n/iso8601cal.cpp @@ -0,0 +1,37 @@ +// © 2022 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "iso8601cal.h" +#include "unicode/gregocal.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ISO8601Calendar) + +ISO8601Calendar::ISO8601Calendar(const Locale& aLocale, UErrorCode& success) +: GregorianCalendar(aLocale, success) +{ + setFirstDayOfWeek(UCAL_MONDAY); + setMinimalDaysInFirstWeek(4); +} + +ISO8601Calendar::~ISO8601Calendar() +{ +} + +ISO8601Calendar* ISO8601Calendar::clone() const +{ + return new ISO8601Calendar(*this); +} + +const char *ISO8601Calendar::getType() const +{ + return "iso8601"; +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/iso8601cal.h b/intl/icu/source/i18n/iso8601cal.h new file mode 100644 index 0000000000..688fac3588 --- /dev/null +++ b/intl/icu/source/i18n/iso8601cal.h @@ -0,0 +1,102 @@ +// © 2022 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +#ifndef ISO8601CAL_H +#define ISO8601CAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/gregocal.h" +#include "unicode/timezone.h" + +U_NAMESPACE_BEGIN + +/** + * Concrete class which provides the ISO8601 calendar. + *

+ * ISO8601Calendar is a subclass of GregorianCalendar + * that the first day of a week is Monday and the minimal days in the first + * week of a year or month is four days. + *

+ * The ISO8601 calendar is identical to the Gregorian calendar in all respects + * except for the first day of week and the minimal days in the first week + * of a year. + * @internal + */ +class ISO8601Calendar : public GregorianCalendar { + public: + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs a DangiCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of ISO8601Calendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + ISO8601Calendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + ISO8601Calendar(const ISO8601Calendar& other) = default; + + /** + * Destructor. + * @internal + */ + virtual ~ISO8601Calendar(); + + /** + * Clone. + * @internal + */ + virtual ISO8601Calendar* clone() const override; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "iso8601". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + + private: + + ISO8601Calendar(); // default constructor not implemented +}; + +U_NAMESPACE_END + +#endif +#endif diff --git a/intl/icu/source/i18n/japancal.cpp b/intl/icu/source/i18n/japancal.cpp new file mode 100644 index 0000000000..fc18d6c0eb --- /dev/null +++ b/intl/icu/source/i18n/japancal.cpp @@ -0,0 +1,309 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2003-2009,2012,2016 International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File JAPANCAL.CPP +* +* Modification History: +* 05/16/2003 srl copied from buddhcal.cpp +* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#if U_PLATFORM_HAS_WINUWP_API == 0 +#include // getenv() is not available in UWP env +#else +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +# define VC_EXTRALEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +#include +#endif +#include "cmemory.h" +#include "erarules.h" +#include "japancal.h" +#include "unicode/gregocal.h" +#include "umutex.h" +#include "uassert.h" +#include "ucln_in.h" +#include "cstring.h" + +static icu::EraRules * gJapaneseEraRules = nullptr; +static icu::UInitOnce gJapaneseEraRulesInitOnce {}; +static int32_t gCurrentEra = 0; + +U_CDECL_BEGIN +static UBool japanese_calendar_cleanup() { + if (gJapaneseEraRules) { + delete gJapaneseEraRules; + gJapaneseEraRules = nullptr; + } + gCurrentEra = 0; + gJapaneseEraRulesInitOnce.reset(); + return true; +} +U_CDECL_END + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(JapaneseCalendar) + +static const int32_t kGregorianEpoch = 1970; // used as the default value of EXTENDED_YEAR +static const char* TENTATIVE_ERA_VAR_NAME = "ICU_ENABLE_TENTATIVE_ERA"; + + +// Export the following for use by test code. +UBool JapaneseCalendar::enableTentativeEra() { + // Although start date of next Japanese era is planned ahead, a name of + // new era might not be available. This implementation allows tester to + // check a new era without era names by settings below (in priority order). + // By default, such tentative era is disabled. + + // 1. Environment variable ICU_ENABLE_TENTATIVE_ERA=true or false + + UBool includeTentativeEra = false; + +#if U_PLATFORM_HAS_WINUWP_API == 1 + // UWP doesn't allow access to getenv(), but we can call GetEnvironmentVariableW to do the same thing. + char16_t varName[26] = {}; + u_charsToUChars(TENTATIVE_ERA_VAR_NAME, varName, static_cast(uprv_strlen(TENTATIVE_ERA_VAR_NAME))); + WCHAR varValue[5] = {}; + DWORD ret = GetEnvironmentVariableW(reinterpret_cast(varName), varValue, UPRV_LENGTHOF(varValue)); + if ((ret == 4) && (_wcsicmp(varValue, L"true") == 0)) { + includeTentativeEra = true; + } +#else + char *envVarVal = getenv(TENTATIVE_ERA_VAR_NAME); + if (envVarVal != nullptr && uprv_stricmp(envVarVal, "true") == 0) { + includeTentativeEra = true; + } +#endif + return includeTentativeEra; +} + + +// Initialize global Japanese era data +static void U_CALLCONV initializeEras(UErrorCode &status) { + gJapaneseEraRules = EraRules::createInstance("japanese", JapaneseCalendar::enableTentativeEra(), status); + if (U_FAILURE(status)) { + return; + } + gCurrentEra = gJapaneseEraRules->getCurrentEraIndex(); +} + +static void init(UErrorCode &status) { + umtx_initOnce(gJapaneseEraRulesInitOnce, &initializeEras, status); + ucln_i18n_registerCleanup(UCLN_I18N_JAPANESE_CALENDAR, japanese_calendar_cleanup); +} + +/* Some platforms don't like to export constants, like old Palm OS and some z/OS configurations. */ +uint32_t JapaneseCalendar::getCurrentEra() { + return gCurrentEra; +} + +JapaneseCalendar::JapaneseCalendar(const Locale& aLocale, UErrorCode& success) +: GregorianCalendar(aLocale, success) +{ + init(success); + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +JapaneseCalendar::~JapaneseCalendar() +{ +} + +JapaneseCalendar::JapaneseCalendar(const JapaneseCalendar& source) +: GregorianCalendar(source) +{ + UErrorCode status = U_ZERO_ERROR; + init(status); + U_ASSERT(U_SUCCESS(status)); +} + +JapaneseCalendar& JapaneseCalendar::operator= ( const JapaneseCalendar& right) +{ + GregorianCalendar::operator=(right); + return *this; +} + +JapaneseCalendar* JapaneseCalendar::clone() const +{ + return new JapaneseCalendar(*this); +} + +const char *JapaneseCalendar::getType() const +{ + return "japanese"; +} + +int32_t JapaneseCalendar::getDefaultMonthInYear(int32_t eyear) +{ + int32_t era = internalGetEra(); + // TODO do we assume we can trust 'era'? What if it is denormalized? + + int32_t month = 0; + + // Find out if we are at the edge of an era + int32_t eraStart[3] = { 0,0,0 }; + UErrorCode status = U_ZERO_ERROR; + gJapaneseEraRules->getStartDate(era, eraStart, status); + U_ASSERT(U_SUCCESS(status)); + if(eyear == eraStart[0]) { + // Yes, we're in the first year of this era. + return eraStart[1] // month + -1; // return 0-based month + } + + return month; +} + +int32_t JapaneseCalendar::getDefaultDayInMonth(int32_t eyear, int32_t month) +{ + int32_t era = internalGetEra(); + int32_t day = 1; + + int32_t eraStart[3] = { 0,0,0 }; + UErrorCode status = U_ZERO_ERROR; + gJapaneseEraRules->getStartDate(era, eraStart, status); + U_ASSERT(U_SUCCESS(status)); + if(eyear == eraStart[0]) { + if(month == eraStart[1] - 1) { + return eraStart[2]; + } + } + + return day; +} + + +int32_t JapaneseCalendar::internalGetEra() const +{ + return internalGet(UCAL_ERA, gCurrentEra); +} + +int32_t JapaneseCalendar::handleGetExtendedYear() +{ + // EXTENDED_YEAR in JapaneseCalendar is a Gregorian year + // The default value of EXTENDED_YEAR is 1970 (Showa 45) + int32_t year; + + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR && + newerField(UCAL_EXTENDED_YEAR, UCAL_ERA) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, kGregorianEpoch); + } else { + UErrorCode status = U_ZERO_ERROR; + int32_t eraStartYear = gJapaneseEraRules->getStartYear(internalGet(UCAL_ERA, gCurrentEra), status); + U_ASSERT(U_SUCCESS(status)); + + // extended year is a gregorian year, where 1 = 1AD, 0 = 1BC, -1 = 2BC, etc + year = internalGet(UCAL_YEAR, 1) // pin to minimum of year 1 (first year) + + eraStartYear // add gregorian starting year + - 1; // Subtract one because year starts at 1 + } + return year; +} + + +void JapaneseCalendar::handleComputeFields(int32_t julianDay, UErrorCode& status) +{ + //Calendar::timeToFields(theTime, quick, status); + GregorianCalendar::handleComputeFields(julianDay, status); + int32_t year = internalGet(UCAL_EXTENDED_YEAR); // Gregorian year + int32_t eraIdx = gJapaneseEraRules->getEraIndex(year, internalGetMonth() + 1, internalGet(UCAL_DAY_OF_MONTH), status); + + internalSet(UCAL_ERA, eraIdx); + internalSet(UCAL_YEAR, year - gJapaneseEraRules->getStartYear(eraIdx, status) + 1); +} + +/* +Disable pivoting +*/ +UBool JapaneseCalendar::haveDefaultCentury() const +{ + return false; +} + +UDate JapaneseCalendar::defaultCenturyStart() const +{ + return 0;// WRONG +} + +int32_t JapaneseCalendar::defaultCenturyStartYear() const +{ + return 0; +} + +int32_t JapaneseCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const +{ + switch(field) { + case UCAL_ERA: + if (limitType == UCAL_LIMIT_MINIMUM || limitType == UCAL_LIMIT_GREATEST_MINIMUM) { + return 0; + } + return gJapaneseEraRules->getNumberOfEras() - 1; // max known era, not gCurrentEra + case UCAL_YEAR: + { + switch (limitType) { + case UCAL_LIMIT_MINIMUM: + case UCAL_LIMIT_GREATEST_MINIMUM: + return 1; + case UCAL_LIMIT_LEAST_MAXIMUM: + return 1; + case UCAL_LIMIT_COUNT: //added to avoid warning + case UCAL_LIMIT_MAXIMUM: + { + UErrorCode status = U_ZERO_ERROR; + int32_t eraStartYear = gJapaneseEraRules->getStartYear(gCurrentEra, status); + U_ASSERT(U_SUCCESS(status)); + return GregorianCalendar::handleGetLimit(UCAL_YEAR, UCAL_LIMIT_MAXIMUM) - eraStartYear; + } + default: + return 1; // Error condition, invalid limitType + } + } + default: + return GregorianCalendar::handleGetLimit(field,limitType); + } +} + +int32_t JapaneseCalendar::getActualMaximum(UCalendarDateFields field, UErrorCode& status) const { + if (field == UCAL_YEAR) { + int32_t era = get(UCAL_ERA, status); + if (U_FAILURE(status)) { + return 0; // error case... any value + } + if (era == gJapaneseEraRules->getNumberOfEras() - 1) { // max known era, not gCurrentEra + // TODO: Investigate what value should be used here - revisit after 4.0. + return handleGetLimit(UCAL_YEAR, UCAL_LIMIT_MAXIMUM); + } else { + int32_t nextEraStart[3] = { 0,0,0 }; + gJapaneseEraRules->getStartDate(era + 1, nextEraStart, status); + int32_t nextEraYear = nextEraStart[0]; + int32_t nextEraMonth = nextEraStart[1]; // 1-base + int32_t nextEraDate = nextEraStart[2]; + + int32_t eraStartYear = gJapaneseEraRules->getStartYear(era, status); + int32_t maxYear = nextEraYear - eraStartYear + 1; // 1-base + if (nextEraMonth == 1 && nextEraDate == 1) { + // Subtract 1, because the next era starts at Jan 1 + maxYear--; + } + return maxYear; + } + } + return GregorianCalendar::getActualMaximum(field, status); +} + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/japancal.h b/intl/icu/source/i18n/japancal.h new file mode 100644 index 0000000000..3ae4900a2c --- /dev/null +++ b/intl/icu/source/i18n/japancal.h @@ -0,0 +1,234 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ******************************************************************************** + * Copyright (C) 2003-2008, International Business Machines Corporation + * and others. All Rights Reserved. + ******************************************************************************** + * + * File JAPANCAL.H + * + * Modification History: + * + * Date Name Description + * 05/13/2003 srl copied from gregocal.h + ******************************************************************************** + */ + +#ifndef JAPANCAL_H +#define JAPANCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" +#include "unicode/gregocal.h" + +U_NAMESPACE_BEGIN + +/** + * Concrete class which provides the Japanese calendar. + *

+ * JapaneseCalendar is a subclass of GregorianCalendar + * that numbers years and eras based on the reigns of the Japanese emperors. + * The Japanese calendar is identical to the Gregorian calendar in all respects + * except for the year and era. The ascension of each emperor to the throne + * begins a new era, and the years of that era are numbered starting with the + * year of ascension as year 1. + *

+ * Note that in the year of an imperial ascension, there are two possible sets + * of year and era values: that for the old era and for the new. For example, a + * new era began on January 7, 1989 AD. Strictly speaking, the first six days + * of that year were in the Showa era, e.g. "January 6, 64 Showa", while the rest + * of the year was in the Heisei era, e.g. "January 7, 1 Heisei". This class + * handles this distinction correctly when computing dates. However, in lenient + * mode either form of date is acceptable as input. + *

+ * In modern times, eras have started on January 8, 1868 AD, Gregorian (Meiji), + * July 30, 1912 (Taisho), December 25, 1926 (Showa), and January 7, 1989 (Heisei). Constants + * for these eras, suitable for use in the UCAL_ERA field, are provided + * in this class. Note that the number used for each era is more or + * less arbitrary. Currently, the era starting in 645 AD is era #0; however this + * may change in the future. Use the predefined constants rather than using actual, + * absolute numbers. + *

+ * Since ICU4C 63, start date of each era is imported from CLDR. CLDR era data + * may contain tentative era in near future with placeholder names. By default, + * such era data is not enabled. ICU4C users who want to test the behavior of + * the future era can enable this one of following settings (in the priority + * order): + *

    + *
  1. Environment variable ICU_ENABLE_TENTATIVE_ERA=true.
    2. + * + * @internal + */ +class JapaneseCalendar : public GregorianCalendar { +public: + + /** + * Check environment variable. + * @internal + */ + U_I18N_API static UBool U_EXPORT2 enableTentativeEra(); + + /** + * Useful constants for JapaneseCalendar. + * Exported for use by test code. + * @internal + */ + U_I18N_API static uint32_t U_EXPORT2 getCurrentEra(); // the current era + + /** + * Constructs a JapaneseCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of JapaneseCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @stable ICU 2.0 + */ + JapaneseCalendar(const Locale& aLocale, UErrorCode& success); + + + /** + * Destructor + * @internal + */ + virtual ~JapaneseCalendar(); + + /** + * Copy constructor + * @param source the object to be copied. + * @internal + */ + JapaneseCalendar(const JapaneseCalendar& source); + + /** + * Default assignment operator + * @param right the object to be copied. + * @internal + */ + JapaneseCalendar& operator=(const JapaneseCalendar& right); + + /** + * Create and return a polymorphic copy of this calendar. + * @return return a polymorphic copy of this calendar. + * @internal + */ + virtual JapaneseCalendar* clone() const override; + + /** + * Return the extended year defined by the current fields. In the + * Japanese calendar case, this is equal to the equivalent extended Gregorian year. + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Return the maximum value that this field could have, given the current date. + * @internal + */ + virtual int32_t getActualMaximum(UCalendarDateFields field, UErrorCode& status) const override; + + +public: + /** + * Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual + * override. This method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() methods call + * this method. + * + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "japanese". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * @return false - no default century in Japanese + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Not used - no default century. + * @internal + */ + virtual UDate defaultCenturyStart() const override; + /** + * Not used - no default century. + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; + +private: + JapaneseCalendar(); // default constructor not implemented + +protected: + /** + * Calculate the era for internal computation + * @internal + */ + virtual int32_t internalGetEra() const override; + + /** + * Compute fields from the JD + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode& status) override; + + /** + * Calculate the limit for a specified type of limit and field + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /*** + * Called by computeJulianDay. Returns the default month (0-based) for the year, + * taking year and era into account. Will return the first month of the given era, if + * the current year is an ascension year. + * @param eyear the extended year + * @internal + */ + virtual int32_t getDefaultMonthInYear(int32_t eyear) override; + + /*** + * Called by computeJulianDay. Returns the default day (1-based) for the month, + * taking currently-set year and era into account. Will return the first day of the given + * era, if the current month is an ascension year and month. + * @param eyear the extended year + * @param mon the month in the year + * @internal + */ + virtual int32_t getDefaultDayInMonth(int32_t eyear, int32_t month) override; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif +//eof + diff --git a/intl/icu/source/i18n/listformatter.cpp b/intl/icu/source/i18n/listformatter.cpp new file mode 100644 index 0000000000..3405b5de38 --- /dev/null +++ b/intl/icu/source/i18n/listformatter.cpp @@ -0,0 +1,732 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2013-2016, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: listformatter.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2012aug27 +* created by: Umesh P. Nair +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "cmemory.h" +#include "unicode/fpositer.h" // FieldPositionIterator +#include "unicode/listformatter.h" +#include "unicode/simpleformatter.h" +#include "unicode/ulistformatter.h" +#include "unicode/uscript.h" +#include "fphdlimp.h" +#include "mutex.h" +#include "hash.h" +#include "cstring.h" +#include "uarrsort.h" +#include "ulocimp.h" +#include "charstr.h" +#include "ucln_in.h" +#include "uresimp.h" +#include "resource.h" +#include "formattedval_impl.h" + +U_NAMESPACE_BEGIN + +namespace { + +class PatternHandler : public UObject { +public: + PatternHandler(const UnicodeString& two, const UnicodeString& end, UErrorCode& errorCode) : + twoPattern(two, 2, 2, errorCode), + endPattern(end, 2, 2, errorCode) { } + + PatternHandler(const SimpleFormatter& two, const SimpleFormatter& end) : + twoPattern(two), + endPattern(end) { } + + virtual ~PatternHandler(); + + virtual PatternHandler* clone() const { return new PatternHandler(twoPattern, endPattern); } + + /** Argument: final string in the list. */ + virtual const SimpleFormatter& getTwoPattern(const UnicodeString&) const { + return twoPattern; + } + + /** Argument: final string in the list. */ + virtual const SimpleFormatter& getEndPattern(const UnicodeString&) const { + return endPattern; + } + +protected: + SimpleFormatter twoPattern; + SimpleFormatter endPattern; +}; + +PatternHandler::~PatternHandler() { +} + +class ContextualHandler : public PatternHandler { +public: + ContextualHandler(bool (*testFunc)(const UnicodeString& text), + const UnicodeString& thenTwo, + const UnicodeString& elseTwo, + const UnicodeString& thenEnd, + const UnicodeString& elseEnd, + UErrorCode& errorCode) : + PatternHandler(elseTwo, elseEnd, errorCode), + test(testFunc), + thenTwoPattern(thenTwo, 2, 2, errorCode), + thenEndPattern(thenEnd, 2, 2, errorCode) { } + + ContextualHandler(bool (*testFunc)(const UnicodeString& text), + const SimpleFormatter& thenTwo, SimpleFormatter elseTwo, + const SimpleFormatter& thenEnd, SimpleFormatter elseEnd) : + PatternHandler(elseTwo, elseEnd), + test(testFunc), + thenTwoPattern(thenTwo), + thenEndPattern(thenEnd) { } + + ~ContextualHandler() override; + + PatternHandler* clone() const override { + return new ContextualHandler( + test, thenTwoPattern, twoPattern, thenEndPattern, endPattern); + } + + const SimpleFormatter& getTwoPattern( + const UnicodeString& text) const override { + return (test)(text) ? thenTwoPattern : twoPattern; + } + + const SimpleFormatter& getEndPattern( + const UnicodeString& text) const override { + return (test)(text) ? thenEndPattern : endPattern; + } + +private: + bool (*test)(const UnicodeString&); + SimpleFormatter thenTwoPattern; + SimpleFormatter thenEndPattern; +}; + +ContextualHandler::~ContextualHandler() { +} + +static const char16_t *spanishY = u"{0} y {1}"; +static const char16_t *spanishE = u"{0} e {1}"; +static const char16_t *spanishO = u"{0} o {1}"; +static const char16_t *spanishU = u"{0} u {1}"; +static const char16_t *hebrewVav = u"{0} \u05D5{1}"; +static const char16_t *hebrewVavDash = u"{0} \u05D5-{1}"; + +// Condiction to change to e. +// Starts with "hi" or "i" but not with "hie" nor "hia" +static bool shouldChangeToE(const UnicodeString& text) { + int32_t len = text.length(); + if (len == 0) { return false; } + // Case insensitive match hi but not hie nor hia. + if ((text[0] == u'h' || text[0] == u'H') && + ((len > 1) && (text[1] == u'i' || text[1] == u'I')) && + ((len == 2) || !(text[2] == u'a' || text[2] == u'A' || text[2] == u'e' || text[2] == u'E'))) { + return true; + } + // Case insensitive for "start with i" + if (text[0] == u'i' || text[0] == u'I') { return true; } + return false; +} + +// Condiction to change to u. +// Starts with "o", "ho", and "8". Also "11" by itself. +// re: ^((o|ho|8).*|11)$ +static bool shouldChangeToU(const UnicodeString& text) { + int32_t len = text.length(); + if (len == 0) { return false; } + // Case insensitive match o.* and 8.* + if (text[0] == u'o' || text[0] == u'O' || text[0] == u'8') { return true; } + // Case insensitive match ho.* + if ((text[0] == u'h' || text[0] == u'H') && + ((len > 1) && (text[1] == 'o' || text[1] == u'O'))) { + return true; + } + // match "^11$" and "^11 .*" + if ((len >= 2) && text[0] == u'1' && text[1] == u'1' && (len == 2 || text[2] == u' ')) { return true; } + return false; +} + +// Condiction to change to VAV follow by a dash. +// Starts with non Hebrew letter. +static bool shouldChangeToVavDash(const UnicodeString& text) { + if (text.isEmpty()) { return false; } + UErrorCode status = U_ZERO_ERROR; + return uscript_getScript(text.char32At(0), &status) != USCRIPT_HEBREW; +} + +PatternHandler* createPatternHandler( + const char* lang, const UnicodeString& two, const UnicodeString& end, + UErrorCode& status) { + if (uprv_strcmp(lang, "es") == 0) { + // Spanish + UnicodeString spanishYStr(true, spanishY, -1); + bool twoIsY = two == spanishYStr; + bool endIsY = end == spanishYStr; + if (twoIsY || endIsY) { + UnicodeString replacement(true, spanishE, -1); + return new ContextualHandler( + shouldChangeToE, + twoIsY ? replacement : two, two, + endIsY ? replacement : end, end, status); + } + UnicodeString spanishOStr(true, spanishO, -1); + bool twoIsO = two == spanishOStr; + bool endIsO = end == spanishOStr; + if (twoIsO || endIsO) { + UnicodeString replacement(true, spanishU, -1); + return new ContextualHandler( + shouldChangeToU, + twoIsO ? replacement : two, two, + endIsO ? replacement : end, end, status); + } + } else if (uprv_strcmp(lang, "he") == 0 || uprv_strcmp(lang, "iw") == 0) { + // Hebrew + UnicodeString hebrewVavStr(true, hebrewVav, -1); + bool twoIsVav = two == hebrewVavStr; + bool endIsVav = end == hebrewVavStr; + if (twoIsVav || endIsVav) { + UnicodeString replacement(true, hebrewVavDash, -1); + return new ContextualHandler( + shouldChangeToVavDash, + twoIsVav ? replacement : two, two, + endIsVav ? replacement : end, end, status); + } + } + return new PatternHandler(two, end, status); +} + +} // namespace + +struct ListFormatInternal : public UMemory { + SimpleFormatter startPattern; + SimpleFormatter middlePattern; + LocalPointer patternHandler; + +ListFormatInternal( + const UnicodeString& two, + const UnicodeString& start, + const UnicodeString& middle, + const UnicodeString& end, + const Locale& locale, + UErrorCode &errorCode) : + startPattern(start, 2, 2, errorCode), + middlePattern(middle, 2, 2, errorCode), + patternHandler(createPatternHandler(locale.getLanguage(), two, end, errorCode), errorCode) { } + +ListFormatInternal(const ListFormatData &data, UErrorCode &errorCode) : + startPattern(data.startPattern, errorCode), + middlePattern(data.middlePattern, errorCode), + patternHandler(createPatternHandler( + data.locale.getLanguage(), data.twoPattern, data.endPattern, errorCode), errorCode) { } + +ListFormatInternal(const ListFormatInternal &other) : + startPattern(other.startPattern), + middlePattern(other.middlePattern), + patternHandler(other.patternHandler->clone()) { } +}; + + +class FormattedListData : public FormattedValueStringBuilderImpl { +public: + FormattedListData(UErrorCode&) : FormattedValueStringBuilderImpl(kUndefinedField) {} + virtual ~FormattedListData(); +}; + +FormattedListData::~FormattedListData() = default; + +UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(FormattedList) + + +static Hashtable* listPatternHash = nullptr; + +U_CDECL_BEGIN +static UBool U_CALLCONV uprv_listformatter_cleanup() { + delete listPatternHash; + listPatternHash = nullptr; + return true; +} + +static void U_CALLCONV +uprv_deleteListFormatInternal(void *obj) { + delete static_cast(obj); +} + +U_CDECL_END + +ListFormatter::ListFormatter(const ListFormatter& other) : + owned(other.owned), data(other.data) { + if (other.owned != nullptr) { + owned = new ListFormatInternal(*other.owned); + data = owned; + } +} + +ListFormatter& ListFormatter::operator=(const ListFormatter& other) { + if (this == &other) { + return *this; + } + delete owned; + if (other.owned) { + owned = new ListFormatInternal(*other.owned); + data = owned; + } else { + owned = nullptr; + data = other.data; + } + return *this; +} + +void ListFormatter::initializeHash(UErrorCode& errorCode) { + if (U_FAILURE(errorCode)) { + return; + } + + listPatternHash = new Hashtable(); + if (listPatternHash == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + + listPatternHash->setValueDeleter(uprv_deleteListFormatInternal); + ucln_i18n_registerCleanup(UCLN_I18N_LIST_FORMATTER, uprv_listformatter_cleanup); + +} + +const ListFormatInternal* ListFormatter::getListFormatInternal( + const Locale& locale, const char *style, UErrorCode& errorCode) { + if (U_FAILURE(errorCode)) { + return nullptr; + } + CharString keyBuffer(locale.getName(), errorCode); + keyBuffer.append(':', errorCode).append(style, errorCode); + UnicodeString key(keyBuffer.data(), -1, US_INV); + ListFormatInternal* result = nullptr; + static UMutex listFormatterMutex; + { + Mutex m(&listFormatterMutex); + if (listPatternHash == nullptr) { + initializeHash(errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + } + result = static_cast(listPatternHash->get(key)); + } + if (result != nullptr) { + return result; + } + result = loadListFormatInternal(locale, style, errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + + { + Mutex m(&listFormatterMutex); + ListFormatInternal* temp = static_cast(listPatternHash->get(key)); + if (temp != nullptr) { + delete result; + result = temp; + } else { + listPatternHash->put(key, result, errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + } + } + return result; +} + +static const char* typeWidthToStyleString(UListFormatterType type, UListFormatterWidth width) { + switch (type) { + case ULISTFMT_TYPE_AND: + switch (width) { + case ULISTFMT_WIDTH_WIDE: + return "standard"; + case ULISTFMT_WIDTH_SHORT: + return "standard-short"; + case ULISTFMT_WIDTH_NARROW: + return "standard-narrow"; + default: + return nullptr; + } + break; + + case ULISTFMT_TYPE_OR: + switch (width) { + case ULISTFMT_WIDTH_WIDE: + return "or"; + case ULISTFMT_WIDTH_SHORT: + return "or-short"; + case ULISTFMT_WIDTH_NARROW: + return "or-narrow"; + default: + return nullptr; + } + break; + + case ULISTFMT_TYPE_UNITS: + switch (width) { + case ULISTFMT_WIDTH_WIDE: + return "unit"; + case ULISTFMT_WIDTH_SHORT: + return "unit-short"; + case ULISTFMT_WIDTH_NARROW: + return "unit-narrow"; + default: + return nullptr; + } + } + + return nullptr; +} + +static const char16_t solidus = 0x2F; +static const char16_t aliasPrefix[] = { 0x6C,0x69,0x73,0x74,0x50,0x61,0x74,0x74,0x65,0x72,0x6E,0x2F }; // "listPattern/" +enum { + kAliasPrefixLen = UPRV_LENGTHOF(aliasPrefix), + kStyleLenMax = 24 // longest currently is 14 +}; + +struct ListFormatter::ListPatternsSink : public ResourceSink { + UnicodeString two, start, middle, end; + char aliasedStyle[kStyleLenMax+1] = {0}; + + ListPatternsSink() {} + virtual ~ListPatternsSink(); + + void setAliasedStyle(UnicodeString alias) { + int32_t startIndex = alias.indexOf(aliasPrefix, kAliasPrefixLen, 0); + if (startIndex < 0) { + return; + } + startIndex += kAliasPrefixLen; + int32_t endIndex = alias.indexOf(solidus, startIndex); + if (endIndex < 0) { + endIndex = alias.length(); + } + alias.extract(startIndex, endIndex-startIndex, aliasedStyle, kStyleLenMax+1, US_INV); + aliasedStyle[kStyleLenMax] = 0; + } + + void handleValueForPattern(ResourceValue &value, UnicodeString &pattern, UErrorCode &errorCode) { + if (pattern.isEmpty()) { + if (value.getType() == URES_ALIAS) { + if (aliasedStyle[0] == 0) { + setAliasedStyle(value.getAliasUnicodeString(errorCode)); + } + } else { + pattern = value.getUnicodeString(errorCode); + } + } + } + + virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/, + UErrorCode &errorCode) override { + aliasedStyle[0] = 0; + if (value.getType() == URES_ALIAS) { + setAliasedStyle(value.getAliasUnicodeString(errorCode)); + return; + } + ResourceTable listPatterns = value.getTable(errorCode); + for (int i = 0; U_SUCCESS(errorCode) && listPatterns.getKeyAndValue(i, key, value); ++i) { + if (uprv_strcmp(key, "2") == 0) { + handleValueForPattern(value, two, errorCode); + } else if (uprv_strcmp(key, "end") == 0) { + handleValueForPattern(value, end, errorCode); + } else if (uprv_strcmp(key, "middle") == 0) { + handleValueForPattern(value, middle, errorCode); + } else if (uprv_strcmp(key, "start") == 0) { + handleValueForPattern(value, start, errorCode); + } + } + } +}; + +// Virtual destructors must be defined out of line. +ListFormatter::ListPatternsSink::~ListPatternsSink() {} + +ListFormatInternal* ListFormatter::loadListFormatInternal( + const Locale& locale, const char * style, UErrorCode& errorCode) { + UResourceBundle* rb = ures_open(nullptr, locale.getName(), &errorCode); + rb = ures_getByKeyWithFallback(rb, "listPattern", rb, &errorCode); + if (U_FAILURE(errorCode)) { + ures_close(rb); + return nullptr; + } + ListFormatter::ListPatternsSink sink; + char currentStyle[kStyleLenMax+1]; + uprv_strncpy(currentStyle, style, kStyleLenMax); + currentStyle[kStyleLenMax] = 0; + + for (;;) { + ures_getAllItemsWithFallback(rb, currentStyle, sink, errorCode); + if (U_FAILURE(errorCode) || sink.aliasedStyle[0] == 0 || uprv_strcmp(currentStyle, sink.aliasedStyle) == 0) { + break; + } + uprv_strcpy(currentStyle, sink.aliasedStyle); + } + ures_close(rb); + if (U_FAILURE(errorCode)) { + return nullptr; + } + if (sink.two.isEmpty() || sink.start.isEmpty() || sink.middle.isEmpty() || sink.end.isEmpty()) { + errorCode = U_MISSING_RESOURCE_ERROR; + return nullptr; + } + + ListFormatInternal* result = new ListFormatInternal(sink.two, sink.start, sink.middle, sink.end, locale, errorCode); + if (result == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if (U_FAILURE(errorCode)) { + delete result; + return nullptr; + } + return result; +} + +ListFormatter* ListFormatter::createInstance(UErrorCode& errorCode) { + Locale locale; // The default locale. + return createInstance(locale, errorCode); +} + +ListFormatter* ListFormatter::createInstance(const Locale& locale, UErrorCode& errorCode) { + return createInstance(locale, ULISTFMT_TYPE_AND, ULISTFMT_WIDTH_WIDE, errorCode); +} + +ListFormatter* ListFormatter::createInstance( + const Locale& locale, UListFormatterType type, UListFormatterWidth width, UErrorCode& errorCode) { + const char* style = typeWidthToStyleString(type, width); + if (style == nullptr) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + return createInstance(locale, style, errorCode); +} + +ListFormatter* ListFormatter::createInstance(const Locale& locale, const char *style, UErrorCode& errorCode) { + const ListFormatInternal* listFormatInternal = getListFormatInternal(locale, style, errorCode); + if (U_FAILURE(errorCode)) { + return nullptr; + } + ListFormatter* p = new ListFormatter(listFormatInternal); + if (p == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return p; +} + +ListFormatter::ListFormatter(const ListFormatData& listFormatData, UErrorCode &errorCode) { + owned = new ListFormatInternal(listFormatData, errorCode); + data = owned; +} + +ListFormatter::ListFormatter(const ListFormatInternal* listFormatterInternal) : owned(nullptr), data(listFormatterInternal) { +} + +ListFormatter::~ListFormatter() { + delete owned; +} + +namespace { + +class FormattedListBuilder { +public: + LocalPointer data; + + /** For lists of length 1+ */ + FormattedListBuilder(const UnicodeString& start, UErrorCode& status) + : data(new FormattedListData(status), status) { + if (U_SUCCESS(status)) { + data->getStringRef().append( + start, + {UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD}, + status); + data->appendSpanInfo(UFIELD_CATEGORY_LIST_SPAN, 0, -1, start.length(), status); + } + } + + /** For lists of length 0 */ + FormattedListBuilder(UErrorCode& status) + : data(new FormattedListData(status), status) { + } + + void append(const SimpleFormatter& pattern, const UnicodeString& next, int32_t position, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (pattern.getArgumentLimit() != 2) { + status = U_INTERNAL_PROGRAM_ERROR; + return; + } + // In the pattern, {0} are the pre-existing elements and {1} is the new element. + int32_t offsets[] = {0, 0}; + UnicodeString temp = pattern.getTextWithNoArguments(offsets, 2); + if (offsets[0] <= offsets[1]) { + // prefix{0}infix{1}suffix + // Prepend prefix, then append infix, element, and suffix + data->getStringRef().insert( + 0, + temp.tempSubStringBetween(0, offsets[0]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + data->getStringRef().append( + temp.tempSubStringBetween(offsets[0], offsets[1]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + data->getStringRef().append( + next, + {UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD}, + status); + data->appendSpanInfo(UFIELD_CATEGORY_LIST_SPAN, position, -1, next.length(), status); + data->getStringRef().append( + temp.tempSubString(offsets[1]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + } else { + // prefix{1}infix{0}suffix + // Prepend infix, element, and prefix, then append suffix. + // (We prepend in reverse order because prepending at index 0 is fast.) + data->getStringRef().insert( + 0, + temp.tempSubStringBetween(offsets[1], offsets[0]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + data->getStringRef().insert( + 0, + next, + {UFIELD_CATEGORY_LIST, ULISTFMT_ELEMENT_FIELD}, + status); + data->prependSpanInfo(UFIELD_CATEGORY_LIST_SPAN, position, -1, next.length(), status); + data->getStringRef().insert( + 0, + temp.tempSubStringBetween(0, offsets[1]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + data->getStringRef().append( + temp.tempSubString(offsets[0]), + {UFIELD_CATEGORY_LIST, ULISTFMT_LITERAL_FIELD}, + status); + } + } +}; + +} + +UnicodeString& ListFormatter::format( + const UnicodeString items[], + int32_t nItems, + UnicodeString& appendTo, + UErrorCode& errorCode) const { + int32_t offset; + return format(items, nItems, appendTo, -1, offset, errorCode); +} + +UnicodeString& ListFormatter::format( + const UnicodeString items[], + int32_t nItems, + UnicodeString& appendTo, + int32_t index, + int32_t &offset, + UErrorCode& errorCode) const { + int32_t initialOffset = appendTo.length(); + auto result = formatStringsToValue(items, nItems, errorCode); + UnicodeStringAppendable appendable(appendTo); + result.appendTo(appendable, errorCode); + if (index >= 0) { + ConstrainedFieldPosition cfpos; + cfpos.constrainField(UFIELD_CATEGORY_LIST_SPAN, index); + result.nextPosition(cfpos, errorCode); + offset = initialOffset + cfpos.getStart(); + } + return appendTo; +} + +FormattedList ListFormatter::formatStringsToValue( + const UnicodeString items[], + int32_t nItems, + UErrorCode& errorCode) const { + if (nItems == 0) { + FormattedListBuilder result(errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } else { + return FormattedList(result.data.orphan()); + } + } else if (nItems == 1) { + FormattedListBuilder result(items[0], errorCode); + result.data->getStringRef().writeTerminator(errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } else { + return FormattedList(result.data.orphan()); + } + } else if (nItems == 2) { + FormattedListBuilder result(items[0], errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } + result.append( + data->patternHandler->getTwoPattern(items[1]), + items[1], + 1, + errorCode); + result.data->getStringRef().writeTerminator(errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } else { + return FormattedList(result.data.orphan()); + } + } + + FormattedListBuilder result(items[0], errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } + result.append( + data->startPattern, + items[1], + 1, + errorCode); + for (int32_t i = 2; i < nItems - 1; i++) { + result.append( + data->middlePattern, + items[i], + i, + errorCode); + } + result.append( + data->patternHandler->getEndPattern(items[nItems-1]), + items[nItems-1], + nItems-1, + errorCode); + result.data->getStringRef().writeTerminator(errorCode); + if (U_FAILURE(errorCode)) { + return FormattedList(errorCode); + } else { + return FormattedList(result.data.orphan()); + } +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/measfmt.cpp b/intl/icu/source/i18n/measfmt.cpp new file mode 100644 index 0000000000..da4e69b49b --- /dev/null +++ b/intl/icu/source/i18n/measfmt.cpp @@ -0,0 +1,893 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2016, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 20, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "utypeinfo.h" // for 'typeid' to work +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/measfmt.h" +#include "unicode/numfmt.h" +#include "currfmt.h" +#include "unicode/localpointer.h" +#include "resource.h" +#include "unicode/simpleformatter.h" +#include "quantityformatter.h" +#include "unicode/plurrule.h" +#include "unicode/decimfmt.h" +#include "uresimp.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "ureslocs.h" +#include "cstring.h" +#include "mutex.h" +#include "ucln_in.h" +#include "unicode/listformatter.h" +#include "charstr.h" +#include "unicode/putil.h" +#include "unicode/smpdtfmt.h" +#include "uassert.h" +#include "unicode/numberformatter.h" +#include "number_longnames.h" +#include "number_utypes.h" + +#include "sharednumberformat.h" +#include "sharedpluralrules.h" +#include "standardplural.h" +#include "unifiedcache.h" + + +U_NAMESPACE_BEGIN + +using number::impl::UFormattedNumberData; + +static constexpr int32_t WIDTH_INDEX_COUNT = UMEASFMT_WIDTH_NARROW + 1; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MeasureFormat) + +// Used to format durations like 5:47 or 21:35:42. +class NumericDateFormatters : public UMemory { +public: + // Formats like H:mm + UnicodeString hourMinute; + + // formats like M:ss + UnicodeString minuteSecond; + + // formats like H:mm:ss + UnicodeString hourMinuteSecond; + + // Constructor that takes the actual patterns for hour-minute, + // minute-second, and hour-minute-second respectively. + NumericDateFormatters( + const UnicodeString &hm, + const UnicodeString &ms, + const UnicodeString &hms) : + hourMinute(hm), + minuteSecond(ms), + hourMinuteSecond(hms) { + } +private: + NumericDateFormatters(const NumericDateFormatters &other); + NumericDateFormatters &operator=(const NumericDateFormatters &other); +}; + +static UMeasureFormatWidth getRegularWidth(UMeasureFormatWidth width) { + if (width >= WIDTH_INDEX_COUNT) { + return UMEASFMT_WIDTH_NARROW; + } + return width; +} + +static UNumberUnitWidth getUnitWidth(UMeasureFormatWidth width) { + switch (width) { + case UMEASFMT_WIDTH_WIDE: + return UNUM_UNIT_WIDTH_FULL_NAME; + case UMEASFMT_WIDTH_NARROW: + case UMEASFMT_WIDTH_NUMERIC: + return UNUM_UNIT_WIDTH_NARROW; + case UMEASFMT_WIDTH_SHORT: + default: + return UNUM_UNIT_WIDTH_SHORT; + } +} + +/** + * Instances contain all MeasureFormat specific data for a particular locale. + * This data is cached. It is never copied, but is shared via shared pointers. + * + * Note: We might change the cache data to have an array[WIDTH_INDEX_COUNT] of + * complete sets of unit & per patterns, + * to correspond to the resource data and its aliases. + * + * TODO: Maybe store more sparsely in general, with pointers rather than potentially-empty objects. + */ +class MeasureFormatCacheData : public SharedObject { +public: + + /** + * Redirection data from root-bundle, top-level sideways aliases. + * - UMEASFMT_WIDTH_COUNT: initial value, just fall back to root + * - UMEASFMT_WIDTH_WIDE/SHORT/NARROW: sideways alias for missing data + */ + UMeasureFormatWidth widthFallback[WIDTH_INDEX_COUNT]; + + MeasureFormatCacheData(); + virtual ~MeasureFormatCacheData(); + + void adoptCurrencyFormat(int32_t widthIndex, NumberFormat *nfToAdopt) { + delete currencyFormats[widthIndex]; + currencyFormats[widthIndex] = nfToAdopt; + } + const NumberFormat *getCurrencyFormat(UMeasureFormatWidth width) const { + return currencyFormats[getRegularWidth(width)]; + } + void adoptIntegerFormat(NumberFormat *nfToAdopt) { + delete integerFormat; + integerFormat = nfToAdopt; + } + const NumberFormat *getIntegerFormat() const { + return integerFormat; + } + void adoptNumericDateFormatters(NumericDateFormatters *formattersToAdopt) { + delete numericDateFormatters; + numericDateFormatters = formattersToAdopt; + } + const NumericDateFormatters *getNumericDateFormatters() const { + return numericDateFormatters; + } + +private: + NumberFormat* currencyFormats[WIDTH_INDEX_COUNT]; + NumberFormat* integerFormat; + NumericDateFormatters* numericDateFormatters; + + MeasureFormatCacheData(const MeasureFormatCacheData &other); + MeasureFormatCacheData &operator=(const MeasureFormatCacheData &other); +}; + +MeasureFormatCacheData::MeasureFormatCacheData() + : integerFormat(nullptr), numericDateFormatters(nullptr) { + for (int32_t i = 0; i < WIDTH_INDEX_COUNT; ++i) { + widthFallback[i] = UMEASFMT_WIDTH_COUNT; + } + memset(currencyFormats, 0, sizeof(currencyFormats)); +} + +MeasureFormatCacheData::~MeasureFormatCacheData() { + for (int32_t i = 0; i < UPRV_LENGTHOF(currencyFormats); ++i) { + delete currencyFormats[i]; + } + // Note: the contents of 'dnams' are pointers into the resource bundle + delete integerFormat; + delete numericDateFormatters; +} + +static UBool isCurrency(const MeasureUnit &unit) { + return (uprv_strcmp(unit.getType(), "currency") == 0); +} + +static UBool getString( + const UResourceBundle *resource, + UnicodeString &result, + UErrorCode &status) { + int32_t len = 0; + const char16_t *resStr = ures_getString(resource, &len, &status); + if (U_FAILURE(status)) { + return false; + } + result.setTo(true, resStr, len); + return true; +} + +static UnicodeString loadNumericDateFormatterPattern( + const UResourceBundle *resource, + const char *pattern, + UErrorCode &status) { + UnicodeString result; + if (U_FAILURE(status)) { + return result; + } + CharString chs; + chs.append("durationUnits", status) + .append("/", status).append(pattern, status); + LocalUResourceBundlePointer patternBundle( + ures_getByKeyWithFallback( + resource, + chs.data(), + nullptr, + &status)); + if (U_FAILURE(status)) { + return result; + } + getString(patternBundle.getAlias(), result, status); + // Replace 'h' with 'H' + int32_t len = result.length(); + char16_t *buffer = result.getBuffer(len); + for (int32_t i = 0; i < len; ++i) { + if (buffer[i] == 0x68) { // 'h' + buffer[i] = 0x48; // 'H' + } + } + result.releaseBuffer(len); + return result; +} + +static NumericDateFormatters *loadNumericDateFormatters( + const UResourceBundle *resource, + UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + NumericDateFormatters *result = new NumericDateFormatters( + loadNumericDateFormatterPattern(resource, "hm", status), + loadNumericDateFormatterPattern(resource, "ms", status), + loadNumericDateFormatterPattern(resource, "hms", status)); + if (U_FAILURE(status)) { + delete result; + return nullptr; + } + return result; +} + +template<> +const MeasureFormatCacheData *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_UNIT, localeId, &status)); + static UNumberFormatStyle currencyStyles[] = { + UNUM_CURRENCY_PLURAL, UNUM_CURRENCY_ISO, UNUM_CURRENCY}; + LocalPointer result(new MeasureFormatCacheData(), status); + if (U_FAILURE(status)) { + return nullptr; + } + result->adoptNumericDateFormatters(loadNumericDateFormatters( + unitsBundle.getAlias(), status)); + if (U_FAILURE(status)) { + return nullptr; + } + + for (int32_t i = 0; i < WIDTH_INDEX_COUNT; ++i) { + // NumberFormat::createInstance can erase warning codes from status, so pass it + // a separate status instance + UErrorCode localStatus = U_ZERO_ERROR; + result->adoptCurrencyFormat(i, NumberFormat::createInstance( + localeId, currencyStyles[i], localStatus)); + if (localStatus != U_ZERO_ERROR) { + status = localStatus; + } + if (U_FAILURE(status)) { + return nullptr; + } + } + NumberFormat *inf = NumberFormat::createInstance( + localeId, UNUM_DECIMAL, status); + if (U_FAILURE(status)) { + return nullptr; + } + inf->setMaximumFractionDigits(0); + DecimalFormat *decfmt = dynamic_cast(inf); + if (decfmt != nullptr) { + decfmt->setRoundingMode(DecimalFormat::kRoundDown); + } + result->adoptIntegerFormat(inf); + result->addRef(); + return result.orphan(); +} + +static UBool isTimeUnit(const MeasureUnit &mu, const char *tu) { + return uprv_strcmp(mu.getType(), "duration") == 0 && + uprv_strcmp(mu.getSubtype(), tu) == 0; +} + +// Converts a composite measure into hours-minutes-seconds and stores at hms +// array. [0] is hours; [1] is minutes; [2] is seconds. Returns a bit map of +// units found: 1=hours, 2=minutes, 4=seconds. For example, if measures +// contains hours-minutes, this function would return 3. +// +// If measures cannot be converted into hours, minutes, seconds or if amounts +// are negative, or if hours, minutes, seconds are out of order, returns 0. +static int32_t toHMS( + const Measure *measures, + int32_t measureCount, + Formattable *hms, + UErrorCode &status) { + if (U_FAILURE(status)) { + return 0; + } + int32_t result = 0; + if (U_FAILURE(status)) { + return 0; + } + // We use copy constructor to ensure that both sides of equality operator + // are instances of MeasureUnit base class and not a subclass. Otherwise, + // operator== will immediately return false. + for (int32_t i = 0; i < measureCount; ++i) { + if (isTimeUnit(measures[i].getUnit(), "hour")) { + // hour must come first + if (result >= 1) { + return 0; + } + hms[0] = measures[i].getNumber(); + if (hms[0].getDouble() < 0.0) { + return 0; + } + result |= 1; + } else if (isTimeUnit(measures[i].getUnit(), "minute")) { + // minute must come after hour + if (result >= 2) { + return 0; + } + hms[1] = measures[i].getNumber(); + if (hms[1].getDouble() < 0.0) { + return 0; + } + result |= 2; + } else if (isTimeUnit(measures[i].getUnit(), "second")) { + // second must come after hour and minute + if (result >= 4) { + return 0; + } + hms[2] = measures[i].getNumber(); + if (hms[2].getDouble() < 0.0) { + return 0; + } + result |= 4; + } else { + return 0; + } + } + return result; +} + + +MeasureFormat::MeasureFormat( + const Locale &locale, UMeasureFormatWidth w, UErrorCode &status) + : cache(nullptr), + numberFormat(nullptr), + pluralRules(nullptr), + fWidth(w), + listFormatter(nullptr) { + initMeasureFormat(locale, w, nullptr, status); +} + +MeasureFormat::MeasureFormat( + const Locale &locale, + UMeasureFormatWidth w, + NumberFormat *nfToAdopt, + UErrorCode &status) + : cache(nullptr), + numberFormat(nullptr), + pluralRules(nullptr), + fWidth(w), + listFormatter(nullptr) { + initMeasureFormat(locale, w, nfToAdopt, status); +} + +MeasureFormat::MeasureFormat(const MeasureFormat &other) : + Format(other), + cache(other.cache), + numberFormat(other.numberFormat), + pluralRules(other.pluralRules), + fWidth(other.fWidth), + listFormatter(nullptr) { + cache->addRef(); + numberFormat->addRef(); + pluralRules->addRef(); + if (other.listFormatter != nullptr) { + listFormatter = new ListFormatter(*other.listFormatter); + } +} + +MeasureFormat &MeasureFormat::operator=(const MeasureFormat &other) { + if (this == &other) { + return *this; + } + Format::operator=(other); + SharedObject::copyPtr(other.cache, cache); + SharedObject::copyPtr(other.numberFormat, numberFormat); + SharedObject::copyPtr(other.pluralRules, pluralRules); + fWidth = other.fWidth; + delete listFormatter; + if (other.listFormatter != nullptr) { + listFormatter = new ListFormatter(*other.listFormatter); + } else { + listFormatter = nullptr; + } + return *this; +} + +MeasureFormat::MeasureFormat() : + cache(nullptr), + numberFormat(nullptr), + pluralRules(nullptr), + fWidth(UMEASFMT_WIDTH_SHORT), + listFormatter(nullptr) { +} + +MeasureFormat::~MeasureFormat() { + if (cache != nullptr) { + cache->removeRef(); + } + if (numberFormat != nullptr) { + numberFormat->removeRef(); + } + if (pluralRules != nullptr) { + pluralRules->removeRef(); + } + delete listFormatter; +} + +bool MeasureFormat::operator==(const Format &other) const { + if (this == &other) { // Same object, equal + return true; + } + if (!Format::operator==(other)) { + return false; + } + const MeasureFormat &rhs = static_cast(other); + + // Note: Since the ListFormatter depends only on Locale and width, we + // don't have to check it here. + + // differing widths aren't equivalent + if (fWidth != rhs.fWidth) { + return false; + } + // Width the same check locales. + // We don't need to check locales if both objects have same cache. + if (cache != rhs.cache) { + UErrorCode status = U_ZERO_ERROR; + const char *localeId = getLocaleID(status); + const char *rhsLocaleId = rhs.getLocaleID(status); + if (U_FAILURE(status)) { + // On failure, assume not equal + return false; + } + if (uprv_strcmp(localeId, rhsLocaleId) != 0) { + return false; + } + } + // Locales same, check NumberFormat if shared data differs. + return ( + numberFormat == rhs.numberFormat || + **numberFormat == **rhs.numberFormat); +} + +MeasureFormat *MeasureFormat::clone() const { + return new MeasureFormat(*this); +} + +UnicodeString &MeasureFormat::format( + const Formattable &obj, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const { + if (U_FAILURE(status)) return appendTo; + if (obj.getType() == Formattable::kObject) { + const UObject* formatObj = obj.getObject(); + const Measure* amount = dynamic_cast(formatObj); + if (amount != nullptr) { + return formatMeasure( + *amount, **numberFormat, appendTo, pos, status); + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; +} + +void MeasureFormat::parseObject( + const UnicodeString & /*source*/, + Formattable & /*result*/, + ParsePosition& /*pos*/) const { + return; +} + +UnicodeString &MeasureFormat::formatMeasurePerUnit( + const Measure &measure, + const MeasureUnit &perUnit, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return appendTo; + } + auto* df = dynamic_cast(&getNumberFormatInternal()); + if (df == nullptr) { + // Don't know how to handle other types of NumberFormat + status = U_UNSUPPORTED_ERROR; + return appendTo; + } + UFormattedNumberData result; + if (auto* lnf = df->toNumberFormatter(status)) { + result.quantity.setToDouble(measure.getNumber().getDouble(status)); + lnf->unit(measure.getUnit()) + .perUnit(perUnit) + .unitWidth(getUnitWidth(fWidth)) + .formatImpl(&result, status); + } + DecimalFormat::fieldPositionHelper(result, pos, appendTo.length(), status); + appendTo.append(result.toTempString(status)); + return appendTo; +} + +UnicodeString &MeasureFormat::formatMeasures( + const Measure *measures, + int32_t measureCount, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return appendTo; + } + if (measureCount == 0) { + return appendTo; + } + if (measureCount == 1) { + return formatMeasure(measures[0], **numberFormat, appendTo, pos, status); + } + if (fWidth == UMEASFMT_WIDTH_NUMERIC) { + Formattable hms[3]; + int32_t bitMap = toHMS(measures, measureCount, hms, status); + if (bitMap > 0) { + return formatNumeric(hms, bitMap, appendTo, status); + } + } + if (pos.getField() != FieldPosition::DONT_CARE) { + return formatMeasuresSlowTrack( + measures, measureCount, appendTo, pos, status); + } + UnicodeString *results = new UnicodeString[measureCount]; + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return appendTo; + } + for (int32_t i = 0; i < measureCount; ++i) { + const NumberFormat *nf = cache->getIntegerFormat(); + if (i == measureCount - 1) { + nf = numberFormat->get(); + } + formatMeasure( + measures[i], + *nf, + results[i], + pos, + status); + } + listFormatter->format(results, measureCount, appendTo, status); + delete [] results; + return appendTo; +} + +UnicodeString MeasureFormat::getUnitDisplayName(const MeasureUnit& unit, UErrorCode& status) const { + return number::impl::LongNameHandler::getUnitDisplayName( + getLocale(status), + unit, + getUnitWidth(fWidth), + status); +} + +void MeasureFormat::initMeasureFormat( + const Locale &locale, + UMeasureFormatWidth w, + NumberFormat *nfToAdopt, + UErrorCode &status) { + static const UListFormatterWidth listWidths[] = { + ULISTFMT_WIDTH_WIDE, + ULISTFMT_WIDTH_SHORT, + ULISTFMT_WIDTH_NARROW}; + LocalPointer nf(nfToAdopt); + if (U_FAILURE(status)) { + return; + } + const char *name = locale.getName(); + setLocaleIDs(name, name); + + UnifiedCache::getByLocale(locale, cache, status); + if (U_FAILURE(status)) { + return; + } + + const SharedPluralRules *pr = PluralRules::createSharedInstance( + locale, UPLURAL_TYPE_CARDINAL, status); + if (U_FAILURE(status)) { + return; + } + SharedObject::copyPtr(pr, pluralRules); + pr->removeRef(); + if (nf.isNull()) { + // TODO: Clean this up + const SharedNumberFormat *shared = NumberFormat::createSharedInstance( + locale, UNUM_DECIMAL, status); + if (U_FAILURE(status)) { + return; + } + SharedObject::copyPtr(shared, numberFormat); + shared->removeRef(); + } else { + adoptNumberFormat(nf.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } + fWidth = w; + delete listFormatter; + listFormatter = ListFormatter::createInstance( + locale, + ULISTFMT_TYPE_UNITS, + listWidths[getRegularWidth(fWidth)], + status); +} + +void MeasureFormat::adoptNumberFormat( + NumberFormat *nfToAdopt, UErrorCode &status) { + LocalPointer nf(nfToAdopt); + if (U_FAILURE(status)) { + return; + } + SharedNumberFormat *shared = new SharedNumberFormat(nf.getAlias()); + if (shared == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + nf.orphan(); + SharedObject::copyPtr(shared, numberFormat); +} + +UBool MeasureFormat::setMeasureFormatLocale(const Locale &locale, UErrorCode &status) { + if (U_FAILURE(status) || locale == getLocale(status)) { + return false; + } + initMeasureFormat(locale, fWidth, nullptr, status); + return U_SUCCESS(status); +} + +const NumberFormat &MeasureFormat::getNumberFormatInternal() const { + return **numberFormat; +} + +const NumberFormat &MeasureFormat::getCurrencyFormatInternal() const { + return *cache->getCurrencyFormat(UMEASFMT_WIDTH_NARROW); +} + +const PluralRules &MeasureFormat::getPluralRules() const { + return **pluralRules; +} + +Locale MeasureFormat::getLocale(UErrorCode &status) const { + return Format::getLocale(ULOC_VALID_LOCALE, status); +} + +const char *MeasureFormat::getLocaleID(UErrorCode &status) const { + return Format::getLocaleID(ULOC_VALID_LOCALE, status); +} + +UnicodeString &MeasureFormat::formatMeasure( + const Measure &measure, + const NumberFormat &nf, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return appendTo; + } + const Formattable& amtNumber = measure.getNumber(); + const MeasureUnit& amtUnit = measure.getUnit(); + if (isCurrency(amtUnit)) { + char16_t isoCode[4]; + u_charsToUChars(amtUnit.getSubtype(), isoCode, 4); + return cache->getCurrencyFormat(fWidth)->format( + new CurrencyAmount(amtNumber, isoCode, status), + appendTo, + pos, + status); + } + auto* df = dynamic_cast(&nf); + if (df == nullptr) { + // Handle other types of NumberFormat using the ICU 63 code, modified to + // get the unitPattern from LongNameHandler and handle fallback to OTHER. + UnicodeString formattedNumber; + StandardPlural::Form pluralForm = QuantityFormatter::selectPlural( + amtNumber, nf, **pluralRules, formattedNumber, pos, status); + UnicodeString pattern = number::impl::LongNameHandler::getUnitPattern(getLocale(status), + amtUnit, getUnitWidth(fWidth), pluralForm, status); + // The above handles fallback from other widths to short, and from other plural forms to OTHER + if (U_FAILURE(status)) { + return appendTo; + } + SimpleFormatter formatter(pattern, 0, 1, status); + return QuantityFormatter::format(formatter, formattedNumber, appendTo, pos, status); + } + UFormattedNumberData result; + if (auto* lnf = df->toNumberFormatter(status)) { + result.quantity.setToDouble(amtNumber.getDouble(status)); + lnf->unit(amtUnit) + .unitWidth(getUnitWidth(fWidth)) + .formatImpl(&result, status); + } + DecimalFormat::fieldPositionHelper(result, pos, appendTo.length(), status); + appendTo.append(result.toTempString(status)); + return appendTo; +} + + +// Formats numeric time duration as 5:00:47 or 3:54. +UnicodeString &MeasureFormat::formatNumeric( + const Formattable *hms, // always length 3 + int32_t bitMap, // 1=hour set, 2=minute set, 4=second set + UnicodeString &appendTo, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return appendTo; + } + + UnicodeString pattern; + + double hours = hms[0].getDouble(status); + double minutes = hms[1].getDouble(status); + double seconds = hms[2].getDouble(status); + if (U_FAILURE(status)) { + return appendTo; + } + + // All possible combinations: "h", "m", "s", "hm", "hs", "ms", "hms" + if (bitMap == 5 || bitMap == 7) { // "hms" & "hs" (we add minutes if "hs") + pattern = cache->getNumericDateFormatters()->hourMinuteSecond; + hours = uprv_trunc(hours); + minutes = uprv_trunc(minutes); + } else if (bitMap == 3) { // "hm" + pattern = cache->getNumericDateFormatters()->hourMinute; + hours = uprv_trunc(hours); + } else if (bitMap == 6) { // "ms" + pattern = cache->getNumericDateFormatters()->minuteSecond; + minutes = uprv_trunc(minutes); + } else { // h m s, handled outside formatNumeric. No value is also an error. + status = U_INTERNAL_PROGRAM_ERROR; + return appendTo; + } + + const DecimalFormat *numberFormatter = dynamic_cast(numberFormat->get()); + if (!numberFormatter) { + status = U_INTERNAL_PROGRAM_ERROR; + return appendTo; + } + number::LocalizedNumberFormatter numberFormatter2; + if (auto* lnf = numberFormatter->toNumberFormatter(status)) { + numberFormatter2 = lnf->integerWidth(number::IntegerWidth::zeroFillTo(2)); + } else { + return appendTo; + } + + FormattedStringBuilder fsb; + + UBool protect = false; + const int32_t patternLength = pattern.length(); + for (int32_t i = 0; i < patternLength; i++) { + char16_t c = pattern[i]; + + // Also set the proper field in this switch + // We don't use DateFormat.Field because this is not a date / time, is a duration. + double value = 0; + switch (c) { + case u'H': value = hours; break; + case u'm': value = minutes; break; + case u's': value = seconds; break; + } + + // There is not enough info to add Field(s) for the unit because all we have are plain + // text patterns. For example in "21:51" there is no text for something like "hour", + // while in something like "21h51" there is ("h"). But we can't really tell... + switch (c) { + case u'H': + case u'm': + case u's': + if (protect) { + fsb.appendChar16(c, kUndefinedField, status); + } else { + UnicodeString tmp; + if ((i + 1 < patternLength) && pattern[i + 1] == c) { // doubled + tmp = numberFormatter2.formatDouble(value, status).toString(status); + i++; + } else { + numberFormatter->format(value, tmp, status); + } + // TODO: Use proper Field + fsb.append(tmp, kUndefinedField, status); + } + break; + case u'\'': + // '' is escaped apostrophe + if ((i + 1 < patternLength) && pattern[i + 1] == c) { + fsb.appendChar16(c, kUndefinedField, status); + i++; + } else { + protect = !protect; + } + break; + default: + fsb.appendChar16(c, kUndefinedField, status); + } + } + + appendTo.append(fsb.toTempUnicodeString()); + + return appendTo; +} + +UnicodeString &MeasureFormat::formatMeasuresSlowTrack( + const Measure *measures, + int32_t measureCount, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; + } + FieldPosition dontCare(FieldPosition::DONT_CARE); + FieldPosition fpos(pos.getField()); + LocalArray results(new UnicodeString[measureCount], status); + int32_t fieldPositionFoundIndex = -1; + for (int32_t i = 0; i < measureCount; ++i) { + const NumberFormat *nf = cache->getIntegerFormat(); + if (i == measureCount - 1) { + nf = numberFormat->get(); + } + if (fieldPositionFoundIndex == -1) { + formatMeasure(measures[i], *nf, results[i], fpos, status); + if (U_FAILURE(status)) { + return appendTo; + } + if (fpos.getBeginIndex() != 0 || fpos.getEndIndex() != 0) { + fieldPositionFoundIndex = i; + } + } else { + formatMeasure(measures[i], *nf, results[i], dontCare, status); + } + } + int32_t offset; + listFormatter->format( + results.getAlias(), + measureCount, + appendTo, + fieldPositionFoundIndex, + offset, + status); + if (U_FAILURE(status)) { + return appendTo; + } + // Fix up FieldPosition indexes if our field is found. + if (fieldPositionFoundIndex != -1 && offset != -1) { + pos.setBeginIndex(fpos.getBeginIndex() + offset); + pos.setEndIndex(fpos.getEndIndex() + offset); + } + return appendTo; +} + +MeasureFormat* U_EXPORT2 MeasureFormat::createCurrencyFormat(const Locale& locale, + UErrorCode& ec) { + if (U_FAILURE(ec)) { + return nullptr; + } + LocalPointer fmt(new CurrencyFormat(locale, ec), ec); + return fmt.orphan(); +} + +MeasureFormat* U_EXPORT2 MeasureFormat::createCurrencyFormat(UErrorCode& ec) { + if (U_FAILURE(ec)) { + return nullptr; + } + return MeasureFormat::createCurrencyFormat(Locale::getDefault(), ec); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/measunit.cpp b/intl/icu/source/i18n/measunit.cpp new file mode 100644 index 0000000000..47ae5bcf5d --- /dev/null +++ b/intl/icu/source/i18n/measunit.cpp @@ -0,0 +1,2391 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2016, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 26, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/measunit.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/uenum.h" +#include "unicode/errorcode.h" +#include "ustrenum.h" +#include "cstring.h" +#include "uassert.h" +#include "measunit_impl.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MeasureUnit) + +// All code between the "Start generated code" comment and +// the "End generated code" comment is auto generated code +// and must not be edited manually. For instructions on how to correctly +// update this code, refer to: +// https://icu.unicode.org/design/formatting/measureformat/updating-measure-unit +// +// Start generated code for measunit.cpp + +// Maps from Type ID to offset in gSubTypes. +static const int32_t gOffsets[] = { + 0, + 2, + 7, + 17, + 27, + 31, + 332, + 343, + 360, + 364, + 373, + 376, + 380, + 388, + 410, + 414, + 429, + 430, + 436, + 446, + 451, + 455, + 457, + 491 +}; + +static const int32_t kCurrencyOffset = 5; + +// Must be sorted alphabetically. +static const char * const gTypes[] = { + "acceleration", + "angle", + "area", + "concentr", + "consumption", + "currency", + "digital", + "duration", + "electric", + "energy", + "force", + "frequency", + "graphics", + "length", + "light", + "mass", + "none", + "power", + "pressure", + "speed", + "temperature", + "torque", + "volume" +}; + +// Must be grouped by type and sorted alphabetically within each type. +static const char * const gSubTypes[] = { + "g-force", + "meter-per-square-second", + "arc-minute", + "arc-second", + "degree", + "radian", + "revolution", + "acre", + "dunam", + "hectare", + "square-centimeter", + "square-foot", + "square-inch", + "square-kilometer", + "square-meter", + "square-mile", + "square-yard", + "item", + "karat", + "milligram-ofglucose-per-deciliter", + "milligram-per-deciliter", + "millimole-per-liter", + "mole", + "percent", + "permille", + "permillion", + "permyriad", + "liter-per-100-kilometer", + "liter-per-kilometer", + "mile-per-gallon", + "mile-per-gallon-imperial", + "ADP", + "AED", + "AFA", + "AFN", + "ALK", + "ALL", + "AMD", + "ANG", + "AOA", + "AOK", + "AON", + "AOR", + "ARA", + "ARP", + "ARS", + "ARY", + "ATS", + "AUD", + "AWG", + "AYM", + "AZM", + "AZN", + "BAD", + "BAM", + "BBD", + "BDT", + "BEC", + "BEF", + "BEL", + "BGJ", + "BGK", + "BGL", + "BGN", + "BHD", + "BIF", + "BMD", + "BND", + "BOB", + "BOP", + "BOV", + "BRB", + "BRC", + "BRE", + "BRL", + "BRN", + "BRR", + "BSD", + "BTN", + "BUK", + "BWP", + "BYB", + "BYN", + "BYR", + "BZD", + "CAD", + "CDF", + "CHC", + "CHE", + "CHF", + "CHW", + "CLF", + "CLP", + "CNY", + "COP", + "COU", + "CRC", + "CSD", + "CSJ", + "CSK", + "CUC", + "CUP", + "CVE", + "CYP", + "CZK", + "DDM", + "DEM", + "DJF", + "DKK", + "DOP", + "DZD", + "ECS", + "ECV", + "EEK", + "EGP", + "ERN", + "ESA", + "ESB", + "ESP", + "ETB", + "EUR", + "FIM", + "FJD", + "FKP", + "FRF", + "GBP", + "GEK", + "GEL", + "GHC", + "GHP", + "GHS", + "GIP", + "GMD", + "GNE", + "GNF", + "GNS", + "GQE", + "GRD", + "GTQ", + "GWE", + "GWP", + "GYD", + "HKD", + "HNL", + "HRD", + "HRK", + "HTG", + "HUF", + "IDR", + "IEP", + "ILP", + "ILR", + "ILS", + "INR", + "IQD", + "IRR", + "ISJ", + "ISK", + "ITL", + "JMD", + "JOD", + "JPY", + "KES", + "KGS", + "KHR", + "KMF", + "KPW", + "KRW", + "KWD", + "KYD", + "KZT", + "LAJ", + "LAK", + "LBP", + "LKR", + "LRD", + "LSL", + "LSM", + "LTL", + "LTT", + "LUC", + "LUF", + "LUL", + "LVL", + "LVR", + "LYD", + "MAD", + "MDL", + "MGA", + "MGF", + "MKD", + "MLF", + "MMK", + "MNT", + "MOP", + "MRO", + "MRU", + "MTL", + "MTP", + "MUR", + "MVQ", + "MVR", + "MWK", + "MXN", + "MXP", + "MXV", + "MYR", + "MZE", + "MZM", + "MZN", + "NAD", + "NGN", + "NIC", + "NIO", + "NLG", + "NOK", + "NPR", + "NZD", + "OMR", + "PAB", + "PEH", + "PEI", + "PEN", + "PES", + "PGK", + "PHP", + "PKR", + "PLN", + "PLZ", + "PTE", + "PYG", + "QAR", + "RHD", + "ROK", + "ROL", + "RON", + "RSD", + "RUB", + "RUR", + "RWF", + "SAR", + "SBD", + "SCR", + "SDD", + "SDG", + "SDP", + "SEK", + "SGD", + "SHP", + "SIT", + "SKK", + "SLE", + "SLL", + "SOS", + "SRD", + "SRG", + "SSP", + "STD", + "STN", + "SUR", + "SVC", + "SYP", + "SZL", + "THB", + "TJR", + "TJS", + "TMM", + "TMT", + "TND", + "TOP", + "TPE", + "TRL", + "TRY", + "TTD", + "TWD", + "TZS", + "UAH", + "UAK", + "UGS", + "UGW", + "UGX", + "USD", + "USN", + "USS", + "UYI", + "UYN", + "UYP", + "UYU", + "UYW", + "UZS", + "VEB", + "VED", + "VEF", + "VES", + "VNC", + "VND", + "VUV", + "WST", + "XAF", + "XAG", + "XAU", + "XBA", + "XBB", + "XBC", + "XBD", + "XCD", + "XDR", + "XEU", + "XOF", + "XPD", + "XPF", + "XPT", + "XSU", + "XTS", + "XUA", + "XXX", + "YDD", + "YER", + "YUD", + "YUM", + "YUN", + "ZAL", + "ZAR", + "ZMK", + "ZMW", + "ZRN", + "ZRZ", + "ZWC", + "ZWD", + "ZWL", + "ZWN", + "ZWR", + "bit", + "byte", + "gigabit", + "gigabyte", + "kilobit", + "kilobyte", + "megabit", + "megabyte", + "petabyte", + "terabit", + "terabyte", + "century", + "day", + "day-person", + "decade", + "hour", + "microsecond", + "millisecond", + "minute", + "month", + "month-person", + "nanosecond", + "quarter", + "second", + "week", + "week-person", + "year", + "year-person", + "ampere", + "milliampere", + "ohm", + "volt", + "british-thermal-unit", + "calorie", + "electronvolt", + "foodcalorie", + "joule", + "kilocalorie", + "kilojoule", + "kilowatt-hour", + "therm-us", + "kilowatt-hour-per-100-kilometer", + "newton", + "pound-force", + "gigahertz", + "hertz", + "kilohertz", + "megahertz", + "dot", + "dot-per-centimeter", + "dot-per-inch", + "em", + "megapixel", + "pixel", + "pixel-per-centimeter", + "pixel-per-inch", + "astronomical-unit", + "centimeter", + "decimeter", + "earth-radius", + "fathom", + "foot", + "furlong", + "inch", + "kilometer", + "light-year", + "meter", + "micrometer", + "mile", + "mile-scandinavian", + "millimeter", + "nanometer", + "nautical-mile", + "parsec", + "picometer", + "point", + "solar-radius", + "yard", + "candela", + "lumen", + "lux", + "solar-luminosity", + "carat", + "dalton", + "earth-mass", + "grain", + "gram", + "kilogram", + "microgram", + "milligram", + "ounce", + "ounce-troy", + "pound", + "solar-mass", + "stone", + "ton", + "tonne", + "", + "gigawatt", + "horsepower", + "kilowatt", + "megawatt", + "milliwatt", + "watt", + "atmosphere", + "bar", + "hectopascal", + "inch-ofhg", + "kilopascal", + "megapascal", + "millibar", + "millimeter-ofhg", + "pascal", + "pound-force-per-square-inch", + "beaufort", + "kilometer-per-hour", + "knot", + "meter-per-second", + "mile-per-hour", + "celsius", + "fahrenheit", + "generic", + "kelvin", + "newton-meter", + "pound-force-foot", + "acre-foot", + "barrel", + "bushel", + "centiliter", + "cubic-centimeter", + "cubic-foot", + "cubic-inch", + "cubic-kilometer", + "cubic-meter", + "cubic-mile", + "cubic-yard", + "cup", + "cup-metric", + "deciliter", + "dessert-spoon", + "dessert-spoon-imperial", + "dram", + "drop", + "fluid-ounce", + "fluid-ounce-imperial", + "gallon", + "gallon-imperial", + "hectoliter", + "jigger", + "liter", + "megaliter", + "milliliter", + "pinch", + "pint", + "pint-metric", + "quart", + "quart-imperial", + "tablespoon", + "teaspoon" +}; + +// Shortcuts to the base unit in order to make the default constructor fast +static const int32_t kBaseTypeIdx = 16; +static const int32_t kBaseSubTypeIdx = 0; + +MeasureUnit *MeasureUnit::createGForce(UErrorCode &status) { + return MeasureUnit::create(0, 0, status); +} + +MeasureUnit MeasureUnit::getGForce() { + return MeasureUnit(0, 0); +} + +MeasureUnit *MeasureUnit::createMeterPerSecondSquared(UErrorCode &status) { + return MeasureUnit::create(0, 1, status); +} + +MeasureUnit MeasureUnit::getMeterPerSecondSquared() { + return MeasureUnit(0, 1); +} + +MeasureUnit *MeasureUnit::createArcMinute(UErrorCode &status) { + return MeasureUnit::create(1, 0, status); +} + +MeasureUnit MeasureUnit::getArcMinute() { + return MeasureUnit(1, 0); +} + +MeasureUnit *MeasureUnit::createArcSecond(UErrorCode &status) { + return MeasureUnit::create(1, 1, status); +} + +MeasureUnit MeasureUnit::getArcSecond() { + return MeasureUnit(1, 1); +} + +MeasureUnit *MeasureUnit::createDegree(UErrorCode &status) { + return MeasureUnit::create(1, 2, status); +} + +MeasureUnit MeasureUnit::getDegree() { + return MeasureUnit(1, 2); +} + +MeasureUnit *MeasureUnit::createRadian(UErrorCode &status) { + return MeasureUnit::create(1, 3, status); +} + +MeasureUnit MeasureUnit::getRadian() { + return MeasureUnit(1, 3); +} + +MeasureUnit *MeasureUnit::createRevolutionAngle(UErrorCode &status) { + return MeasureUnit::create(1, 4, status); +} + +MeasureUnit MeasureUnit::getRevolutionAngle() { + return MeasureUnit(1, 4); +} + +MeasureUnit *MeasureUnit::createAcre(UErrorCode &status) { + return MeasureUnit::create(2, 0, status); +} + +MeasureUnit MeasureUnit::getAcre() { + return MeasureUnit(2, 0); +} + +MeasureUnit *MeasureUnit::createDunam(UErrorCode &status) { + return MeasureUnit::create(2, 1, status); +} + +MeasureUnit MeasureUnit::getDunam() { + return MeasureUnit(2, 1); +} + +MeasureUnit *MeasureUnit::createHectare(UErrorCode &status) { + return MeasureUnit::create(2, 2, status); +} + +MeasureUnit MeasureUnit::getHectare() { + return MeasureUnit(2, 2); +} + +MeasureUnit *MeasureUnit::createSquareCentimeter(UErrorCode &status) { + return MeasureUnit::create(2, 3, status); +} + +MeasureUnit MeasureUnit::getSquareCentimeter() { + return MeasureUnit(2, 3); +} + +MeasureUnit *MeasureUnit::createSquareFoot(UErrorCode &status) { + return MeasureUnit::create(2, 4, status); +} + +MeasureUnit MeasureUnit::getSquareFoot() { + return MeasureUnit(2, 4); +} + +MeasureUnit *MeasureUnit::createSquareInch(UErrorCode &status) { + return MeasureUnit::create(2, 5, status); +} + +MeasureUnit MeasureUnit::getSquareInch() { + return MeasureUnit(2, 5); +} + +MeasureUnit *MeasureUnit::createSquareKilometer(UErrorCode &status) { + return MeasureUnit::create(2, 6, status); +} + +MeasureUnit MeasureUnit::getSquareKilometer() { + return MeasureUnit(2, 6); +} + +MeasureUnit *MeasureUnit::createSquareMeter(UErrorCode &status) { + return MeasureUnit::create(2, 7, status); +} + +MeasureUnit MeasureUnit::getSquareMeter() { + return MeasureUnit(2, 7); +} + +MeasureUnit *MeasureUnit::createSquareMile(UErrorCode &status) { + return MeasureUnit::create(2, 8, status); +} + +MeasureUnit MeasureUnit::getSquareMile() { + return MeasureUnit(2, 8); +} + +MeasureUnit *MeasureUnit::createSquareYard(UErrorCode &status) { + return MeasureUnit::create(2, 9, status); +} + +MeasureUnit MeasureUnit::getSquareYard() { + return MeasureUnit(2, 9); +} + +MeasureUnit *MeasureUnit::createItem(UErrorCode &status) { + return MeasureUnit::create(3, 0, status); +} + +MeasureUnit MeasureUnit::getItem() { + return MeasureUnit(3, 0); +} + +MeasureUnit *MeasureUnit::createKarat(UErrorCode &status) { + return MeasureUnit::create(3, 1, status); +} + +MeasureUnit MeasureUnit::getKarat() { + return MeasureUnit(3, 1); +} + +MeasureUnit *MeasureUnit::createMilligramOfglucosePerDeciliter(UErrorCode &status) { + return MeasureUnit::create(3, 2, status); +} + +MeasureUnit MeasureUnit::getMilligramOfglucosePerDeciliter() { + return MeasureUnit(3, 2); +} + +MeasureUnit *MeasureUnit::createMilligramPerDeciliter(UErrorCode &status) { + return MeasureUnit::create(3, 3, status); +} + +MeasureUnit MeasureUnit::getMilligramPerDeciliter() { + return MeasureUnit(3, 3); +} + +MeasureUnit *MeasureUnit::createMillimolePerLiter(UErrorCode &status) { + return MeasureUnit::create(3, 4, status); +} + +MeasureUnit MeasureUnit::getMillimolePerLiter() { + return MeasureUnit(3, 4); +} + +MeasureUnit *MeasureUnit::createMole(UErrorCode &status) { + return MeasureUnit::create(3, 5, status); +} + +MeasureUnit MeasureUnit::getMole() { + return MeasureUnit(3, 5); +} + +MeasureUnit *MeasureUnit::createPercent(UErrorCode &status) { + return MeasureUnit::create(3, 6, status); +} + +MeasureUnit MeasureUnit::getPercent() { + return MeasureUnit(3, 6); +} + +MeasureUnit *MeasureUnit::createPermille(UErrorCode &status) { + return MeasureUnit::create(3, 7, status); +} + +MeasureUnit MeasureUnit::getPermille() { + return MeasureUnit(3, 7); +} + +MeasureUnit *MeasureUnit::createPartPerMillion(UErrorCode &status) { + return MeasureUnit::create(3, 8, status); +} + +MeasureUnit MeasureUnit::getPartPerMillion() { + return MeasureUnit(3, 8); +} + +MeasureUnit *MeasureUnit::createPermyriad(UErrorCode &status) { + return MeasureUnit::create(3, 9, status); +} + +MeasureUnit MeasureUnit::getPermyriad() { + return MeasureUnit(3, 9); +} + +MeasureUnit *MeasureUnit::createLiterPer100Kilometers(UErrorCode &status) { + return MeasureUnit::create(4, 0, status); +} + +MeasureUnit MeasureUnit::getLiterPer100Kilometers() { + return MeasureUnit(4, 0); +} + +MeasureUnit *MeasureUnit::createLiterPerKilometer(UErrorCode &status) { + return MeasureUnit::create(4, 1, status); +} + +MeasureUnit MeasureUnit::getLiterPerKilometer() { + return MeasureUnit(4, 1); +} + +MeasureUnit *MeasureUnit::createMilePerGallon(UErrorCode &status) { + return MeasureUnit::create(4, 2, status); +} + +MeasureUnit MeasureUnit::getMilePerGallon() { + return MeasureUnit(4, 2); +} + +MeasureUnit *MeasureUnit::createMilePerGallonImperial(UErrorCode &status) { + return MeasureUnit::create(4, 3, status); +} + +MeasureUnit MeasureUnit::getMilePerGallonImperial() { + return MeasureUnit(4, 3); +} + +MeasureUnit *MeasureUnit::createBit(UErrorCode &status) { + return MeasureUnit::create(6, 0, status); +} + +MeasureUnit MeasureUnit::getBit() { + return MeasureUnit(6, 0); +} + +MeasureUnit *MeasureUnit::createByte(UErrorCode &status) { + return MeasureUnit::create(6, 1, status); +} + +MeasureUnit MeasureUnit::getByte() { + return MeasureUnit(6, 1); +} + +MeasureUnit *MeasureUnit::createGigabit(UErrorCode &status) { + return MeasureUnit::create(6, 2, status); +} + +MeasureUnit MeasureUnit::getGigabit() { + return MeasureUnit(6, 2); +} + +MeasureUnit *MeasureUnit::createGigabyte(UErrorCode &status) { + return MeasureUnit::create(6, 3, status); +} + +MeasureUnit MeasureUnit::getGigabyte() { + return MeasureUnit(6, 3); +} + +MeasureUnit *MeasureUnit::createKilobit(UErrorCode &status) { + return MeasureUnit::create(6, 4, status); +} + +MeasureUnit MeasureUnit::getKilobit() { + return MeasureUnit(6, 4); +} + +MeasureUnit *MeasureUnit::createKilobyte(UErrorCode &status) { + return MeasureUnit::create(6, 5, status); +} + +MeasureUnit MeasureUnit::getKilobyte() { + return MeasureUnit(6, 5); +} + +MeasureUnit *MeasureUnit::createMegabit(UErrorCode &status) { + return MeasureUnit::create(6, 6, status); +} + +MeasureUnit MeasureUnit::getMegabit() { + return MeasureUnit(6, 6); +} + +MeasureUnit *MeasureUnit::createMegabyte(UErrorCode &status) { + return MeasureUnit::create(6, 7, status); +} + +MeasureUnit MeasureUnit::getMegabyte() { + return MeasureUnit(6, 7); +} + +MeasureUnit *MeasureUnit::createPetabyte(UErrorCode &status) { + return MeasureUnit::create(6, 8, status); +} + +MeasureUnit MeasureUnit::getPetabyte() { + return MeasureUnit(6, 8); +} + +MeasureUnit *MeasureUnit::createTerabit(UErrorCode &status) { + return MeasureUnit::create(6, 9, status); +} + +MeasureUnit MeasureUnit::getTerabit() { + return MeasureUnit(6, 9); +} + +MeasureUnit *MeasureUnit::createTerabyte(UErrorCode &status) { + return MeasureUnit::create(6, 10, status); +} + +MeasureUnit MeasureUnit::getTerabyte() { + return MeasureUnit(6, 10); +} + +MeasureUnit *MeasureUnit::createCentury(UErrorCode &status) { + return MeasureUnit::create(7, 0, status); +} + +MeasureUnit MeasureUnit::getCentury() { + return MeasureUnit(7, 0); +} + +MeasureUnit *MeasureUnit::createDay(UErrorCode &status) { + return MeasureUnit::create(7, 1, status); +} + +MeasureUnit MeasureUnit::getDay() { + return MeasureUnit(7, 1); +} + +MeasureUnit *MeasureUnit::createDayPerson(UErrorCode &status) { + return MeasureUnit::create(7, 2, status); +} + +MeasureUnit MeasureUnit::getDayPerson() { + return MeasureUnit(7, 2); +} + +MeasureUnit *MeasureUnit::createDecade(UErrorCode &status) { + return MeasureUnit::create(7, 3, status); +} + +MeasureUnit MeasureUnit::getDecade() { + return MeasureUnit(7, 3); +} + +MeasureUnit *MeasureUnit::createHour(UErrorCode &status) { + return MeasureUnit::create(7, 4, status); +} + +MeasureUnit MeasureUnit::getHour() { + return MeasureUnit(7, 4); +} + +MeasureUnit *MeasureUnit::createMicrosecond(UErrorCode &status) { + return MeasureUnit::create(7, 5, status); +} + +MeasureUnit MeasureUnit::getMicrosecond() { + return MeasureUnit(7, 5); +} + +MeasureUnit *MeasureUnit::createMillisecond(UErrorCode &status) { + return MeasureUnit::create(7, 6, status); +} + +MeasureUnit MeasureUnit::getMillisecond() { + return MeasureUnit(7, 6); +} + +MeasureUnit *MeasureUnit::createMinute(UErrorCode &status) { + return MeasureUnit::create(7, 7, status); +} + +MeasureUnit MeasureUnit::getMinute() { + return MeasureUnit(7, 7); +} + +MeasureUnit *MeasureUnit::createMonth(UErrorCode &status) { + return MeasureUnit::create(7, 8, status); +} + +MeasureUnit MeasureUnit::getMonth() { + return MeasureUnit(7, 8); +} + +MeasureUnit *MeasureUnit::createMonthPerson(UErrorCode &status) { + return MeasureUnit::create(7, 9, status); +} + +MeasureUnit MeasureUnit::getMonthPerson() { + return MeasureUnit(7, 9); +} + +MeasureUnit *MeasureUnit::createNanosecond(UErrorCode &status) { + return MeasureUnit::create(7, 10, status); +} + +MeasureUnit MeasureUnit::getNanosecond() { + return MeasureUnit(7, 10); +} + +MeasureUnit *MeasureUnit::createQuarter(UErrorCode &status) { + return MeasureUnit::create(7, 11, status); +} + +MeasureUnit MeasureUnit::getQuarter() { + return MeasureUnit(7, 11); +} + +MeasureUnit *MeasureUnit::createSecond(UErrorCode &status) { + return MeasureUnit::create(7, 12, status); +} + +MeasureUnit MeasureUnit::getSecond() { + return MeasureUnit(7, 12); +} + +MeasureUnit *MeasureUnit::createWeek(UErrorCode &status) { + return MeasureUnit::create(7, 13, status); +} + +MeasureUnit MeasureUnit::getWeek() { + return MeasureUnit(7, 13); +} + +MeasureUnit *MeasureUnit::createWeekPerson(UErrorCode &status) { + return MeasureUnit::create(7, 14, status); +} + +MeasureUnit MeasureUnit::getWeekPerson() { + return MeasureUnit(7, 14); +} + +MeasureUnit *MeasureUnit::createYear(UErrorCode &status) { + return MeasureUnit::create(7, 15, status); +} + +MeasureUnit MeasureUnit::getYear() { + return MeasureUnit(7, 15); +} + +MeasureUnit *MeasureUnit::createYearPerson(UErrorCode &status) { + return MeasureUnit::create(7, 16, status); +} + +MeasureUnit MeasureUnit::getYearPerson() { + return MeasureUnit(7, 16); +} + +MeasureUnit *MeasureUnit::createAmpere(UErrorCode &status) { + return MeasureUnit::create(8, 0, status); +} + +MeasureUnit MeasureUnit::getAmpere() { + return MeasureUnit(8, 0); +} + +MeasureUnit *MeasureUnit::createMilliampere(UErrorCode &status) { + return MeasureUnit::create(8, 1, status); +} + +MeasureUnit MeasureUnit::getMilliampere() { + return MeasureUnit(8, 1); +} + +MeasureUnit *MeasureUnit::createOhm(UErrorCode &status) { + return MeasureUnit::create(8, 2, status); +} + +MeasureUnit MeasureUnit::getOhm() { + return MeasureUnit(8, 2); +} + +MeasureUnit *MeasureUnit::createVolt(UErrorCode &status) { + return MeasureUnit::create(8, 3, status); +} + +MeasureUnit MeasureUnit::getVolt() { + return MeasureUnit(8, 3); +} + +MeasureUnit *MeasureUnit::createBritishThermalUnit(UErrorCode &status) { + return MeasureUnit::create(9, 0, status); +} + +MeasureUnit MeasureUnit::getBritishThermalUnit() { + return MeasureUnit(9, 0); +} + +MeasureUnit *MeasureUnit::createCalorie(UErrorCode &status) { + return MeasureUnit::create(9, 1, status); +} + +MeasureUnit MeasureUnit::getCalorie() { + return MeasureUnit(9, 1); +} + +MeasureUnit *MeasureUnit::createElectronvolt(UErrorCode &status) { + return MeasureUnit::create(9, 2, status); +} + +MeasureUnit MeasureUnit::getElectronvolt() { + return MeasureUnit(9, 2); +} + +MeasureUnit *MeasureUnit::createFoodcalorie(UErrorCode &status) { + return MeasureUnit::create(9, 3, status); +} + +MeasureUnit MeasureUnit::getFoodcalorie() { + return MeasureUnit(9, 3); +} + +MeasureUnit *MeasureUnit::createJoule(UErrorCode &status) { + return MeasureUnit::create(9, 4, status); +} + +MeasureUnit MeasureUnit::getJoule() { + return MeasureUnit(9, 4); +} + +MeasureUnit *MeasureUnit::createKilocalorie(UErrorCode &status) { + return MeasureUnit::create(9, 5, status); +} + +MeasureUnit MeasureUnit::getKilocalorie() { + return MeasureUnit(9, 5); +} + +MeasureUnit *MeasureUnit::createKilojoule(UErrorCode &status) { + return MeasureUnit::create(9, 6, status); +} + +MeasureUnit MeasureUnit::getKilojoule() { + return MeasureUnit(9, 6); +} + +MeasureUnit *MeasureUnit::createKilowattHour(UErrorCode &status) { + return MeasureUnit::create(9, 7, status); +} + +MeasureUnit MeasureUnit::getKilowattHour() { + return MeasureUnit(9, 7); +} + +MeasureUnit *MeasureUnit::createThermUs(UErrorCode &status) { + return MeasureUnit::create(9, 8, status); +} + +MeasureUnit MeasureUnit::getThermUs() { + return MeasureUnit(9, 8); +} + +MeasureUnit *MeasureUnit::createKilowattHourPer100Kilometer(UErrorCode &status) { + return MeasureUnit::create(10, 0, status); +} + +MeasureUnit MeasureUnit::getKilowattHourPer100Kilometer() { + return MeasureUnit(10, 0); +} + +MeasureUnit *MeasureUnit::createNewton(UErrorCode &status) { + return MeasureUnit::create(10, 1, status); +} + +MeasureUnit MeasureUnit::getNewton() { + return MeasureUnit(10, 1); +} + +MeasureUnit *MeasureUnit::createPoundForce(UErrorCode &status) { + return MeasureUnit::create(10, 2, status); +} + +MeasureUnit MeasureUnit::getPoundForce() { + return MeasureUnit(10, 2); +} + +MeasureUnit *MeasureUnit::createGigahertz(UErrorCode &status) { + return MeasureUnit::create(11, 0, status); +} + +MeasureUnit MeasureUnit::getGigahertz() { + return MeasureUnit(11, 0); +} + +MeasureUnit *MeasureUnit::createHertz(UErrorCode &status) { + return MeasureUnit::create(11, 1, status); +} + +MeasureUnit MeasureUnit::getHertz() { + return MeasureUnit(11, 1); +} + +MeasureUnit *MeasureUnit::createKilohertz(UErrorCode &status) { + return MeasureUnit::create(11, 2, status); +} + +MeasureUnit MeasureUnit::getKilohertz() { + return MeasureUnit(11, 2); +} + +MeasureUnit *MeasureUnit::createMegahertz(UErrorCode &status) { + return MeasureUnit::create(11, 3, status); +} + +MeasureUnit MeasureUnit::getMegahertz() { + return MeasureUnit(11, 3); +} + +MeasureUnit *MeasureUnit::createDot(UErrorCode &status) { + return MeasureUnit::create(12, 0, status); +} + +MeasureUnit MeasureUnit::getDot() { + return MeasureUnit(12, 0); +} + +MeasureUnit *MeasureUnit::createDotPerCentimeter(UErrorCode &status) { + return MeasureUnit::create(12, 1, status); +} + +MeasureUnit MeasureUnit::getDotPerCentimeter() { + return MeasureUnit(12, 1); +} + +MeasureUnit *MeasureUnit::createDotPerInch(UErrorCode &status) { + return MeasureUnit::create(12, 2, status); +} + +MeasureUnit MeasureUnit::getDotPerInch() { + return MeasureUnit(12, 2); +} + +MeasureUnit *MeasureUnit::createEm(UErrorCode &status) { + return MeasureUnit::create(12, 3, status); +} + +MeasureUnit MeasureUnit::getEm() { + return MeasureUnit(12, 3); +} + +MeasureUnit *MeasureUnit::createMegapixel(UErrorCode &status) { + return MeasureUnit::create(12, 4, status); +} + +MeasureUnit MeasureUnit::getMegapixel() { + return MeasureUnit(12, 4); +} + +MeasureUnit *MeasureUnit::createPixel(UErrorCode &status) { + return MeasureUnit::create(12, 5, status); +} + +MeasureUnit MeasureUnit::getPixel() { + return MeasureUnit(12, 5); +} + +MeasureUnit *MeasureUnit::createPixelPerCentimeter(UErrorCode &status) { + return MeasureUnit::create(12, 6, status); +} + +MeasureUnit MeasureUnit::getPixelPerCentimeter() { + return MeasureUnit(12, 6); +} + +MeasureUnit *MeasureUnit::createPixelPerInch(UErrorCode &status) { + return MeasureUnit::create(12, 7, status); +} + +MeasureUnit MeasureUnit::getPixelPerInch() { + return MeasureUnit(12, 7); +} + +MeasureUnit *MeasureUnit::createAstronomicalUnit(UErrorCode &status) { + return MeasureUnit::create(13, 0, status); +} + +MeasureUnit MeasureUnit::getAstronomicalUnit() { + return MeasureUnit(13, 0); +} + +MeasureUnit *MeasureUnit::createCentimeter(UErrorCode &status) { + return MeasureUnit::create(13, 1, status); +} + +MeasureUnit MeasureUnit::getCentimeter() { + return MeasureUnit(13, 1); +} + +MeasureUnit *MeasureUnit::createDecimeter(UErrorCode &status) { + return MeasureUnit::create(13, 2, status); +} + +MeasureUnit MeasureUnit::getDecimeter() { + return MeasureUnit(13, 2); +} + +MeasureUnit *MeasureUnit::createEarthRadius(UErrorCode &status) { + return MeasureUnit::create(13, 3, status); +} + +MeasureUnit MeasureUnit::getEarthRadius() { + return MeasureUnit(13, 3); +} + +MeasureUnit *MeasureUnit::createFathom(UErrorCode &status) { + return MeasureUnit::create(13, 4, status); +} + +MeasureUnit MeasureUnit::getFathom() { + return MeasureUnit(13, 4); +} + +MeasureUnit *MeasureUnit::createFoot(UErrorCode &status) { + return MeasureUnit::create(13, 5, status); +} + +MeasureUnit MeasureUnit::getFoot() { + return MeasureUnit(13, 5); +} + +MeasureUnit *MeasureUnit::createFurlong(UErrorCode &status) { + return MeasureUnit::create(13, 6, status); +} + +MeasureUnit MeasureUnit::getFurlong() { + return MeasureUnit(13, 6); +} + +MeasureUnit *MeasureUnit::createInch(UErrorCode &status) { + return MeasureUnit::create(13, 7, status); +} + +MeasureUnit MeasureUnit::getInch() { + return MeasureUnit(13, 7); +} + +MeasureUnit *MeasureUnit::createKilometer(UErrorCode &status) { + return MeasureUnit::create(13, 8, status); +} + +MeasureUnit MeasureUnit::getKilometer() { + return MeasureUnit(13, 8); +} + +MeasureUnit *MeasureUnit::createLightYear(UErrorCode &status) { + return MeasureUnit::create(13, 9, status); +} + +MeasureUnit MeasureUnit::getLightYear() { + return MeasureUnit(13, 9); +} + +MeasureUnit *MeasureUnit::createMeter(UErrorCode &status) { + return MeasureUnit::create(13, 10, status); +} + +MeasureUnit MeasureUnit::getMeter() { + return MeasureUnit(13, 10); +} + +MeasureUnit *MeasureUnit::createMicrometer(UErrorCode &status) { + return MeasureUnit::create(13, 11, status); +} + +MeasureUnit MeasureUnit::getMicrometer() { + return MeasureUnit(13, 11); +} + +MeasureUnit *MeasureUnit::createMile(UErrorCode &status) { + return MeasureUnit::create(13, 12, status); +} + +MeasureUnit MeasureUnit::getMile() { + return MeasureUnit(13, 12); +} + +MeasureUnit *MeasureUnit::createMileScandinavian(UErrorCode &status) { + return MeasureUnit::create(13, 13, status); +} + +MeasureUnit MeasureUnit::getMileScandinavian() { + return MeasureUnit(13, 13); +} + +MeasureUnit *MeasureUnit::createMillimeter(UErrorCode &status) { + return MeasureUnit::create(13, 14, status); +} + +MeasureUnit MeasureUnit::getMillimeter() { + return MeasureUnit(13, 14); +} + +MeasureUnit *MeasureUnit::createNanometer(UErrorCode &status) { + return MeasureUnit::create(13, 15, status); +} + +MeasureUnit MeasureUnit::getNanometer() { + return MeasureUnit(13, 15); +} + +MeasureUnit *MeasureUnit::createNauticalMile(UErrorCode &status) { + return MeasureUnit::create(13, 16, status); +} + +MeasureUnit MeasureUnit::getNauticalMile() { + return MeasureUnit(13, 16); +} + +MeasureUnit *MeasureUnit::createParsec(UErrorCode &status) { + return MeasureUnit::create(13, 17, status); +} + +MeasureUnit MeasureUnit::getParsec() { + return MeasureUnit(13, 17); +} + +MeasureUnit *MeasureUnit::createPicometer(UErrorCode &status) { + return MeasureUnit::create(13, 18, status); +} + +MeasureUnit MeasureUnit::getPicometer() { + return MeasureUnit(13, 18); +} + +MeasureUnit *MeasureUnit::createPoint(UErrorCode &status) { + return MeasureUnit::create(13, 19, status); +} + +MeasureUnit MeasureUnit::getPoint() { + return MeasureUnit(13, 19); +} + +MeasureUnit *MeasureUnit::createSolarRadius(UErrorCode &status) { + return MeasureUnit::create(13, 20, status); +} + +MeasureUnit MeasureUnit::getSolarRadius() { + return MeasureUnit(13, 20); +} + +MeasureUnit *MeasureUnit::createYard(UErrorCode &status) { + return MeasureUnit::create(13, 21, status); +} + +MeasureUnit MeasureUnit::getYard() { + return MeasureUnit(13, 21); +} + +MeasureUnit *MeasureUnit::createCandela(UErrorCode &status) { + return MeasureUnit::create(14, 0, status); +} + +MeasureUnit MeasureUnit::getCandela() { + return MeasureUnit(14, 0); +} + +MeasureUnit *MeasureUnit::createLumen(UErrorCode &status) { + return MeasureUnit::create(14, 1, status); +} + +MeasureUnit MeasureUnit::getLumen() { + return MeasureUnit(14, 1); +} + +MeasureUnit *MeasureUnit::createLux(UErrorCode &status) { + return MeasureUnit::create(14, 2, status); +} + +MeasureUnit MeasureUnit::getLux() { + return MeasureUnit(14, 2); +} + +MeasureUnit *MeasureUnit::createSolarLuminosity(UErrorCode &status) { + return MeasureUnit::create(14, 3, status); +} + +MeasureUnit MeasureUnit::getSolarLuminosity() { + return MeasureUnit(14, 3); +} + +MeasureUnit *MeasureUnit::createCarat(UErrorCode &status) { + return MeasureUnit::create(15, 0, status); +} + +MeasureUnit MeasureUnit::getCarat() { + return MeasureUnit(15, 0); +} + +MeasureUnit *MeasureUnit::createDalton(UErrorCode &status) { + return MeasureUnit::create(15, 1, status); +} + +MeasureUnit MeasureUnit::getDalton() { + return MeasureUnit(15, 1); +} + +MeasureUnit *MeasureUnit::createEarthMass(UErrorCode &status) { + return MeasureUnit::create(15, 2, status); +} + +MeasureUnit MeasureUnit::getEarthMass() { + return MeasureUnit(15, 2); +} + +MeasureUnit *MeasureUnit::createGrain(UErrorCode &status) { + return MeasureUnit::create(15, 3, status); +} + +MeasureUnit MeasureUnit::getGrain() { + return MeasureUnit(15, 3); +} + +MeasureUnit *MeasureUnit::createGram(UErrorCode &status) { + return MeasureUnit::create(15, 4, status); +} + +MeasureUnit MeasureUnit::getGram() { + return MeasureUnit(15, 4); +} + +MeasureUnit *MeasureUnit::createKilogram(UErrorCode &status) { + return MeasureUnit::create(15, 5, status); +} + +MeasureUnit MeasureUnit::getKilogram() { + return MeasureUnit(15, 5); +} + +MeasureUnit *MeasureUnit::createMetricTon(UErrorCode &status) { + return MeasureUnit::create(15, 14, status); +} + +MeasureUnit MeasureUnit::getMetricTon() { + return MeasureUnit(15, 14); +} + +MeasureUnit *MeasureUnit::createMicrogram(UErrorCode &status) { + return MeasureUnit::create(15, 6, status); +} + +MeasureUnit MeasureUnit::getMicrogram() { + return MeasureUnit(15, 6); +} + +MeasureUnit *MeasureUnit::createMilligram(UErrorCode &status) { + return MeasureUnit::create(15, 7, status); +} + +MeasureUnit MeasureUnit::getMilligram() { + return MeasureUnit(15, 7); +} + +MeasureUnit *MeasureUnit::createOunce(UErrorCode &status) { + return MeasureUnit::create(15, 8, status); +} + +MeasureUnit MeasureUnit::getOunce() { + return MeasureUnit(15, 8); +} + +MeasureUnit *MeasureUnit::createOunceTroy(UErrorCode &status) { + return MeasureUnit::create(15, 9, status); +} + +MeasureUnit MeasureUnit::getOunceTroy() { + return MeasureUnit(15, 9); +} + +MeasureUnit *MeasureUnit::createPound(UErrorCode &status) { + return MeasureUnit::create(15, 10, status); +} + +MeasureUnit MeasureUnit::getPound() { + return MeasureUnit(15, 10); +} + +MeasureUnit *MeasureUnit::createSolarMass(UErrorCode &status) { + return MeasureUnit::create(15, 11, status); +} + +MeasureUnit MeasureUnit::getSolarMass() { + return MeasureUnit(15, 11); +} + +MeasureUnit *MeasureUnit::createStone(UErrorCode &status) { + return MeasureUnit::create(15, 12, status); +} + +MeasureUnit MeasureUnit::getStone() { + return MeasureUnit(15, 12); +} + +MeasureUnit *MeasureUnit::createTon(UErrorCode &status) { + return MeasureUnit::create(15, 13, status); +} + +MeasureUnit MeasureUnit::getTon() { + return MeasureUnit(15, 13); +} + +MeasureUnit *MeasureUnit::createTonne(UErrorCode &status) { + return MeasureUnit::create(15, 14, status); +} + +MeasureUnit MeasureUnit::getTonne() { + return MeasureUnit(15, 14); +} + +MeasureUnit *MeasureUnit::createGigawatt(UErrorCode &status) { + return MeasureUnit::create(17, 0, status); +} + +MeasureUnit MeasureUnit::getGigawatt() { + return MeasureUnit(17, 0); +} + +MeasureUnit *MeasureUnit::createHorsepower(UErrorCode &status) { + return MeasureUnit::create(17, 1, status); +} + +MeasureUnit MeasureUnit::getHorsepower() { + return MeasureUnit(17, 1); +} + +MeasureUnit *MeasureUnit::createKilowatt(UErrorCode &status) { + return MeasureUnit::create(17, 2, status); +} + +MeasureUnit MeasureUnit::getKilowatt() { + return MeasureUnit(17, 2); +} + +MeasureUnit *MeasureUnit::createMegawatt(UErrorCode &status) { + return MeasureUnit::create(17, 3, status); +} + +MeasureUnit MeasureUnit::getMegawatt() { + return MeasureUnit(17, 3); +} + +MeasureUnit *MeasureUnit::createMilliwatt(UErrorCode &status) { + return MeasureUnit::create(17, 4, status); +} + +MeasureUnit MeasureUnit::getMilliwatt() { + return MeasureUnit(17, 4); +} + +MeasureUnit *MeasureUnit::createWatt(UErrorCode &status) { + return MeasureUnit::create(17, 5, status); +} + +MeasureUnit MeasureUnit::getWatt() { + return MeasureUnit(17, 5); +} + +MeasureUnit *MeasureUnit::createAtmosphere(UErrorCode &status) { + return MeasureUnit::create(18, 0, status); +} + +MeasureUnit MeasureUnit::getAtmosphere() { + return MeasureUnit(18, 0); +} + +MeasureUnit *MeasureUnit::createBar(UErrorCode &status) { + return MeasureUnit::create(18, 1, status); +} + +MeasureUnit MeasureUnit::getBar() { + return MeasureUnit(18, 1); +} + +MeasureUnit *MeasureUnit::createHectopascal(UErrorCode &status) { + return MeasureUnit::create(18, 2, status); +} + +MeasureUnit MeasureUnit::getHectopascal() { + return MeasureUnit(18, 2); +} + +MeasureUnit *MeasureUnit::createInchHg(UErrorCode &status) { + return MeasureUnit::create(18, 3, status); +} + +MeasureUnit MeasureUnit::getInchHg() { + return MeasureUnit(18, 3); +} + +MeasureUnit *MeasureUnit::createKilopascal(UErrorCode &status) { + return MeasureUnit::create(18, 4, status); +} + +MeasureUnit MeasureUnit::getKilopascal() { + return MeasureUnit(18, 4); +} + +MeasureUnit *MeasureUnit::createMegapascal(UErrorCode &status) { + return MeasureUnit::create(18, 5, status); +} + +MeasureUnit MeasureUnit::getMegapascal() { + return MeasureUnit(18, 5); +} + +MeasureUnit *MeasureUnit::createMillibar(UErrorCode &status) { + return MeasureUnit::create(18, 6, status); +} + +MeasureUnit MeasureUnit::getMillibar() { + return MeasureUnit(18, 6); +} + +MeasureUnit *MeasureUnit::createMillimeterOfMercury(UErrorCode &status) { + return MeasureUnit::create(18, 7, status); +} + +MeasureUnit MeasureUnit::getMillimeterOfMercury() { + return MeasureUnit(18, 7); +} + +MeasureUnit *MeasureUnit::createPascal(UErrorCode &status) { + return MeasureUnit::create(18, 8, status); +} + +MeasureUnit MeasureUnit::getPascal() { + return MeasureUnit(18, 8); +} + +MeasureUnit *MeasureUnit::createPoundPerSquareInch(UErrorCode &status) { + return MeasureUnit::create(18, 9, status); +} + +MeasureUnit MeasureUnit::getPoundPerSquareInch() { + return MeasureUnit(18, 9); +} + +MeasureUnit *MeasureUnit::createBeaufort(UErrorCode &status) { + return MeasureUnit::create(19, 0, status); +} + +MeasureUnit MeasureUnit::getBeaufort() { + return MeasureUnit(19, 0); +} + +MeasureUnit *MeasureUnit::createKilometerPerHour(UErrorCode &status) { + return MeasureUnit::create(19, 1, status); +} + +MeasureUnit MeasureUnit::getKilometerPerHour() { + return MeasureUnit(19, 1); +} + +MeasureUnit *MeasureUnit::createKnot(UErrorCode &status) { + return MeasureUnit::create(19, 2, status); +} + +MeasureUnit MeasureUnit::getKnot() { + return MeasureUnit(19, 2); +} + +MeasureUnit *MeasureUnit::createMeterPerSecond(UErrorCode &status) { + return MeasureUnit::create(19, 3, status); +} + +MeasureUnit MeasureUnit::getMeterPerSecond() { + return MeasureUnit(19, 3); +} + +MeasureUnit *MeasureUnit::createMilePerHour(UErrorCode &status) { + return MeasureUnit::create(19, 4, status); +} + +MeasureUnit MeasureUnit::getMilePerHour() { + return MeasureUnit(19, 4); +} + +MeasureUnit *MeasureUnit::createCelsius(UErrorCode &status) { + return MeasureUnit::create(20, 0, status); +} + +MeasureUnit MeasureUnit::getCelsius() { + return MeasureUnit(20, 0); +} + +MeasureUnit *MeasureUnit::createFahrenheit(UErrorCode &status) { + return MeasureUnit::create(20, 1, status); +} + +MeasureUnit MeasureUnit::getFahrenheit() { + return MeasureUnit(20, 1); +} + +MeasureUnit *MeasureUnit::createGenericTemperature(UErrorCode &status) { + return MeasureUnit::create(20, 2, status); +} + +MeasureUnit MeasureUnit::getGenericTemperature() { + return MeasureUnit(20, 2); +} + +MeasureUnit *MeasureUnit::createKelvin(UErrorCode &status) { + return MeasureUnit::create(20, 3, status); +} + +MeasureUnit MeasureUnit::getKelvin() { + return MeasureUnit(20, 3); +} + +MeasureUnit *MeasureUnit::createNewtonMeter(UErrorCode &status) { + return MeasureUnit::create(21, 0, status); +} + +MeasureUnit MeasureUnit::getNewtonMeter() { + return MeasureUnit(21, 0); +} + +MeasureUnit *MeasureUnit::createPoundFoot(UErrorCode &status) { + return MeasureUnit::create(21, 1, status); +} + +MeasureUnit MeasureUnit::getPoundFoot() { + return MeasureUnit(21, 1); +} + +MeasureUnit *MeasureUnit::createAcreFoot(UErrorCode &status) { + return MeasureUnit::create(22, 0, status); +} + +MeasureUnit MeasureUnit::getAcreFoot() { + return MeasureUnit(22, 0); +} + +MeasureUnit *MeasureUnit::createBarrel(UErrorCode &status) { + return MeasureUnit::create(22, 1, status); +} + +MeasureUnit MeasureUnit::getBarrel() { + return MeasureUnit(22, 1); +} + +MeasureUnit *MeasureUnit::createBushel(UErrorCode &status) { + return MeasureUnit::create(22, 2, status); +} + +MeasureUnit MeasureUnit::getBushel() { + return MeasureUnit(22, 2); +} + +MeasureUnit *MeasureUnit::createCentiliter(UErrorCode &status) { + return MeasureUnit::create(22, 3, status); +} + +MeasureUnit MeasureUnit::getCentiliter() { + return MeasureUnit(22, 3); +} + +MeasureUnit *MeasureUnit::createCubicCentimeter(UErrorCode &status) { + return MeasureUnit::create(22, 4, status); +} + +MeasureUnit MeasureUnit::getCubicCentimeter() { + return MeasureUnit(22, 4); +} + +MeasureUnit *MeasureUnit::createCubicFoot(UErrorCode &status) { + return MeasureUnit::create(22, 5, status); +} + +MeasureUnit MeasureUnit::getCubicFoot() { + return MeasureUnit(22, 5); +} + +MeasureUnit *MeasureUnit::createCubicInch(UErrorCode &status) { + return MeasureUnit::create(22, 6, status); +} + +MeasureUnit MeasureUnit::getCubicInch() { + return MeasureUnit(22, 6); +} + +MeasureUnit *MeasureUnit::createCubicKilometer(UErrorCode &status) { + return MeasureUnit::create(22, 7, status); +} + +MeasureUnit MeasureUnit::getCubicKilometer() { + return MeasureUnit(22, 7); +} + +MeasureUnit *MeasureUnit::createCubicMeter(UErrorCode &status) { + return MeasureUnit::create(22, 8, status); +} + +MeasureUnit MeasureUnit::getCubicMeter() { + return MeasureUnit(22, 8); +} + +MeasureUnit *MeasureUnit::createCubicMile(UErrorCode &status) { + return MeasureUnit::create(22, 9, status); +} + +MeasureUnit MeasureUnit::getCubicMile() { + return MeasureUnit(22, 9); +} + +MeasureUnit *MeasureUnit::createCubicYard(UErrorCode &status) { + return MeasureUnit::create(22, 10, status); +} + +MeasureUnit MeasureUnit::getCubicYard() { + return MeasureUnit(22, 10); +} + +MeasureUnit *MeasureUnit::createCup(UErrorCode &status) { + return MeasureUnit::create(22, 11, status); +} + +MeasureUnit MeasureUnit::getCup() { + return MeasureUnit(22, 11); +} + +MeasureUnit *MeasureUnit::createCupMetric(UErrorCode &status) { + return MeasureUnit::create(22, 12, status); +} + +MeasureUnit MeasureUnit::getCupMetric() { + return MeasureUnit(22, 12); +} + +MeasureUnit *MeasureUnit::createDeciliter(UErrorCode &status) { + return MeasureUnit::create(22, 13, status); +} + +MeasureUnit MeasureUnit::getDeciliter() { + return MeasureUnit(22, 13); +} + +MeasureUnit *MeasureUnit::createDessertSpoon(UErrorCode &status) { + return MeasureUnit::create(22, 14, status); +} + +MeasureUnit MeasureUnit::getDessertSpoon() { + return MeasureUnit(22, 14); +} + +MeasureUnit *MeasureUnit::createDessertSpoonImperial(UErrorCode &status) { + return MeasureUnit::create(22, 15, status); +} + +MeasureUnit MeasureUnit::getDessertSpoonImperial() { + return MeasureUnit(22, 15); +} + +MeasureUnit *MeasureUnit::createDram(UErrorCode &status) { + return MeasureUnit::create(22, 16, status); +} + +MeasureUnit MeasureUnit::getDram() { + return MeasureUnit(22, 16); +} + +MeasureUnit *MeasureUnit::createDrop(UErrorCode &status) { + return MeasureUnit::create(22, 17, status); +} + +MeasureUnit MeasureUnit::getDrop() { + return MeasureUnit(22, 17); +} + +MeasureUnit *MeasureUnit::createFluidOunce(UErrorCode &status) { + return MeasureUnit::create(22, 18, status); +} + +MeasureUnit MeasureUnit::getFluidOunce() { + return MeasureUnit(22, 18); +} + +MeasureUnit *MeasureUnit::createFluidOunceImperial(UErrorCode &status) { + return MeasureUnit::create(22, 19, status); +} + +MeasureUnit MeasureUnit::getFluidOunceImperial() { + return MeasureUnit(22, 19); +} + +MeasureUnit *MeasureUnit::createGallon(UErrorCode &status) { + return MeasureUnit::create(22, 20, status); +} + +MeasureUnit MeasureUnit::getGallon() { + return MeasureUnit(22, 20); +} + +MeasureUnit *MeasureUnit::createGallonImperial(UErrorCode &status) { + return MeasureUnit::create(22, 21, status); +} + +MeasureUnit MeasureUnit::getGallonImperial() { + return MeasureUnit(22, 21); +} + +MeasureUnit *MeasureUnit::createHectoliter(UErrorCode &status) { + return MeasureUnit::create(22, 22, status); +} + +MeasureUnit MeasureUnit::getHectoliter() { + return MeasureUnit(22, 22); +} + +MeasureUnit *MeasureUnit::createJigger(UErrorCode &status) { + return MeasureUnit::create(22, 23, status); +} + +MeasureUnit MeasureUnit::getJigger() { + return MeasureUnit(22, 23); +} + +MeasureUnit *MeasureUnit::createLiter(UErrorCode &status) { + return MeasureUnit::create(22, 24, status); +} + +MeasureUnit MeasureUnit::getLiter() { + return MeasureUnit(22, 24); +} + +MeasureUnit *MeasureUnit::createMegaliter(UErrorCode &status) { + return MeasureUnit::create(22, 25, status); +} + +MeasureUnit MeasureUnit::getMegaliter() { + return MeasureUnit(22, 25); +} + +MeasureUnit *MeasureUnit::createMilliliter(UErrorCode &status) { + return MeasureUnit::create(22, 26, status); +} + +MeasureUnit MeasureUnit::getMilliliter() { + return MeasureUnit(22, 26); +} + +MeasureUnit *MeasureUnit::createPinch(UErrorCode &status) { + return MeasureUnit::create(22, 27, status); +} + +MeasureUnit MeasureUnit::getPinch() { + return MeasureUnit(22, 27); +} + +MeasureUnit *MeasureUnit::createPint(UErrorCode &status) { + return MeasureUnit::create(22, 28, status); +} + +MeasureUnit MeasureUnit::getPint() { + return MeasureUnit(22, 28); +} + +MeasureUnit *MeasureUnit::createPintMetric(UErrorCode &status) { + return MeasureUnit::create(22, 29, status); +} + +MeasureUnit MeasureUnit::getPintMetric() { + return MeasureUnit(22, 29); +} + +MeasureUnit *MeasureUnit::createQuart(UErrorCode &status) { + return MeasureUnit::create(22, 30, status); +} + +MeasureUnit MeasureUnit::getQuart() { + return MeasureUnit(22, 30); +} + +MeasureUnit *MeasureUnit::createQuartImperial(UErrorCode &status) { + return MeasureUnit::create(22, 31, status); +} + +MeasureUnit MeasureUnit::getQuartImperial() { + return MeasureUnit(22, 31); +} + +MeasureUnit *MeasureUnit::createTablespoon(UErrorCode &status) { + return MeasureUnit::create(22, 32, status); +} + +MeasureUnit MeasureUnit::getTablespoon() { + return MeasureUnit(22, 32); +} + +MeasureUnit *MeasureUnit::createTeaspoon(UErrorCode &status) { + return MeasureUnit::create(22, 33, status); +} + +MeasureUnit MeasureUnit::getTeaspoon() { + return MeasureUnit(22, 33); +} + +// End generated code for measunit.cpp + +static int32_t binarySearch( + const char * const * array, int32_t start, int32_t end, StringPiece key) { + while (start < end) { + int32_t mid = (start + end) / 2; + int32_t cmp = StringPiece(array[mid]).compare(key); + if (cmp < 0) { + start = mid + 1; + continue; + } + if (cmp == 0) { + return mid; + } + end = mid; + } + return -1; +} + +MeasureUnit::MeasureUnit() : MeasureUnit(kBaseTypeIdx, kBaseSubTypeIdx) { +} + +MeasureUnit::MeasureUnit(int32_t typeId, int32_t subTypeId) + : fImpl(nullptr), fSubTypeId(subTypeId), fTypeId(typeId) { +} + +MeasureUnit::MeasureUnit(const MeasureUnit &other) + : fImpl(nullptr) { + *this = other; +} + +MeasureUnit::MeasureUnit(MeasureUnit &&other) noexcept + : fImpl(other.fImpl), + fSubTypeId(other.fSubTypeId), + fTypeId(other.fTypeId) { + other.fImpl = nullptr; +} + +MeasureUnit::MeasureUnit(MeasureUnitImpl&& impl) + : fImpl(nullptr), fSubTypeId(-1), fTypeId(-1) { + if (!findBySubType(impl.identifier.toStringPiece(), this)) { + fImpl = new MeasureUnitImpl(std::move(impl)); + } +} + +MeasureUnit &MeasureUnit::operator=(const MeasureUnit &other) { + if (this == &other) { + return *this; + } + if (fImpl != nullptr) { + delete fImpl; + } + if (other.fImpl) { + ErrorCode localStatus; + fImpl = new MeasureUnitImpl(other.fImpl->copy(localStatus)); + if (!fImpl || localStatus.isFailure()) { + // Unrecoverable allocation error; set to the default unit + *this = MeasureUnit(); + return *this; + } + } else { + fImpl = nullptr; + } + fTypeId = other.fTypeId; + fSubTypeId = other.fSubTypeId; + return *this; +} + +MeasureUnit &MeasureUnit::operator=(MeasureUnit &&other) noexcept { + if (this == &other) { + return *this; + } + if (fImpl != nullptr) { + delete fImpl; + } + fImpl = other.fImpl; + other.fImpl = nullptr; + fTypeId = other.fTypeId; + fSubTypeId = other.fSubTypeId; + return *this; +} + +MeasureUnit *MeasureUnit::clone() const { + return new MeasureUnit(*this); +} + +MeasureUnit::~MeasureUnit() { + if (fImpl != nullptr) { + delete fImpl; + fImpl = nullptr; + } +} + +const char *MeasureUnit::getType() const { + // We have a type & subtype only if fTypeId is present. + if (fTypeId == -1) { + return ""; + } + return gTypes[fTypeId]; +} + +const char *MeasureUnit::getSubtype() const { + // We have a type & subtype only if fTypeId is present. + if (fTypeId == -1) { + return ""; + } + return getIdentifier(); +} + +const char *MeasureUnit::getIdentifier() const { + return fImpl ? fImpl->identifier.data() : gSubTypes[getOffset()]; +} + +bool MeasureUnit::operator==(const UObject& other) const { + if (this == &other) { // Same object, equal + return true; + } + if (typeid(*this) != typeid(other)) { // Different types, not equal + return false; + } + const MeasureUnit &rhs = static_cast(other); + return uprv_strcmp(getIdentifier(), rhs.getIdentifier()) == 0; +} + +int32_t MeasureUnit::getAvailable( + MeasureUnit *dest, + int32_t destCapacity, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return 0; + } + if (destCapacity < UPRV_LENGTHOF(gSubTypes)) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return UPRV_LENGTHOF(gSubTypes); + } + int32_t idx = 0; + for (int32_t typeIdx = 0; typeIdx < UPRV_LENGTHOF(gTypes); ++typeIdx) { + int32_t len = gOffsets[typeIdx + 1] - gOffsets[typeIdx]; + for (int32_t subTypeIdx = 0; subTypeIdx < len; ++subTypeIdx) { + dest[idx].setTo(typeIdx, subTypeIdx); + ++idx; + } + } + U_ASSERT(idx == UPRV_LENGTHOF(gSubTypes)); + return UPRV_LENGTHOF(gSubTypes); +} + +int32_t MeasureUnit::getAvailable( + const char *type, + MeasureUnit *dest, + int32_t destCapacity, + UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return 0; + } + int32_t typeIdx = binarySearch(gTypes, 0, UPRV_LENGTHOF(gTypes), type); + if (typeIdx == -1) { + return 0; + } + int32_t len = gOffsets[typeIdx + 1] - gOffsets[typeIdx]; + if (destCapacity < len) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return len; + } + for (int subTypeIdx = 0; subTypeIdx < len; ++subTypeIdx) { + dest[subTypeIdx].setTo(typeIdx, subTypeIdx); + } + return len; +} + +StringEnumeration* MeasureUnit::getAvailableTypes(UErrorCode &errorCode) { + UEnumeration *uenum = uenum_openCharStringsEnumeration( + gTypes, UPRV_LENGTHOF(gTypes), &errorCode); + if (U_FAILURE(errorCode)) { + uenum_close(uenum); + return nullptr; + } + StringEnumeration *result = new UStringEnumeration(uenum); + if (result == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + uenum_close(uenum); + return nullptr; + } + return result; +} + +bool MeasureUnit::findBySubType(StringPiece subType, MeasureUnit* output) { + // Sanity checking kCurrencyOffset and final entry in gOffsets + U_ASSERT(uprv_strcmp(gTypes[kCurrencyOffset], "currency") == 0); + U_ASSERT(gOffsets[UPRV_LENGTHOF(gOffsets) - 1] == UPRV_LENGTHOF(gSubTypes)); + + for (int32_t t = 0; t < UPRV_LENGTHOF(gOffsets) - 1; t++) { + // Skip currency units + if (t == kCurrencyOffset) { + continue; + } + int32_t st = binarySearch(gSubTypes, gOffsets[t], gOffsets[t + 1], subType); + if (st >= 0) { + output->setTo(t, st - gOffsets[t]); + return true; + } + } + return false; +} + +MeasureUnit *MeasureUnit::create(int typeId, int subTypeId, UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + MeasureUnit *result = new MeasureUnit(typeId, subTypeId); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return result; +} + +void MeasureUnit::initTime(const char *timeId) { + int32_t result = binarySearch(gTypes, 0, UPRV_LENGTHOF(gTypes), "duration"); + U_ASSERT(result != -1); + fTypeId = result; + result = binarySearch(gSubTypes, gOffsets[fTypeId], gOffsets[fTypeId + 1], timeId); + U_ASSERT(result != -1); + fSubTypeId = result - gOffsets[fTypeId]; +} + +void MeasureUnit::initCurrency(StringPiece isoCurrency) { + int32_t result = binarySearch(gTypes, 0, UPRV_LENGTHOF(gTypes), "currency"); + U_ASSERT(result != -1); + fTypeId = result; + result = binarySearch( + gSubTypes, gOffsets[fTypeId], gOffsets[fTypeId + 1], isoCurrency); + if (result == -1) { + fImpl = new MeasureUnitImpl(MeasureUnitImpl::forCurrencyCode(isoCurrency)); + if (fImpl) { + fSubTypeId = -1; + return; + } + // malloc error: fall back to the undefined currency + result = binarySearch( + gSubTypes, gOffsets[fTypeId], gOffsets[fTypeId + 1], kDefaultCurrency8); + U_ASSERT(result != -1); + } + fSubTypeId = result - gOffsets[fTypeId]; +} + +void MeasureUnit::setTo(int32_t typeId, int32_t subTypeId) { + fTypeId = typeId; + fSubTypeId = subTypeId; + if (fImpl != nullptr) { + delete fImpl; + fImpl = nullptr; + } +} + +int32_t MeasureUnit::getOffset() const { + if (fTypeId < 0 || fSubTypeId < 0) { + return -1; + } + return gOffsets[fTypeId] + fSubTypeId; +} + +MeasureUnitImpl MeasureUnitImpl::copy(UErrorCode &status) const { + MeasureUnitImpl result; + result.complexity = complexity; + result.identifier.append(identifier, status); + for (int32_t i = 0; i < singleUnits.length(); i++) { + SingleUnitImpl *item = result.singleUnits.emplaceBack(*singleUnits[i]); + if (!item) { + status = U_MEMORY_ALLOCATION_ERROR; + return result; + } + } + return result; +} + +U_NAMESPACE_END + +#endif /* !UNCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/measunit_extra.cpp b/intl/icu/source/i18n/measunit_extra.cpp new file mode 100644 index 0000000000..295d6a8ce8 --- /dev/null +++ b/intl/icu/source/i18n/measunit_extra.cpp @@ -0,0 +1,1260 @@ +// © 2020 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +// Extra functions for MeasureUnit not needed for all clients. +// Separate .o file so that it can be removed for modularity. + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "charstr.h" +#include "cmemory.h" +#include "cstring.h" +#include "measunit_impl.h" +#include "resource.h" +#include "uarrsort.h" +#include "uassert.h" +#include "ucln_in.h" +#include "umutex.h" +#include "unicode/bytestrie.h" +#include "unicode/bytestriebuilder.h" +#include "unicode/localpointer.h" +#include "unicode/stringpiece.h" +#include "unicode/stringtriebuilder.h" +#include "unicode/ures.h" +#include "unicode/ustringtrie.h" +#include "uresimp.h" +#include "util.h" +#include + +U_NAMESPACE_BEGIN + + +namespace { + +// TODO: Propose a new error code for this? +constexpr UErrorCode kUnitIdentifierSyntaxError = U_ILLEGAL_ARGUMENT_ERROR; + +// Trie value offset for SI or binary prefixes. This is big enough to ensure we only +// insert positive integers into the trie. +constexpr int32_t kPrefixOffset = 64; +static_assert(kPrefixOffset + UMEASURE_PREFIX_INTERNAL_MIN_BIN > 0, + "kPrefixOffset is too small for minimum UMeasurePrefix value"); +static_assert(kPrefixOffset + UMEASURE_PREFIX_INTERNAL_MIN_SI > 0, + "kPrefixOffset is too small for minimum UMeasurePrefix value"); + +// Trie value offset for compound parts, e.g. "-per-", "-", "-and-". +constexpr int32_t kCompoundPartOffset = 128; +static_assert(kCompoundPartOffset > kPrefixOffset + UMEASURE_PREFIX_INTERNAL_MAX_BIN, + "Ambiguous token values: prefix tokens are overlapping with CompoundPart tokens"); +static_assert(kCompoundPartOffset > kPrefixOffset + UMEASURE_PREFIX_INTERNAL_MAX_SI, + "Ambiguous token values: prefix tokens are overlapping with CompoundPart tokens"); + +enum CompoundPart { + // Represents "-per-" + COMPOUND_PART_PER = kCompoundPartOffset, + // Represents "-" + COMPOUND_PART_TIMES, + // Represents "-and-" + COMPOUND_PART_AND, +}; + +// Trie value offset for "per-". +constexpr int32_t kInitialCompoundPartOffset = 192; + +enum InitialCompoundPart { + // Represents "per-", the only compound part that can appear at the start of + // an identifier. + INITIAL_COMPOUND_PART_PER = kInitialCompoundPartOffset, +}; + +// Trie value offset for powers like "square-", "cubic-", "pow2-" etc. +constexpr int32_t kPowerPartOffset = 256; + +enum PowerPart { + POWER_PART_P2 = kPowerPartOffset + 2, + POWER_PART_P3, + POWER_PART_P4, + POWER_PART_P5, + POWER_PART_P6, + POWER_PART_P7, + POWER_PART_P8, + POWER_PART_P9, + POWER_PART_P10, + POWER_PART_P11, + POWER_PART_P12, + POWER_PART_P13, + POWER_PART_P14, + POWER_PART_P15, +}; + +// Trie value offset for simple units, e.g. "gram", "nautical-mile", +// "fluid-ounce-imperial". +constexpr int32_t kSimpleUnitOffset = 512; + +const struct UnitPrefixStrings { + const char* const string; + UMeasurePrefix value; +} gUnitPrefixStrings[] = { + // SI prefixes + { "yotta", UMEASURE_PREFIX_YOTTA }, + { "zetta", UMEASURE_PREFIX_ZETTA }, + { "exa", UMEASURE_PREFIX_EXA }, + { "peta", UMEASURE_PREFIX_PETA }, + { "tera", UMEASURE_PREFIX_TERA }, + { "giga", UMEASURE_PREFIX_GIGA }, + { "mega", UMEASURE_PREFIX_MEGA }, + { "kilo", UMEASURE_PREFIX_KILO }, + { "hecto", UMEASURE_PREFIX_HECTO }, + { "deka", UMEASURE_PREFIX_DEKA }, + { "deci", UMEASURE_PREFIX_DECI }, + { "centi", UMEASURE_PREFIX_CENTI }, + { "milli", UMEASURE_PREFIX_MILLI }, + { "micro", UMEASURE_PREFIX_MICRO }, + { "nano", UMEASURE_PREFIX_NANO }, + { "pico", UMEASURE_PREFIX_PICO }, + { "femto", UMEASURE_PREFIX_FEMTO }, + { "atto", UMEASURE_PREFIX_ATTO }, + { "zepto", UMEASURE_PREFIX_ZEPTO }, + { "yocto", UMEASURE_PREFIX_YOCTO }, + // Binary prefixes + { "yobi", UMEASURE_PREFIX_YOBI }, + { "zebi", UMEASURE_PREFIX_ZEBI }, + { "exbi", UMEASURE_PREFIX_EXBI }, + { "pebi", UMEASURE_PREFIX_PEBI }, + { "tebi", UMEASURE_PREFIX_TEBI }, + { "gibi", UMEASURE_PREFIX_GIBI }, + { "mebi", UMEASURE_PREFIX_MEBI }, + { "kibi", UMEASURE_PREFIX_KIBI }, +}; + +/** + * A ResourceSink that collects simple unit identifiers from the keys of the + * convertUnits table into an array, and adds these values to a TrieBuilder, + * with associated values being their index into this array plus a specified + * offset. + * + * Example code: + * + * UErrorCode status = U_ZERO_ERROR; + * BytesTrieBuilder b(status); + * int32_t ARR_SIZE = 200; + * const char *unitIdentifiers[ARR_SIZE]; + * int32_t *unitCategories[ARR_SIZE]; + * SimpleUnitIdentifiersSink identifierSink(gSerializedUnitCategoriesTrie, unitIdentifiers, + * unitCategories, ARR_SIZE, b, kTrieValueOffset); + * LocalUResourceBundlePointer unitsBundle(ures_openDirect(nullptr, "units", &status)); + * ures_getAllItemsWithFallback(unitsBundle.getAlias(), "convertUnits", identifierSink, status); + */ +class SimpleUnitIdentifiersSink : public icu::ResourceSink { + public: + /** + * Constructor. + * @param quantitiesTrieData The data for constructing a quantitiesTrie, + * which maps from a simple unit identifier to an index into the + * gCategories array. + * @param out Array of char* to which pointers to the simple unit + * identifiers will be saved. (Does not take ownership.) + * @param outCategories Array of int32_t to which category indexes will be + * saved: this corresponds to simple unit IDs saved to `out`, mapping + * from the ID to the value produced by the quantitiesTrie (which is an + * index into the gCategories array). + * @param outSize The size of `out` and `outCategories`. + * @param trieBuilder The trie builder to which the simple unit identifier + * should be added. The trie builder must outlive this resource sink. + * @param trieValueOffset This is added to the index of the identifier in + * the `out` array, before adding to `trieBuilder` as the value + * associated with the identifier. + */ + explicit SimpleUnitIdentifiersSink(StringPiece quantitiesTrieData, const char **out, + int32_t *outCategories, int32_t outSize, + BytesTrieBuilder &trieBuilder, int32_t trieValueOffset) + : outArray(out), outCategories(outCategories), outSize(outSize), trieBuilder(trieBuilder), + trieValueOffset(trieValueOffset), quantitiesTrieData(quantitiesTrieData), outIndex(0) {} + + /** + * Adds the table keys found in value to the output vector. + * @param key The key of the resource passed to `value`: the second + * parameter of the ures_getAllItemsWithFallback() call. + * @param value Should be a ResourceTable value, if + * ures_getAllItemsWithFallback() was called correctly for this sink. + * @param noFallback Ignored. + * @param status The standard ICU error code output parameter. + */ + void put(const char * /*key*/, ResourceValue &value, UBool /*noFallback*/, UErrorCode &status) override { + ResourceTable table = value.getTable(status); + if (U_FAILURE(status)) return; + + if (outIndex + table.getSize() > outSize) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + BytesTrie quantitiesTrie(quantitiesTrieData.data()); + + // Collect keys from the table resource. + const char *simpleUnitID; + for (int32_t i = 0; table.getKeyAndValue(i, simpleUnitID, value); ++i) { + U_ASSERT(i < table.getSize()); + U_ASSERT(outIndex < outSize); + if (uprv_strcmp(simpleUnitID, "kilogram") == 0) { + // For parsing, we use "gram", the prefixless metric mass unit. We + // thus ignore the SI Base Unit of Mass: it exists due to being the + // mass conversion target unit, but not needed for MeasureUnit + // parsing. + continue; + } + outArray[outIndex] = simpleUnitID; + trieBuilder.add(simpleUnitID, trieValueOffset + outIndex, status); + + // Find the base target unit for this simple unit + ResourceTable table = value.getTable(status); + if (U_FAILURE(status)) { return; } + if (!table.findValue("target", value)) { + status = U_INVALID_FORMAT_ERROR; + break; + } + int32_t len; + const char16_t* uTarget = value.getString(len, status); + CharString target; + target.appendInvariantChars(uTarget, len, status); + if (U_FAILURE(status)) { return; } + quantitiesTrie.reset(); + UStringTrieResult result = quantitiesTrie.next(target.data(), target.length()); + if (!USTRINGTRIE_HAS_VALUE(result)) { + status = U_INVALID_FORMAT_ERROR; + break; + } + outCategories[outIndex] = quantitiesTrie.getValue(); + + outIndex++; + } + } + + private: + const char **outArray; + int32_t *outCategories; + int32_t outSize; + BytesTrieBuilder &trieBuilder; + int32_t trieValueOffset; + + StringPiece quantitiesTrieData; + + int32_t outIndex; +}; + +/** + * A ResourceSink that collects information from `unitQuantities` in the `units` + * resource to provide key->value lookups from base unit to category, as well as + * preserving ordering information for these categories. See `units.txt`. + * + * For example: "kilogram" -> "mass", "meter-per-second" -> "speed". + * + * In C++ unitQuantity values are collected in order into a char16_t* array, while + * unitQuantity keys are added added to a TrieBuilder, with associated values + * being the index into the aforementioned char16_t* array. + */ +class CategoriesSink : public icu::ResourceSink { + public: + /** + * Constructor. + * @param out Array of char16_t* to which unitQuantity values will be saved. + * The pointers returned not owned: they point directly at the resource + * strings in static memory. + * @param outSize The size of the `out` array. + * @param trieBuilder The trie builder to which the keys (base units) of + * each unitQuantity will be added, each with value being the offset + * into `out`. + */ + explicit CategoriesSink(const char16_t **out, int32_t &outSize, BytesTrieBuilder &trieBuilder) + : outQuantitiesArray(out), outSize(outSize), trieBuilder(trieBuilder), outIndex(0) {} + + void put(const char * /*key*/, ResourceValue &value, UBool /*noFallback*/, UErrorCode &status) override { + ResourceArray array = value.getArray(status); + if (U_FAILURE(status)) { + return; + } + + if (outIndex + array.getSize() > outSize) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + for (int32_t i = 0; array.getValue(i, value); ++i) { + U_ASSERT(outIndex < outSize); + ResourceTable table = value.getTable(status); + if (U_FAILURE(status)) { + return; + } + if (table.getSize() != 1) { + status = U_INVALID_FORMAT_ERROR; + return; + } + const char *key; + table.getKeyAndValue(0, key, value); + int32_t uTmpLen; + outQuantitiesArray[outIndex] = value.getString(uTmpLen, status); + trieBuilder.add(key, outIndex, status); + outIndex++; + } + } + + private: + const char16_t **outQuantitiesArray; + int32_t &outSize; + BytesTrieBuilder &trieBuilder; + + int32_t outIndex; +}; + +icu::UInitOnce gUnitExtrasInitOnce {}; + +// Array of simple unit IDs. +// +// The array memory itself is owned by this pointer, but the individual char* in +// that array point at static memory. (Note that these char* are also returned +// by SingleUnitImpl::getSimpleUnitID().) +const char **gSimpleUnits = nullptr; + +// Maps from the value associated with each simple unit ID to an index into the +// gCategories array. +int32_t *gSimpleUnitCategories = nullptr; + +char *gSerializedUnitExtrasStemTrie = nullptr; + +// Array of char16_t* pointing at the unit categories (aka "quantities", aka +// "types"), as found in the `unitQuantities` resource. The array memory itself +// is owned by this pointer, but the individual char16_t* in that array point at +// static memory. +const char16_t **gCategories = nullptr; +// Number of items in `gCategories`. +int32_t gCategoriesCount = 0; +// Serialized BytesTrie for mapping from base units to indices into gCategories. +char *gSerializedUnitCategoriesTrie = nullptr; + +UBool U_CALLCONV cleanupUnitExtras() { + uprv_free(gSerializedUnitCategoriesTrie); + gSerializedUnitCategoriesTrie = nullptr; + uprv_free(gCategories); + gCategories = nullptr; + uprv_free(gSerializedUnitExtrasStemTrie); + gSerializedUnitExtrasStemTrie = nullptr; + uprv_free(gSimpleUnitCategories); + gSimpleUnitCategories = nullptr; + uprv_free(gSimpleUnits); + gSimpleUnits = nullptr; + gUnitExtrasInitOnce.reset(); + return true; +} + +void U_CALLCONV initUnitExtras(UErrorCode& status) { + ucln_i18n_registerCleanup(UCLN_I18N_UNIT_EXTRAS, cleanupUnitExtras); + LocalUResourceBundlePointer unitsBundle(ures_openDirect(nullptr, "units", &status)); + + // Collect unitQuantities information into gSerializedUnitCategoriesTrie and gCategories. + const char *CATEGORY_TABLE_NAME = "unitQuantities"; + LocalUResourceBundlePointer unitQuantities( + ures_getByKey(unitsBundle.getAlias(), CATEGORY_TABLE_NAME, nullptr, &status)); + if (U_FAILURE(status)) { return; } + gCategoriesCount = unitQuantities.getAlias()->fSize; + size_t quantitiesMallocSize = sizeof(char16_t *) * gCategoriesCount; + gCategories = static_cast(uprv_malloc(quantitiesMallocSize)); + if (gCategories == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memset(gCategories, 0, quantitiesMallocSize); + BytesTrieBuilder quantitiesBuilder(status); + CategoriesSink categoriesSink(gCategories, gCategoriesCount, quantitiesBuilder); + ures_getAllItemsWithFallback(unitsBundle.getAlias(), CATEGORY_TABLE_NAME, categoriesSink, status); + StringPiece resultQuantities = quantitiesBuilder.buildStringPiece(USTRINGTRIE_BUILD_FAST, status); + if (U_FAILURE(status)) { return; } + // Copy the result into the global constant pointer + size_t numBytesQuantities = resultQuantities.length(); + gSerializedUnitCategoriesTrie = static_cast(uprv_malloc(numBytesQuantities)); + if (gSerializedUnitCategoriesTrie == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memcpy(gSerializedUnitCategoriesTrie, resultQuantities.data(), numBytesQuantities); + + // Build the BytesTrie that Parser needs for parsing unit identifiers. + + BytesTrieBuilder b(status); + if (U_FAILURE(status)) { return; } + + // Add SI and binary prefixes + for (const auto& unitPrefixInfo : gUnitPrefixStrings) { + b.add(unitPrefixInfo.string, unitPrefixInfo.value + kPrefixOffset, status); + } + if (U_FAILURE(status)) { return; } + + // Add syntax parts (compound, power prefixes) + b.add("-per-", COMPOUND_PART_PER, status); + b.add("-", COMPOUND_PART_TIMES, status); + b.add("-and-", COMPOUND_PART_AND, status); + b.add("per-", INITIAL_COMPOUND_PART_PER, status); + b.add("square-", POWER_PART_P2, status); + b.add("cubic-", POWER_PART_P3, status); + b.add("pow2-", POWER_PART_P2, status); + b.add("pow3-", POWER_PART_P3, status); + b.add("pow4-", POWER_PART_P4, status); + b.add("pow5-", POWER_PART_P5, status); + b.add("pow6-", POWER_PART_P6, status); + b.add("pow7-", POWER_PART_P7, status); + b.add("pow8-", POWER_PART_P8, status); + b.add("pow9-", POWER_PART_P9, status); + b.add("pow10-", POWER_PART_P10, status); + b.add("pow11-", POWER_PART_P11, status); + b.add("pow12-", POWER_PART_P12, status); + b.add("pow13-", POWER_PART_P13, status); + b.add("pow14-", POWER_PART_P14, status); + b.add("pow15-", POWER_PART_P15, status); + if (U_FAILURE(status)) { return; } + + // Add sanctioned simple units by offset: simple units all have entries in + // units/convertUnits resources. + LocalUResourceBundlePointer convertUnits( + ures_getByKey(unitsBundle.getAlias(), "convertUnits", nullptr, &status)); + if (U_FAILURE(status)) { return; } + + // Allocate enough space: with identifierSink below skipping kilogram, we're + // probably allocating one more than needed. + int32_t simpleUnitsCount = convertUnits.getAlias()->fSize; + int32_t arrayMallocSize = sizeof(char *) * simpleUnitsCount; + gSimpleUnits = static_cast(uprv_malloc(arrayMallocSize)); + if (gSimpleUnits == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memset(gSimpleUnits, 0, arrayMallocSize); + arrayMallocSize = sizeof(int32_t) * simpleUnitsCount; + gSimpleUnitCategories = static_cast(uprv_malloc(arrayMallocSize)); + if (gSimpleUnitCategories == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memset(gSimpleUnitCategories, 0, arrayMallocSize); + + // Populate gSimpleUnits and build the associated trie. + SimpleUnitIdentifiersSink identifierSink(resultQuantities, gSimpleUnits, gSimpleUnitCategories, + simpleUnitsCount, b, kSimpleUnitOffset); + ures_getAllItemsWithFallback(unitsBundle.getAlias(), "convertUnits", identifierSink, status); + + // Build the CharsTrie + // TODO: Use SLOW or FAST here? + StringPiece result = b.buildStringPiece(USTRINGTRIE_BUILD_FAST, status); + if (U_FAILURE(status)) { return; } + + // Copy the result into the global constant pointer + size_t numBytes = result.length(); + gSerializedUnitExtrasStemTrie = static_cast(uprv_malloc(numBytes)); + if (gSerializedUnitExtrasStemTrie == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memcpy(gSerializedUnitExtrasStemTrie, result.data(), numBytes); +} + +class Token { +public: + Token(int32_t match) : fMatch(match) {} + + enum Type { + TYPE_UNDEFINED, + TYPE_PREFIX, + // Token type for "-per-", "-", and "-and-". + TYPE_COMPOUND_PART, + // Token type for "per-". + TYPE_INITIAL_COMPOUND_PART, + TYPE_POWER_PART, + TYPE_SIMPLE_UNIT, + }; + + // Calling getType() is invalid, resulting in an assertion failure, if Token + // value isn't positive. + Type getType() const { + U_ASSERT(fMatch > 0); + if (fMatch < kCompoundPartOffset) { + return TYPE_PREFIX; + } + if (fMatch < kInitialCompoundPartOffset) { + return TYPE_COMPOUND_PART; + } + if (fMatch < kPowerPartOffset) { + return TYPE_INITIAL_COMPOUND_PART; + } + if (fMatch < kSimpleUnitOffset) { + return TYPE_POWER_PART; + } + return TYPE_SIMPLE_UNIT; + } + + UMeasurePrefix getUnitPrefix() const { + U_ASSERT(getType() == TYPE_PREFIX); + return static_cast(fMatch - kPrefixOffset); + } + + // Valid only for tokens with type TYPE_COMPOUND_PART. + int32_t getMatch() const { + U_ASSERT(getType() == TYPE_COMPOUND_PART); + return fMatch; + } + + int32_t getInitialCompoundPart() const { + // Even if there is only one InitialCompoundPart value, we have this + // function for the simplicity of code consistency. + U_ASSERT(getType() == TYPE_INITIAL_COMPOUND_PART); + // Defensive: if this assert fails, code using this function also needs + // to change. + U_ASSERT(fMatch == INITIAL_COMPOUND_PART_PER); + return fMatch; + } + + int8_t getPower() const { + U_ASSERT(getType() == TYPE_POWER_PART); + return static_cast(fMatch - kPowerPartOffset); + } + + int32_t getSimpleUnitIndex() const { + U_ASSERT(getType() == TYPE_SIMPLE_UNIT); + return fMatch - kSimpleUnitOffset; + } + +private: + int32_t fMatch; +}; + +class Parser { +public: + /** + * Factory function for parsing the given identifier. + * + * @param source The identifier to parse. This function does not make a copy + * of source: the underlying string that source points at, must outlive the + * parser. + * @param status ICU error code. + */ + static Parser from(StringPiece source, UErrorCode& status) { + if (U_FAILURE(status)) { + return Parser(); + } + umtx_initOnce(gUnitExtrasInitOnce, &initUnitExtras, status); + if (U_FAILURE(status)) { + return Parser(); + } + return Parser(source); + } + + MeasureUnitImpl parse(UErrorCode& status) { + MeasureUnitImpl result; + + if (U_FAILURE(status)) { + return result; + } + if (fSource.empty()) { + // The dimenionless unit: nothing to parse. leave result as is. + return result; + } + + while (hasNext()) { + bool sawAnd = false; + + SingleUnitImpl singleUnit = nextSingleUnit(sawAnd, status); + if (U_FAILURE(status)) { + return result; + } + + bool added = result.appendSingleUnit(singleUnit, status); + if (U_FAILURE(status)) { + return result; + } + + if (sawAnd && !added) { + // Two similar units are not allowed in a mixed unit. + status = kUnitIdentifierSyntaxError; + return result; + } + + if (result.singleUnits.length() >= 2) { + // nextSingleUnit fails appropriately for "per" and "and" in the + // same identifier. It doesn't fail for other compound units + // (COMPOUND_PART_TIMES). Consequently we take care of that + // here. + UMeasureUnitComplexity complexity = + sawAnd ? UMEASURE_UNIT_MIXED : UMEASURE_UNIT_COMPOUND; + if (result.singleUnits.length() == 2) { + // After appending two singleUnits, the complexity will be `UMEASURE_UNIT_COMPOUND` + U_ASSERT(result.complexity == UMEASURE_UNIT_COMPOUND); + result.complexity = complexity; + } else if (result.complexity != complexity) { + // Can't have mixed compound units + status = kUnitIdentifierSyntaxError; + return result; + } + } + } + + return result; + } + +private: + // Tracks parser progress: the offset into fSource. + int32_t fIndex = 0; + + // Since we're not owning this memory, whatever is passed to the constructor + // should live longer than this Parser - and the parser shouldn't return any + // references to that string. + StringPiece fSource; + BytesTrie fTrie; + + // Set to true when we've seen a "-per-" or a "per-", after which all units + // are in the denominator. Until we find an "-and-", at which point the + // identifier is invalid pending TODO(CLDR-13701). + bool fAfterPer = false; + + Parser() : fSource(""), fTrie(u"") {} + + Parser(StringPiece source) + : fSource(source), fTrie(gSerializedUnitExtrasStemTrie) {} + + inline bool hasNext() const { + return fIndex < fSource.length(); + } + + // Returns the next Token parsed from fSource, advancing fIndex to the end + // of that token in fSource. In case of U_FAILURE(status), the token + // returned will cause an abort if getType() is called on it. + Token nextToken(UErrorCode& status) { + fTrie.reset(); + int32_t match = -1; + // Saves the position in the fSource string for the end of the most + // recent matching token. + int32_t previ = -1; + // Find the longest token that matches a value in the trie: + while (fIndex < fSource.length()) { + auto result = fTrie.next(fSource.data()[fIndex++]); + if (result == USTRINGTRIE_NO_MATCH) { + break; + } else if (result == USTRINGTRIE_NO_VALUE) { + continue; + } + U_ASSERT(USTRINGTRIE_HAS_VALUE(result)); + match = fTrie.getValue(); + previ = fIndex; + if (result == USTRINGTRIE_FINAL_VALUE) { + break; + } + U_ASSERT(result == USTRINGTRIE_INTERMEDIATE_VALUE); + // continue; + } + + if (match < 0) { + status = kUnitIdentifierSyntaxError; + } else { + fIndex = previ; + } + return Token(match); + } + + /** + * Returns the next "single unit" via result. + * + * If a "-per-" was parsed, the result will have appropriate negative + * dimensionality. + * + * Returns an error if we parse both compound units and "-and-", since mixed + * compound units are not yet supported - TODO(CLDR-13701). + * + * @param result Will be overwritten by the result, if status shows success. + * @param sawAnd If an "-and-" was parsed prior to finding the "single + * unit", sawAnd is set to true. If not, it is left as is. + * @param status ICU error code. + */ + SingleUnitImpl nextSingleUnit(bool &sawAnd, UErrorCode &status) { + SingleUnitImpl result; + if (U_FAILURE(status)) { + return result; + } + + // state: + // 0 = no tokens seen yet (will accept power, SI or binary prefix, or simple unit) + // 1 = power token seen (will not accept another power token) + // 2 = SI or binary prefix token seen (will not accept a power, or SI or binary prefix token) + int32_t state = 0; + + bool atStart = fIndex == 0; + Token token = nextToken(status); + if (U_FAILURE(status)) { + return result; + } + + if (atStart) { + // Identifiers optionally start with "per-". + if (token.getType() == Token::TYPE_INITIAL_COMPOUND_PART) { + U_ASSERT(token.getInitialCompoundPart() == INITIAL_COMPOUND_PART_PER); + fAfterPer = true; + result.dimensionality = -1; + + token = nextToken(status); + if (U_FAILURE(status)) { + return result; + } + } + } else { + // All other SingleUnit's are separated from previous SingleUnit's + // via a compound part: + if (token.getType() != Token::TYPE_COMPOUND_PART) { + status = kUnitIdentifierSyntaxError; + return result; + } + + switch (token.getMatch()) { + case COMPOUND_PART_PER: + if (sawAnd) { + // Mixed compound units not yet supported, + // TODO(CLDR-13701). + status = kUnitIdentifierSyntaxError; + return result; + } + fAfterPer = true; + result.dimensionality = -1; + break; + + case COMPOUND_PART_TIMES: + if (fAfterPer) { + result.dimensionality = -1; + } + break; + + case COMPOUND_PART_AND: + if (fAfterPer) { + // Can't start with "-and-", and mixed compound units + // not yet supported, TODO(CLDR-13701). + status = kUnitIdentifierSyntaxError; + return result; + } + sawAnd = true; + break; + } + + token = nextToken(status); + if (U_FAILURE(status)) { + return result; + } + } + + // Read tokens until we have a complete SingleUnit or we reach the end. + while (true) { + switch (token.getType()) { + case Token::TYPE_POWER_PART: + if (state > 0) { + status = kUnitIdentifierSyntaxError; + return result; + } + result.dimensionality *= token.getPower(); + state = 1; + break; + + case Token::TYPE_PREFIX: + if (state > 1) { + status = kUnitIdentifierSyntaxError; + return result; + } + result.unitPrefix = token.getUnitPrefix(); + state = 2; + break; + + case Token::TYPE_SIMPLE_UNIT: + result.index = token.getSimpleUnitIndex(); + return result; + + default: + status = kUnitIdentifierSyntaxError; + return result; + } + + if (!hasNext()) { + // We ran out of tokens before finding a complete single unit. + status = kUnitIdentifierSyntaxError; + return result; + } + token = nextToken(status); + if (U_FAILURE(status)) { + return result; + } + } + + return result; + } +}; + +// Sorting function wrapping SingleUnitImpl::compareTo for use with uprv_sortArray. +int32_t U_CALLCONV +compareSingleUnits(const void* /*context*/, const void* left, const void* right) { + auto realLeft = static_cast(left); + auto realRight = static_cast(right); + return (*realLeft)->compareTo(**realRight); +} + +// Returns an index into the gCategories array, for the "unitQuantity" (aka +// "type" or "category") associated with the given base unit identifier. Returns +// -1 on failure, together with U_UNSUPPORTED_ERROR. +int32_t getUnitCategoryIndex(BytesTrie &trie, StringPiece baseUnitIdentifier, UErrorCode &status) { + UStringTrieResult result = trie.reset().next(baseUnitIdentifier.data(), baseUnitIdentifier.length()); + if (!USTRINGTRIE_HAS_VALUE(result)) { + status = U_UNSUPPORTED_ERROR; + return -1; + } + + return trie.getValue(); +} + +} // namespace + +U_CAPI int32_t U_EXPORT2 +umeas_getPrefixPower(UMeasurePrefix unitPrefix) { + if (unitPrefix >= UMEASURE_PREFIX_INTERNAL_MIN_BIN && + unitPrefix <= UMEASURE_PREFIX_INTERNAL_MAX_BIN) { + return unitPrefix - UMEASURE_PREFIX_INTERNAL_ONE_BIN; + } + U_ASSERT(unitPrefix >= UMEASURE_PREFIX_INTERNAL_MIN_SI && + unitPrefix <= UMEASURE_PREFIX_INTERNAL_MAX_SI); + return unitPrefix - UMEASURE_PREFIX_ONE; +} + +U_CAPI int32_t U_EXPORT2 +umeas_getPrefixBase(UMeasurePrefix unitPrefix) { + if (unitPrefix >= UMEASURE_PREFIX_INTERNAL_MIN_BIN && + unitPrefix <= UMEASURE_PREFIX_INTERNAL_MAX_BIN) { + return 1024; + } + U_ASSERT(unitPrefix >= UMEASURE_PREFIX_INTERNAL_MIN_SI && + unitPrefix <= UMEASURE_PREFIX_INTERNAL_MAX_SI); + return 10; +} + +CharString U_I18N_API getUnitQuantity(const MeasureUnitImpl &baseMeasureUnitImpl, UErrorCode &status) { + CharString result; + MeasureUnitImpl baseUnitImpl = baseMeasureUnitImpl.copy(status); + UErrorCode localStatus = U_ZERO_ERROR; + umtx_initOnce(gUnitExtrasInitOnce, &initUnitExtras, status); + if (U_FAILURE(status)) { + return result; + } + BytesTrie trie(gSerializedUnitCategoriesTrie); + + baseUnitImpl.serialize(status); + StringPiece identifier = baseUnitImpl.identifier.data(); + int32_t idx = getUnitCategoryIndex(trie, identifier, localStatus); + if (U_FAILURE(status)) { + return result; + } + + // In case the base unit identifier did not match any entry. + if (U_FAILURE(localStatus)) { + localStatus = U_ZERO_ERROR; + baseUnitImpl.takeReciprocal(status); + baseUnitImpl.serialize(status); + identifier.set(baseUnitImpl.identifier.data()); + idx = getUnitCategoryIndex(trie, identifier, localStatus); + + if (U_FAILURE(status)) { + return result; + } + } + + // In case the reciprocal of the base unit identifier did not match any entry. + MeasureUnitImpl simplifiedUnit = baseMeasureUnitImpl.copyAndSimplify(status); + if (U_FAILURE(status)) { + return result; + } + if (U_FAILURE(localStatus)) { + localStatus = U_ZERO_ERROR; + simplifiedUnit.serialize(status); + identifier.set(simplifiedUnit.identifier.data()); + idx = getUnitCategoryIndex(trie, identifier, localStatus); + + if (U_FAILURE(status)) { + return result; + } + } + + // In case the simplified base unit identifier did not match any entry. + if (U_FAILURE(localStatus)) { + localStatus = U_ZERO_ERROR; + simplifiedUnit.takeReciprocal(status); + simplifiedUnit.serialize(status); + identifier.set(simplifiedUnit.identifier.data()); + idx = getUnitCategoryIndex(trie, identifier, localStatus); + + if (U_FAILURE(status)) { + return result; + } + } + + // If there is no match at all, throw an exception. + if (U_FAILURE(localStatus)) { + status = U_INVALID_FORMAT_ERROR; + return result; + } + + if (idx < 0 || idx >= gCategoriesCount) { + status = U_INVALID_FORMAT_ERROR; + return result; + } + + result.appendInvariantChars(gCategories[idx], u_strlen(gCategories[idx]), status); + return result; +} + +// In ICU4J, this is MeasureUnit.getSingleUnitImpl(). +SingleUnitImpl SingleUnitImpl::forMeasureUnit(const MeasureUnit& measureUnit, UErrorCode& status) { + MeasureUnitImpl temp; + const MeasureUnitImpl& impl = MeasureUnitImpl::forMeasureUnit(measureUnit, temp, status); + if (U_FAILURE(status)) { + return {}; + } + if (impl.singleUnits.length() == 0) { + return {}; + } + if (impl.singleUnits.length() == 1) { + return *impl.singleUnits[0]; + } + status = U_ILLEGAL_ARGUMENT_ERROR; + return {}; +} + +MeasureUnit SingleUnitImpl::build(UErrorCode& status) const { + MeasureUnitImpl temp; + temp.appendSingleUnit(*this, status); + // TODO(icu-units#28): the MeasureUnitImpl::build() method uses + // findBySubtype, which is relatively slow. + // - At the time of loading the simple unit IDs, we could also save a + // mapping to the builtin MeasureUnit type and subtype they correspond to. + // - This method could then check dimensionality and index, and if both are + // 1, directly return MeasureUnit instances very quickly. + return std::move(temp).build(status); +} + +const char *SingleUnitImpl::getSimpleUnitID() const { + return gSimpleUnits[index]; +} + +void SingleUnitImpl::appendNeutralIdentifier(CharString &result, UErrorCode &status) const UPRV_NO_SANITIZE_UNDEFINED { + int32_t absPower = std::abs(this->dimensionality); + + U_ASSERT(absPower > 0); // "this function does not support the dimensionless single units"; + + if (absPower == 1) { + // no-op + } else if (absPower == 2) { + result.append(StringPiece("square-"), status); + } else if (absPower == 3) { + result.append(StringPiece("cubic-"), status); + } else if (absPower <= 15) { + result.append(StringPiece("pow"), status); + result.appendNumber(absPower, status); + result.append(StringPiece("-"), status); + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; // Unit Identifier Syntax Error + return; + } + + if (U_FAILURE(status)) { + return; + } + + if (this->unitPrefix != UMEASURE_PREFIX_ONE) { + bool found = false; + for (const auto &unitPrefixInfo : gUnitPrefixStrings) { + // TODO: consider using binary search? If we do this, add a unit + // test to ensure gUnitPrefixStrings is sorted? + if (unitPrefixInfo.value == this->unitPrefix) { + result.append(unitPrefixInfo.string, status); + found = true; + break; + } + } + if (!found) { + status = U_UNSUPPORTED_ERROR; + return; + } + } + + result.append(StringPiece(this->getSimpleUnitID()), status); +} + +int32_t SingleUnitImpl::getUnitCategoryIndex() const { + return gSimpleUnitCategories[index]; +} + +MeasureUnitImpl::MeasureUnitImpl(const SingleUnitImpl &singleUnit, UErrorCode &status) { + this->appendSingleUnit(singleUnit, status); +} + +MeasureUnitImpl MeasureUnitImpl::forIdentifier(StringPiece identifier, UErrorCode& status) { + return Parser::from(identifier, status).parse(status); +} + +const MeasureUnitImpl& MeasureUnitImpl::forMeasureUnit( + const MeasureUnit& measureUnit, MeasureUnitImpl& memory, UErrorCode& status) { + if (measureUnit.fImpl) { + return *measureUnit.fImpl; + } else { + memory = Parser::from(measureUnit.getIdentifier(), status).parse(status); + return memory; + } +} + +MeasureUnitImpl MeasureUnitImpl::forMeasureUnitMaybeCopy( + const MeasureUnit& measureUnit, UErrorCode& status) { + if (measureUnit.fImpl) { + return measureUnit.fImpl->copy(status); + } else { + return Parser::from(measureUnit.getIdentifier(), status).parse(status); + } +} + +void MeasureUnitImpl::takeReciprocal(UErrorCode& /*status*/) { + identifier.clear(); + for (int32_t i = 0; i < singleUnits.length(); i++) { + singleUnits[i]->dimensionality *= -1; + } +} + +MeasureUnitImpl MeasureUnitImpl::copyAndSimplify(UErrorCode &status) const { + MeasureUnitImpl result; + for (int32_t i = 0; i < singleUnits.length(); i++) { + const SingleUnitImpl &singleUnit = *this->singleUnits[i]; + + // The following `for` loop will cause time complexity to be O(n^2). + // However, n is very small (number of units, generally, at maximum equal to 10) + bool unitExist = false; + for (int32_t j = 0; j < result.singleUnits.length(); j++) { + if (uprv_strcmp(result.singleUnits[j]->getSimpleUnitID(), singleUnit.getSimpleUnitID()) == + 0 && + result.singleUnits[j]->unitPrefix == singleUnit.unitPrefix) { + unitExist = true; + result.singleUnits[j]->dimensionality = + result.singleUnits[j]->dimensionality + singleUnit.dimensionality; + break; + } + } + + if (!unitExist) { + result.appendSingleUnit(singleUnit, status); + } + } + + return result; +} + +bool MeasureUnitImpl::appendSingleUnit(const SingleUnitImpl &singleUnit, UErrorCode &status) { + identifier.clear(); + + if (singleUnit.isDimensionless()) { + // Do not append dimensionless units. + return false; + } + + // Find a similar unit that already exists, to attempt to coalesce + SingleUnitImpl *oldUnit = nullptr; + for (int32_t i = 0; i < this->singleUnits.length(); i++) { + auto *candidate = this->singleUnits[i]; + if (candidate->isCompatibleWith(singleUnit)) { + oldUnit = candidate; + } + } + + if (oldUnit) { + // Both dimensionalities will be positive, or both will be negative, by + // virtue of isCompatibleWith(). + oldUnit->dimensionality += singleUnit.dimensionality; + + return false; + } + + // Add a copy of singleUnit + // NOTE: MaybeStackVector::emplaceBackAndCheckErrorCode creates new copy of singleUnit. + this->singleUnits.emplaceBackAndCheckErrorCode(status, singleUnit); + if (U_FAILURE(status)) { + return false; + } + + // If the MeasureUnitImpl is `UMEASURE_UNIT_SINGLE` and after the appending a unit, the `singleUnits` + // contains more than one. thus means the complexity should be `UMEASURE_UNIT_COMPOUND` + if (this->singleUnits.length() > 1 && + this->complexity == UMeasureUnitComplexity::UMEASURE_UNIT_SINGLE) { + this->complexity = UMeasureUnitComplexity::UMEASURE_UNIT_COMPOUND; + } + + return true; +} + +MaybeStackVector +MeasureUnitImpl::extractIndividualUnitsWithIndices(UErrorCode &status) const { + MaybeStackVector result; + + if (this->complexity != UMeasureUnitComplexity::UMEASURE_UNIT_MIXED) { + result.emplaceBackAndCheckErrorCode(status, 0, *this, status); + return result; + } + + for (int32_t i = 0; i < singleUnits.length(); ++i) { + result.emplaceBackAndCheckErrorCode(status, i, *singleUnits[i], status); + if (U_FAILURE(status)) { + return result; + } + } + + return result; +} + +/** + * Normalize a MeasureUnitImpl and generate the identifier string in place. + */ +void MeasureUnitImpl::serialize(UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + + if (this->singleUnits.length() == 0) { + // Dimensionless, constructed by the default constructor. + return; + } + + if (this->complexity == UMEASURE_UNIT_COMPOUND) { + // Note: don't sort a MIXED unit + uprv_sortArray(this->singleUnits.getAlias(), this->singleUnits.length(), + sizeof(this->singleUnits[0]), compareSingleUnits, nullptr, false, &status); + if (U_FAILURE(status)) { + return; + } + } + + CharString result; + bool beforePer = true; + bool firstTimeNegativeDimension = false; + for (int32_t i = 0; i < this->singleUnits.length(); i++) { + if (beforePer && (*this->singleUnits[i]).dimensionality < 0) { + beforePer = false; + firstTimeNegativeDimension = true; + } else if ((*this->singleUnits[i]).dimensionality < 0) { + firstTimeNegativeDimension = false; + } + + if (U_FAILURE(status)) { + return; + } + + if (this->complexity == UMeasureUnitComplexity::UMEASURE_UNIT_MIXED) { + if (result.length() != 0) { + result.append(StringPiece("-and-"), status); + } + } else { + if (firstTimeNegativeDimension) { + if (result.length() == 0) { + result.append(StringPiece("per-"), status); + } else { + result.append(StringPiece("-per-"), status); + } + } else { + if (result.length() != 0) { + result.append(StringPiece("-"), status); + } + } + } + + this->singleUnits[i]->appendNeutralIdentifier(result, status); + } + + this->identifier = CharString(result, status); +} + +MeasureUnit MeasureUnitImpl::build(UErrorCode& status) && { + this->serialize(status); + return MeasureUnit(std::move(*this)); +} + +MeasureUnit MeasureUnit::forIdentifier(StringPiece identifier, UErrorCode& status) { + return Parser::from(identifier, status).parse(status).build(status); +} + +UMeasureUnitComplexity MeasureUnit::getComplexity(UErrorCode& status) const { + MeasureUnitImpl temp; + return MeasureUnitImpl::forMeasureUnit(*this, temp, status).complexity; +} + +UMeasurePrefix MeasureUnit::getPrefix(UErrorCode& status) const { + return SingleUnitImpl::forMeasureUnit(*this, status).unitPrefix; +} + +MeasureUnit MeasureUnit::withPrefix(UMeasurePrefix prefix, UErrorCode& status) const UPRV_NO_SANITIZE_UNDEFINED { + SingleUnitImpl singleUnit = SingleUnitImpl::forMeasureUnit(*this, status); + singleUnit.unitPrefix = prefix; + return singleUnit.build(status); +} + +int32_t MeasureUnit::getDimensionality(UErrorCode& status) const { + SingleUnitImpl singleUnit = SingleUnitImpl::forMeasureUnit(*this, status); + if (U_FAILURE(status)) { return 0; } + if (singleUnit.isDimensionless()) { + return 0; + } + return singleUnit.dimensionality; +} + +MeasureUnit MeasureUnit::withDimensionality(int32_t dimensionality, UErrorCode& status) const { + SingleUnitImpl singleUnit = SingleUnitImpl::forMeasureUnit(*this, status); + singleUnit.dimensionality = dimensionality; + return singleUnit.build(status); +} + +MeasureUnit MeasureUnit::reciprocal(UErrorCode& status) const { + MeasureUnitImpl impl = MeasureUnitImpl::forMeasureUnitMaybeCopy(*this, status); + impl.takeReciprocal(status); + return std::move(impl).build(status); +} + +MeasureUnit MeasureUnit::product(const MeasureUnit& other, UErrorCode& status) const { + MeasureUnitImpl impl = MeasureUnitImpl::forMeasureUnitMaybeCopy(*this, status); + MeasureUnitImpl temp; + const MeasureUnitImpl& otherImpl = MeasureUnitImpl::forMeasureUnit(other, temp, status); + if (impl.complexity == UMEASURE_UNIT_MIXED || otherImpl.complexity == UMEASURE_UNIT_MIXED) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return {}; + } + for (int32_t i = 0; i < otherImpl.singleUnits.length(); i++) { + impl.appendSingleUnit(*otherImpl.singleUnits[i], status); + } + if (impl.singleUnits.length() > 1) { + impl.complexity = UMEASURE_UNIT_COMPOUND; + } + return std::move(impl).build(status); +} + +LocalArray MeasureUnit::splitToSingleUnitsImpl(int32_t& outCount, UErrorCode& status) const { + MeasureUnitImpl temp; + const MeasureUnitImpl& impl = MeasureUnitImpl::forMeasureUnit(*this, temp, status); + outCount = impl.singleUnits.length(); + MeasureUnit* arr = new MeasureUnit[outCount]; + if (arr == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return LocalArray(); + } + for (int32_t i = 0; i < outCount; i++) { + arr[i] = impl.singleUnits[i]->build(status); + } + return LocalArray(arr, status); +} + + +U_NAMESPACE_END + +#endif /* !UNCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/measunit_impl.h b/intl/icu/source/i18n/measunit_impl.h new file mode 100644 index 0000000000..c60ff2fc33 --- /dev/null +++ b/intl/icu/source/i18n/measunit_impl.h @@ -0,0 +1,376 @@ +// © 2020 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#ifndef __MEASUNIT_IMPL_H__ +#define __MEASUNIT_IMPL_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/measunit.h" +#include "cmemory.h" +#include "charstr.h" + +U_NAMESPACE_BEGIN + +namespace number { +namespace impl { +class LongNameHandler; +} +} // namespace number + +static const char16_t kDefaultCurrency[] = u"XXX"; +static const char kDefaultCurrency8[] = "XXX"; + +/** + * Looks up the "unitQuantity" (aka "type" or "category") of a base unit + * identifier. The category is returned via `result`, which must initially be + * empty. + * + * This only supports base units: other units must be resolved to base units + * before passing to this function, otherwise U_UNSUPPORTED_ERROR status may be + * returned. + * + * Categories are found in `unitQuantities` in the `units` resource (see + * `units.txt`). + */ +// TODO: make this function accepts any `MeasureUnit` as Java and move it to the `UnitsData` class. +CharString U_I18N_API getUnitQuantity(const MeasureUnitImpl &baseMeasureUnitImpl, UErrorCode &status); + +/** + * A struct representing a single unit (optional SI or binary prefix, and dimensionality). + */ +struct U_I18N_API SingleUnitImpl : public UMemory { + /** + * Gets a single unit from the MeasureUnit. If there are multiple single units, sets an error + * code and returns the base dimensionless unit. Parses if necessary. + */ + static SingleUnitImpl forMeasureUnit(const MeasureUnit& measureUnit, UErrorCode& status); + + /** Transform this SingleUnitImpl into a MeasureUnit, simplifying if possible. */ + MeasureUnit build(UErrorCode& status) const; + + /** + * Returns the "simple unit ID", without SI or dimensionality prefix: this + * instance may represent a square-kilometer, but only "meter" will be + * returned. + * + * The returned pointer points at memory that exists for the duration of the + * program's running. + */ + const char *getSimpleUnitID() const; + + /** + * Generates and append a neutral identifier string for a single unit which means we do not include + * the dimension signal. + */ + void appendNeutralIdentifier(CharString &result, UErrorCode &status) const; + + /** + * Returns the index of this unit's "quantity" in unitQuantities (in + * measunit_extra.cpp). The value of this index determines sort order for + * normalization of unit identifiers. + */ + int32_t getUnitCategoryIndex() const; + + /** + * Compare this SingleUnitImpl to another SingleUnitImpl for the sake of + * sorting and coalescing. + * + * Sort order of units is specified by UTS #35 + * (https://unicode.org/reports/tr35/tr35-info.html#Unit_Identifier_Normalization). + * + * Takes the sign of dimensionality into account, but not the absolute + * value: per-meter is not considered the same as meter, but meter is + * considered the same as square-meter. + * + * The dimensionless unit generally does not get compared, but if it did, it + * would sort before other units by virtue of index being < 0 and + * dimensionality not being negative. + */ + int32_t compareTo(const SingleUnitImpl& other) const { + if (dimensionality < 0 && other.dimensionality > 0) { + // Positive dimensions first + return 1; + } + if (dimensionality > 0 && other.dimensionality < 0) { + return -1; + } + + // Sort by official quantity order + int32_t thisQuantity = this->getUnitCategoryIndex(); + int32_t otherQuantity = other.getUnitCategoryIndex(); + if (thisQuantity < otherQuantity) { + return -1; + } + if (thisQuantity > otherQuantity) { + return 1; + } + + // If quantity order didn't help, then we go by index. + if (index < other.index) { + return -1; + } + if (index > other.index) { + return 1; + } + + // When comparing binary prefixes vs SI prefixes, instead of comparing the actual values, we can + // multiply the binary prefix power by 3 and compare the powers. if they are equal, we can can + // compare the bases. + // NOTE: this methodology will fail if the binary prefix more than or equal 98. + int32_t unitBase = umeas_getPrefixBase(unitPrefix); + int32_t otherUnitBase = umeas_getPrefixBase(other.unitPrefix); + + // Values for comparison purposes only. + int32_t unitPower = unitBase == 1024 /* Binary Prefix */ ? umeas_getPrefixPower(unitPrefix) * 3 + : umeas_getPrefixPower(unitPrefix); + int32_t otherUnitPower = + otherUnitBase == 1024 /* Binary Prefix */ ? umeas_getPrefixPower(other.unitPrefix) * 3 + : umeas_getPrefixPower(other.unitPrefix); + + // NOTE: if the unitPower is less than the other, + // we return 1 not -1. Thus because we want th sorting order + // for the bigger prefix to be before the smaller. + // Example: megabyte should come before kilobyte. + if (unitPower < otherUnitPower) { + return 1; + } + if (unitPower > otherUnitPower) { + return -1; + } + + if (unitBase < otherUnitBase) { + return 1; + } + if (unitBase > otherUnitBase) { + return -1; + } + + return 0; + } + + /** + * Return whether this SingleUnitImpl is compatible with another for the purpose of coalescing. + * + * Units with the same base unit and SI or binary prefix should match, except that they must also + * have the same dimensionality sign, such that we don't merge numerator and denominator. + */ + bool isCompatibleWith(const SingleUnitImpl& other) const { + return (compareTo(other) == 0); + } + + /** + * Returns true if this unit is the "dimensionless base unit", as produced + * by the MeasureUnit() default constructor. (This does not include the + * likes of concentrations or angles.) + */ + bool isDimensionless() const { + return index == -1; + } + + /** + * Simple unit index, unique for every simple unit, -1 for the dimensionless + * unit. This is an index into a string list in measunit_extra.cpp, as + * loaded by SimpleUnitIdentifiersSink. + * + * The default value is -1, meaning the dimensionless unit: + * isDimensionless() will return true, until index is changed. + */ + int32_t index = -1; + + /** + * SI or binary prefix. + * + * This is ignored for the dimensionless unit. + */ + UMeasurePrefix unitPrefix = UMEASURE_PREFIX_ONE; + + /** + * Dimensionality. + * + * This is meaningless for the dimensionless unit. + */ + int32_t dimensionality = 1; +}; + +// Forward declaration +struct MeasureUnitImplWithIndex; + +// Export explicit template instantiations of MaybeStackArray, MemoryPool and +// MaybeStackVector. This is required when building DLLs for Windows. (See +// datefmt.h, collationiterator.h, erarules.h and others for similar examples.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +template class U_I18N_API MemoryPool; +template class U_I18N_API MaybeStackVector; +#endif + +/** + * Internal representation of measurement units. Capable of representing all complexities of units, + * including mixed and compound units. + */ +class U_I18N_API MeasureUnitImpl : public UMemory { + public: + MeasureUnitImpl() = default; + MeasureUnitImpl(MeasureUnitImpl &&other) = default; + // No copy constructor, use MeasureUnitImpl::copy() to make it explicit. + MeasureUnitImpl(const MeasureUnitImpl &other, UErrorCode &status) = delete; + MeasureUnitImpl(const SingleUnitImpl &singleUnit, UErrorCode &status); + + MeasureUnitImpl &operator=(MeasureUnitImpl &&other) noexcept = default; + + /** Extract the MeasureUnitImpl from a MeasureUnit. */ + static inline const MeasureUnitImpl *get(const MeasureUnit &measureUnit) { + return measureUnit.fImpl; + } + + /** + * Parse a unit identifier into a MeasureUnitImpl. + * + * @param identifier The unit identifier string. + * @param status Set if the identifier string is not valid. + * @return A newly parsed value object. Behaviour of this unit is + * unspecified if an error is returned via status. + */ + static MeasureUnitImpl forIdentifier(StringPiece identifier, UErrorCode& status); + + /** + * Extract the MeasureUnitImpl from a MeasureUnit, or parse if it is not present. + * + * @param measureUnit The source MeasureUnit. + * @param memory A place to write the new MeasureUnitImpl if parsing is required. + * @param status Set if an error occurs. + * @return A reference to either measureUnit.fImpl or memory. + */ + static const MeasureUnitImpl& forMeasureUnit( + const MeasureUnit& measureUnit, MeasureUnitImpl& memory, UErrorCode& status); + + /** + * Extract the MeasureUnitImpl from a MeasureUnit, or parse if it is not present. + * + * @param measureUnit The source MeasureUnit. + * @param status Set if an error occurs. + * @return A value object, either newly parsed or copied from measureUnit. + */ + static MeasureUnitImpl forMeasureUnitMaybeCopy( + const MeasureUnit& measureUnit, UErrorCode& status); + + /** + * Used for currency units. + */ + static inline MeasureUnitImpl forCurrencyCode(StringPiece currencyCode) { + MeasureUnitImpl result; + UErrorCode localStatus = U_ZERO_ERROR; + result.identifier.append(currencyCode, localStatus); + // localStatus is not expected to fail since currencyCode should be 3 chars long + return result; + } + + /** Transform this MeasureUnitImpl into a MeasureUnit, simplifying if possible. */ + MeasureUnit build(UErrorCode& status) &&; + + /** + * Create a copy of this MeasureUnitImpl. Don't use copy constructor to make this explicit. + */ + MeasureUnitImpl copy(UErrorCode& status) const; + + /** + * Extracts the list of all the individual units inside the `MeasureUnitImpl` with their indices. + * For example: + * - if the `MeasureUnitImpl` is `foot-per-hour` + * it will return a list of 1 {(0, `foot-per-hour`)} + * - if the `MeasureUnitImpl` is `foot-and-inch` + * it will return a list of 2 {(0, `foot`), (1, `inch`)} + */ + MaybeStackVector + extractIndividualUnitsWithIndices(UErrorCode &status) const; + + /** Mutates this MeasureUnitImpl to take the reciprocal. */ + void takeReciprocal(UErrorCode& status); + + /** + * Returns a simplified version of the unit. + * NOTE: the simplification happen when there are two units equals in their base unit and their + * prefixes. + * + * Example 1: "square-meter-per-meter" --> "meter" + * Example 2: "square-millimeter-per-meter" --> "square-millimeter-per-meter" + */ + MeasureUnitImpl copyAndSimplify(UErrorCode &status) const; + + /** + * Mutates this MeasureUnitImpl to append a single unit. + * + * @return true if a new item was added. If unit is the dimensionless unit, + * it is never added: the return value will always be false. + */ + bool appendSingleUnit(const SingleUnitImpl& singleUnit, UErrorCode& status); + + /** + * Normalizes a MeasureUnitImpl and generate the identifier string in place. + */ + void serialize(UErrorCode &status); + + /** The complexity, either SINGLE, COMPOUND, or MIXED. */ + UMeasureUnitComplexity complexity = UMEASURE_UNIT_SINGLE; + + /** + * The list of single units. These may be summed or multiplied, based on the + * value of the complexity field. + * + * The "dimensionless" unit (SingleUnitImpl default constructor) must not be + * added to this list. + */ + MaybeStackVector singleUnits; + + /** + * The full unit identifier. Owned by the MeasureUnitImpl. Empty if not computed. + */ + CharString identifier; + + // For calling serialize + // TODO(icu-units#147): revisit serialization + friend class number::impl::LongNameHandler; +}; + +struct U_I18N_API MeasureUnitImplWithIndex : public UMemory { + const int32_t index; + MeasureUnitImpl unitImpl; + // Makes a copy of unitImpl. + MeasureUnitImplWithIndex(int32_t index, const MeasureUnitImpl &unitImpl, UErrorCode &status) + : index(index), unitImpl(unitImpl.copy(status)) { + } + MeasureUnitImplWithIndex(int32_t index, const SingleUnitImpl &singleUnitImpl, UErrorCode &status) + : index(index), unitImpl(MeasureUnitImpl(singleUnitImpl, status)) { + } +}; + +// Export explicit template instantiations of MaybeStackArray, MemoryPool and +// MaybeStackVector. This is required when building DLLs for Windows. (See +// datefmt.h, collationiterator.h, erarules.h and others for similar examples.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +template class U_I18N_API MemoryPool; +template class U_I18N_API MaybeStackVector; + +// Export an explicit template instantiation of the LocalPointer that is used as a +// data member of MeasureUnitImpl. +// (When building DLLs for Windows this is required.) +#if defined(_MSC_VER) +// Ignore warning 4661 as LocalPointerBase does not use operator== or operator!= +#pragma warning(push) +#pragma warning(disable : 4661) +#endif +template class U_I18N_API LocalPointerBase; +template class U_I18N_API LocalPointer; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif //__MEASUNIT_IMPL_H__ diff --git a/intl/icu/source/i18n/measure.cpp b/intl/icu/source/i18n/measure.cpp new file mode 100644 index 0000000000..cdbd995034 --- /dev/null +++ b/intl/icu/source/i18n/measure.cpp @@ -0,0 +1,78 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2004-2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: April 26, 2004 +* Since: ICU 3.0 +********************************************************************** +*/ +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/measure.h" +#include "unicode/measunit.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Measure) + +Measure::Measure() : unit(nullptr) {} + +Measure::Measure(const Formattable& _number, MeasureUnit* adoptedUnit, + UErrorCode& ec) : + number(_number), unit(adoptedUnit) { + if (U_SUCCESS(ec) && + (!number.isNumeric() || adoptedUnit == 0)) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + } +} + +Measure::Measure(const Measure& other) : + UObject(other), unit(nullptr) { + *this = other; +} + +Measure& Measure::operator=(const Measure& other) { + if (this != &other) { + delete unit; + number = other.number; + if (other.unit != nullptr) { + unit = other.unit->clone(); + } else { + unit = nullptr; + } + } + return *this; +} + +Measure *Measure::clone() const { + return new Measure(*this); +} + +Measure::~Measure() { + delete unit; +} + +bool Measure::operator==(const UObject& other) const { + if (this == &other) { // Same object, equal + return true; + } + if (typeid(*this) != typeid(other)) { // Different types, not equal + return false; + } + const Measure &m = static_cast(other); + return number == m.number && + ((unit == nullptr) == (m.unit == nullptr)) && + (unit == nullptr || *unit == *m.unit); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING diff --git a/intl/icu/source/i18n/msgfmt.cpp b/intl/icu/source/i18n/msgfmt.cpp new file mode 100644 index 0000000000..29fb4b3a01 --- /dev/null +++ b/intl/icu/source/i18n/msgfmt.cpp @@ -0,0 +1,2009 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/******************************************************************** + * COPYRIGHT: + * Copyright (c) 1997-2015, International Business Machines Corporation and + * others. All Rights Reserved. + ******************************************************************** + * + * File MSGFMT.CPP + * + * Modification History: + * + * Date Name Description + * 02/19/97 aliu Converted from java. + * 03/20/97 helena Finished first cut of implementation. + * 04/10/97 aliu Made to work on AIX. Added stoi to replace wtoi. + * 06/11/97 helena Fixed addPattern to take the pattern correctly. + * 06/17/97 helena Fixed the getPattern to return the correct pattern. + * 07/09/97 helena Made ParsePosition into a class. + * 02/22/99 stephen Removed character literals for EBCDIC safety + * 11/01/09 kirtig Added SelectFormat + ********************************************************************/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/appendable.h" +#include "unicode/choicfmt.h" +#include "unicode/datefmt.h" +#include "unicode/decimfmt.h" +#include "unicode/localpointer.h" +#include "unicode/msgfmt.h" +#include "unicode/numberformatter.h" +#include "unicode/plurfmt.h" +#include "unicode/rbnf.h" +#include "unicode/selfmt.h" +#include "unicode/smpdtfmt.h" +#include "unicode/umsg.h" +#include "unicode/ustring.h" +#include "cmemory.h" +#include "patternprops.h" +#include "messageimpl.h" +#include "msgfmt_impl.h" +#include "plurrule_impl.h" +#include "uassert.h" +#include "uelement.h" +#include "uhash.h" +#include "ustrfmt.h" +#include "util.h" +#include "uvector.h" +#include "number_decimalquantity.h" + +// ***************************************************************************** +// class MessageFormat +// ***************************************************************************** + +#define SINGLE_QUOTE ((char16_t)0x0027) +#define COMMA ((char16_t)0x002C) +#define LEFT_CURLY_BRACE ((char16_t)0x007B) +#define RIGHT_CURLY_BRACE ((char16_t)0x007D) + +//--------------------------------------- +// static data + +static const char16_t ID_NUMBER[] = { + 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0 /* "number" */ +}; +static const char16_t ID_DATE[] = { + 0x64, 0x61, 0x74, 0x65, 0 /* "date" */ +}; +static const char16_t ID_TIME[] = { + 0x74, 0x69, 0x6D, 0x65, 0 /* "time" */ +}; +static const char16_t ID_SPELLOUT[] = { + 0x73, 0x70, 0x65, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0 /* "spellout" */ +}; +static const char16_t ID_ORDINAL[] = { + 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0 /* "ordinal" */ +}; +static const char16_t ID_DURATION[] = { + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0 /* "duration" */ +}; + +// MessageFormat Type List Number, Date, Time or Choice +static const char16_t * const TYPE_IDS[] = { + ID_NUMBER, + ID_DATE, + ID_TIME, + ID_SPELLOUT, + ID_ORDINAL, + ID_DURATION, + nullptr, +}; + +static const char16_t ID_EMPTY[] = { + 0 /* empty string, used for default so that null can mark end of list */ +}; +static const char16_t ID_CURRENCY[] = { + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6E, 0x63, 0x79, 0 /* "currency" */ +}; +static const char16_t ID_PERCENT[] = { + 0x70, 0x65, 0x72, 0x63, 0x65, 0x6E, 0x74, 0 /* "percent" */ +}; +static const char16_t ID_INTEGER[] = { + 0x69, 0x6E, 0x74, 0x65, 0x67, 0x65, 0x72, 0 /* "integer" */ +}; + +// NumberFormat modifier list, default, currency, percent or integer +static const char16_t * const NUMBER_STYLE_IDS[] = { + ID_EMPTY, + ID_CURRENCY, + ID_PERCENT, + ID_INTEGER, + nullptr, +}; + +static const char16_t ID_SHORT[] = { + 0x73, 0x68, 0x6F, 0x72, 0x74, 0 /* "short" */ +}; +static const char16_t ID_MEDIUM[] = { + 0x6D, 0x65, 0x64, 0x69, 0x75, 0x6D, 0 /* "medium" */ +}; +static const char16_t ID_LONG[] = { + 0x6C, 0x6F, 0x6E, 0x67, 0 /* "long" */ +}; +static const char16_t ID_FULL[] = { + 0x66, 0x75, 0x6C, 0x6C, 0 /* "full" */ +}; + +// DateFormat modifier list, default, short, medium, long or full +static const char16_t * const DATE_STYLE_IDS[] = { + ID_EMPTY, + ID_SHORT, + ID_MEDIUM, + ID_LONG, + ID_FULL, + nullptr, +}; + +static const icu::DateFormat::EStyle DATE_STYLES[] = { + icu::DateFormat::kDefault, + icu::DateFormat::kShort, + icu::DateFormat::kMedium, + icu::DateFormat::kLong, + icu::DateFormat::kFull, +}; + +static const int32_t DEFAULT_INITIAL_CAPACITY = 10; + +static const char16_t NULL_STRING[] = { + 0x6E, 0x75, 0x6C, 0x6C, 0 // "null" +}; + +static const char16_t OTHER_STRING[] = { + 0x6F, 0x74, 0x68, 0x65, 0x72, 0 // "other" +}; + +U_CDECL_BEGIN +static UBool U_CALLCONV equalFormatsForHash(const UHashTok key1, + const UHashTok key2) { + return icu::MessageFormat::equalFormats(key1.pointer, key2.pointer); +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +// ------------------------------------- +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MessageFormat) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(FormatNameEnumeration) + +//-------------------------------------------------------------------- + +/** + * Convert an integer value to a string and append the result to + * the given UnicodeString. + */ +static UnicodeString& itos(int32_t i, UnicodeString& appendTo) { + char16_t temp[16]; + uprv_itou(temp,16,i,10,0); // 10 == radix + appendTo.append(temp, -1); + return appendTo; +} + + +// AppendableWrapper: encapsulates the result of formatting, keeping track +// of the string and its length. +class AppendableWrapper : public UMemory { +public: + AppendableWrapper(Appendable& appendable) : app(appendable), len(0) { + } + void append(const UnicodeString& s) { + app.appendString(s.getBuffer(), s.length()); + len += s.length(); + } + void append(const char16_t* s, const int32_t sLength) { + app.appendString(s, sLength); + len += sLength; + } + void append(const UnicodeString& s, int32_t start, int32_t length) { + append(s.tempSubString(start, length)); + } + void formatAndAppend(const Format* formatter, const Formattable& arg, UErrorCode& ec) { + UnicodeString s; + formatter->format(arg, s, ec); + if (U_SUCCESS(ec)) { + append(s); + } + } + void formatAndAppend(const Format* formatter, const Formattable& arg, + const UnicodeString &argString, UErrorCode& ec) { + if (!argString.isEmpty()) { + if (U_SUCCESS(ec)) { + append(argString); + } + } else { + formatAndAppend(formatter, arg, ec); + } + } + int32_t length() { + return len; + } +private: + Appendable& app; + int32_t len; +}; + + +// ------------------------------------- +// Creates a MessageFormat instance based on the pattern. + +MessageFormat::MessageFormat(const UnicodeString& pattern, + UErrorCode& success) +: fLocale(Locale::getDefault()), // Uses the default locale + msgPattern(success), + formatAliases(nullptr), + formatAliasesCapacity(0), + argTypes(nullptr), + argTypeCount(0), + argTypeCapacity(0), + hasArgTypeConflicts(false), + defaultNumberFormat(nullptr), + defaultDateFormat(nullptr), + cachedFormatters(nullptr), + customFormatArgStarts(nullptr), + pluralProvider(*this, UPLURAL_TYPE_CARDINAL), + ordinalProvider(*this, UPLURAL_TYPE_ORDINAL) +{ + setLocaleIDs(fLocale.getName(), fLocale.getName()); + applyPattern(pattern, success); +} + +MessageFormat::MessageFormat(const UnicodeString& pattern, + const Locale& newLocale, + UErrorCode& success) +: fLocale(newLocale), + msgPattern(success), + formatAliases(nullptr), + formatAliasesCapacity(0), + argTypes(nullptr), + argTypeCount(0), + argTypeCapacity(0), + hasArgTypeConflicts(false), + defaultNumberFormat(nullptr), + defaultDateFormat(nullptr), + cachedFormatters(nullptr), + customFormatArgStarts(nullptr), + pluralProvider(*this, UPLURAL_TYPE_CARDINAL), + ordinalProvider(*this, UPLURAL_TYPE_ORDINAL) +{ + setLocaleIDs(fLocale.getName(), fLocale.getName()); + applyPattern(pattern, success); +} + +MessageFormat::MessageFormat(const UnicodeString& pattern, + const Locale& newLocale, + UParseError& parseError, + UErrorCode& success) +: fLocale(newLocale), + msgPattern(success), + formatAliases(nullptr), + formatAliasesCapacity(0), + argTypes(nullptr), + argTypeCount(0), + argTypeCapacity(0), + hasArgTypeConflicts(false), + defaultNumberFormat(nullptr), + defaultDateFormat(nullptr), + cachedFormatters(nullptr), + customFormatArgStarts(nullptr), + pluralProvider(*this, UPLURAL_TYPE_CARDINAL), + ordinalProvider(*this, UPLURAL_TYPE_ORDINAL) +{ + setLocaleIDs(fLocale.getName(), fLocale.getName()); + applyPattern(pattern, parseError, success); +} + +MessageFormat::MessageFormat(const MessageFormat& that) +: + Format(that), + fLocale(that.fLocale), + msgPattern(that.msgPattern), + formatAliases(nullptr), + formatAliasesCapacity(0), + argTypes(nullptr), + argTypeCount(0), + argTypeCapacity(0), + hasArgTypeConflicts(that.hasArgTypeConflicts), + defaultNumberFormat(nullptr), + defaultDateFormat(nullptr), + cachedFormatters(nullptr), + customFormatArgStarts(nullptr), + pluralProvider(*this, UPLURAL_TYPE_CARDINAL), + ordinalProvider(*this, UPLURAL_TYPE_ORDINAL) +{ + // This will take care of creating the hash tables (since they are nullptr). + UErrorCode ec = U_ZERO_ERROR; + copyObjects(that, ec); + if (U_FAILURE(ec)) { + resetPattern(); + } +} + +MessageFormat::~MessageFormat() +{ + uhash_close(cachedFormatters); + uhash_close(customFormatArgStarts); + + uprv_free(argTypes); + uprv_free(formatAliases); + delete defaultNumberFormat; + delete defaultDateFormat; +} + +//-------------------------------------------------------------------- +// Variable-size array management + +/** + * Allocate argTypes[] to at least the given capacity and return + * true if successful. If not, leave argTypes[] unchanged. + * + * If argTypes is nullptr, allocate it. If it is not nullptr, enlarge it + * if necessary to be at least as large as specified. + */ +UBool MessageFormat::allocateArgTypes(int32_t capacity, UErrorCode& status) { + if (U_FAILURE(status)) { + return false; + } + if (argTypeCapacity >= capacity) { + return true; + } + if (capacity < DEFAULT_INITIAL_CAPACITY) { + capacity = DEFAULT_INITIAL_CAPACITY; + } else if (capacity < 2*argTypeCapacity) { + capacity = 2*argTypeCapacity; + } + Formattable::Type* a = (Formattable::Type*) + uprv_realloc(argTypes, sizeof(*argTypes) * capacity); + if (a == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + argTypes = a; + argTypeCapacity = capacity; + return true; +} + +// ------------------------------------- +// assignment operator + +const MessageFormat& +MessageFormat::operator=(const MessageFormat& that) +{ + if (this != &that) { + // Calls the super class for assignment first. + Format::operator=(that); + + setLocale(that.fLocale); + msgPattern = that.msgPattern; + hasArgTypeConflicts = that.hasArgTypeConflicts; + + UErrorCode ec = U_ZERO_ERROR; + copyObjects(that, ec); + if (U_FAILURE(ec)) { + resetPattern(); + } + } + return *this; +} + +bool +MessageFormat::operator==(const Format& rhs) const +{ + if (this == &rhs) return true; + + // Check class ID before checking MessageFormat members + if (!Format::operator==(rhs)) return false; + + const MessageFormat& that = static_cast(rhs); + if (msgPattern != that.msgPattern || + fLocale != that.fLocale) { + return false; + } + + // Compare hashtables. + if ((customFormatArgStarts == nullptr) != (that.customFormatArgStarts == nullptr)) { + return false; + } + if (customFormatArgStarts == nullptr) { + return true; + } + + UErrorCode ec = U_ZERO_ERROR; + const int32_t count = uhash_count(customFormatArgStarts); + const int32_t rhs_count = uhash_count(that.customFormatArgStarts); + if (count != rhs_count) { + return false; + } + int32_t idx = 0, rhs_idx = 0, pos = UHASH_FIRST, rhs_pos = UHASH_FIRST; + for (; idx < count && rhs_idx < rhs_count && U_SUCCESS(ec); ++idx, ++rhs_idx) { + const UHashElement* cur = uhash_nextElement(customFormatArgStarts, &pos); + const UHashElement* rhs_cur = uhash_nextElement(that.customFormatArgStarts, &rhs_pos); + if (cur->key.integer != rhs_cur->key.integer) { + return false; + } + const Format* format = (const Format*)uhash_iget(cachedFormatters, cur->key.integer); + const Format* rhs_format = (const Format*)uhash_iget(that.cachedFormatters, rhs_cur->key.integer); + if (*format != *rhs_format) { + return false; + } + } + return true; +} + +// ------------------------------------- +// Creates a copy of this MessageFormat, the caller owns the copy. + +MessageFormat* +MessageFormat::clone() const +{ + return new MessageFormat(*this); +} + +// ------------------------------------- +// Sets the locale of this MessageFormat object to theLocale. + +void +MessageFormat::setLocale(const Locale& theLocale) +{ + if (fLocale != theLocale) { + delete defaultNumberFormat; + defaultNumberFormat = nullptr; + delete defaultDateFormat; + defaultDateFormat = nullptr; + fLocale = theLocale; + setLocaleIDs(fLocale.getName(), fLocale.getName()); + pluralProvider.reset(); + ordinalProvider.reset(); + } +} + +// ------------------------------------- +// Gets the locale of this MessageFormat object. + +const Locale& +MessageFormat::getLocale() const +{ + return fLocale; +} + +void +MessageFormat::applyPattern(const UnicodeString& newPattern, + UErrorCode& status) +{ + UParseError parseError; + applyPattern(newPattern,parseError,status); +} + + +// ------------------------------------- +// Applies the new pattern and returns an error if the pattern +// is not correct. +void +MessageFormat::applyPattern(const UnicodeString& pattern, + UParseError& parseError, + UErrorCode& ec) +{ + if(U_FAILURE(ec)) { + return; + } + msgPattern.parse(pattern, &parseError, ec); + cacheExplicitFormats(ec); + + if (U_FAILURE(ec)) { + resetPattern(); + } +} + +void MessageFormat::resetPattern() { + msgPattern.clear(); + uhash_close(cachedFormatters); + cachedFormatters = nullptr; + uhash_close(customFormatArgStarts); + customFormatArgStarts = nullptr; + argTypeCount = 0; + hasArgTypeConflicts = false; +} + +void +MessageFormat::applyPattern(const UnicodeString& pattern, + UMessagePatternApostropheMode aposMode, + UParseError* parseError, + UErrorCode& status) { + if (aposMode != msgPattern.getApostropheMode()) { + msgPattern.clearPatternAndSetApostropheMode(aposMode); + } + UParseError tempParseError; + applyPattern(pattern, (parseError == nullptr) ? tempParseError : *parseError, status); +} + +// ------------------------------------- +// Converts this MessageFormat instance to a pattern. + +UnicodeString& +MessageFormat::toPattern(UnicodeString& appendTo) const { + if ((customFormatArgStarts != nullptr && 0 != uhash_count(customFormatArgStarts)) || + 0 == msgPattern.countParts() + ) { + appendTo.setToBogus(); + return appendTo; + } + return appendTo.append(msgPattern.getPatternString()); +} + +int32_t MessageFormat::nextTopLevelArgStart(int32_t partIndex) const { + if (partIndex != 0) { + partIndex = msgPattern.getLimitPartIndex(partIndex); + } + for (;;) { + UMessagePatternPartType type = msgPattern.getPartType(++partIndex); + if (type == UMSGPAT_PART_TYPE_ARG_START) { + return partIndex; + } + if (type == UMSGPAT_PART_TYPE_MSG_LIMIT) { + return -1; + } + } +} + +void MessageFormat::setArgStartFormat(int32_t argStart, + Format* formatter, + UErrorCode& status) { + if (U_FAILURE(status)) { + delete formatter; + return; + } + if (cachedFormatters == nullptr) { + cachedFormatters=uhash_open(uhash_hashLong, uhash_compareLong, + equalFormatsForHash, &status); + if (U_FAILURE(status)) { + delete formatter; + return; + } + uhash_setValueDeleter(cachedFormatters, uprv_deleteUObject); + } + if (formatter == nullptr) { + formatter = new DummyFormat(); + } + uhash_iput(cachedFormatters, argStart, formatter, &status); +} + + +UBool MessageFormat::argNameMatches(int32_t partIndex, const UnicodeString& argName, int32_t argNumber) { + const MessagePattern::Part& part = msgPattern.getPart(partIndex); + return part.getType() == UMSGPAT_PART_TYPE_ARG_NAME ? + msgPattern.partSubstringMatches(part, argName) : + part.getValue() == argNumber; // ARG_NUMBER +} + +// Sets a custom formatter for a MessagePattern ARG_START part index. +// "Custom" formatters are provided by the user via setFormat() or similar APIs. +void MessageFormat::setCustomArgStartFormat(int32_t argStart, + Format* formatter, + UErrorCode& status) { + setArgStartFormat(argStart, formatter, status); + if (customFormatArgStarts == nullptr) { + customFormatArgStarts=uhash_open(uhash_hashLong, uhash_compareLong, + nullptr, &status); + } + uhash_iputi(customFormatArgStarts, argStart, 1, &status); +} + +Format* MessageFormat::getCachedFormatter(int32_t argumentNumber) const { + if (cachedFormatters == nullptr) { + return nullptr; + } + void* ptr = uhash_iget(cachedFormatters, argumentNumber); + if (ptr != nullptr && dynamic_cast((Format*)ptr) == nullptr) { + return (Format*) ptr; + } else { + // Not cached, or a DummyFormat representing setFormat(nullptr). + return nullptr; + } +} + +// ------------------------------------- +// Adopts the new formats array and updates the array count. +// This MessageFormat instance owns the new formats. +void +MessageFormat::adoptFormats(Format** newFormats, + int32_t count) { + if (newFormats == nullptr || count < 0) { + return; + } + // Throw away any cached formatters. + if (cachedFormatters != nullptr) { + uhash_removeAll(cachedFormatters); + } + if (customFormatArgStarts != nullptr) { + uhash_removeAll(customFormatArgStarts); + } + + int32_t formatNumber = 0; + UErrorCode status = U_ZERO_ERROR; + for (int32_t partIndex = 0; + formatNumber < count && U_SUCCESS(status) && + (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + setCustomArgStartFormat(partIndex, newFormats[formatNumber], status); + ++formatNumber; + } + // Delete those that didn't get used (if any). + for (; formatNumber < count; ++formatNumber) { + delete newFormats[formatNumber]; + } + +} + +// ------------------------------------- +// Sets the new formats array and updates the array count. +// This MessageFormat instance makes a copy of the new formats. + +void +MessageFormat::setFormats(const Format** newFormats, + int32_t count) { + if (newFormats == nullptr || count < 0) { + return; + } + // Throw away any cached formatters. + if (cachedFormatters != nullptr) { + uhash_removeAll(cachedFormatters); + } + if (customFormatArgStarts != nullptr) { + uhash_removeAll(customFormatArgStarts); + } + + UErrorCode status = U_ZERO_ERROR; + int32_t formatNumber = 0; + for (int32_t partIndex = 0; + formatNumber < count && U_SUCCESS(status) && (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + Format* newFormat = nullptr; + if (newFormats[formatNumber] != nullptr) { + newFormat = newFormats[formatNumber]->clone(); + if (newFormat == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + setCustomArgStartFormat(partIndex, newFormat, status); + ++formatNumber; + } + if (U_FAILURE(status)) { + resetPattern(); + } +} + +// ------------------------------------- +// Adopt a single format by format number. +// Do nothing if the format number is not less than the array count. + +void +MessageFormat::adoptFormat(int32_t n, Format *newFormat) { + LocalPointer p(newFormat); + if (n >= 0) { + int32_t formatNumber = 0; + for (int32_t partIndex = 0; (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + if (n == formatNumber) { + UErrorCode status = U_ZERO_ERROR; + setCustomArgStartFormat(partIndex, p.orphan(), status); + return; + } + ++formatNumber; + } + } +} + +// ------------------------------------- +// Adopt a single format by format name. +// Do nothing if there is no match of formatName. +void +MessageFormat::adoptFormat(const UnicodeString& formatName, + Format* formatToAdopt, + UErrorCode& status) { + LocalPointer p(formatToAdopt); + if (U_FAILURE(status)) { + return; + } + int32_t argNumber = MessagePattern::validateArgumentName(formatName); + if (argNumber < UMSGPAT_ARG_NAME_NOT_NUMBER) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + for (int32_t partIndex = 0; + (partIndex = nextTopLevelArgStart(partIndex)) >= 0 && U_SUCCESS(status); + ) { + if (argNameMatches(partIndex + 1, formatName, argNumber)) { + Format* f; + if (p.isValid()) { + f = p.orphan(); + } else if (formatToAdopt == nullptr) { + f = nullptr; + } else { + f = formatToAdopt->clone(); + if (f == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + setCustomArgStartFormat(partIndex, f, status); + } + } +} + +// ------------------------------------- +// Set a single format. +// Do nothing if the variable is not less than the array count. +void +MessageFormat::setFormat(int32_t n, const Format& newFormat) { + + if (n >= 0) { + int32_t formatNumber = 0; + for (int32_t partIndex = 0; + (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + if (n == formatNumber) { + Format* new_format = newFormat.clone(); + if (new_format) { + UErrorCode status = U_ZERO_ERROR; + setCustomArgStartFormat(partIndex, new_format, status); + } + return; + } + ++formatNumber; + } + } +} + +// ------------------------------------- +// Get a single format by format name. +// Do nothing if the variable is not less than the array count. +Format * +MessageFormat::getFormat(const UnicodeString& formatName, UErrorCode& status) { + if (U_FAILURE(status) || cachedFormatters == nullptr) return nullptr; + + int32_t argNumber = MessagePattern::validateArgumentName(formatName); + if (argNumber < UMSGPAT_ARG_NAME_NOT_NUMBER) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + for (int32_t partIndex = 0; (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + if (argNameMatches(partIndex + 1, formatName, argNumber)) { + return getCachedFormatter(partIndex); + } + } + return nullptr; +} + +// ------------------------------------- +// Set a single format by format name +// Do nothing if the variable is not less than the array count. +void +MessageFormat::setFormat(const UnicodeString& formatName, + const Format& newFormat, + UErrorCode& status) { + if (U_FAILURE(status)) return; + + int32_t argNumber = MessagePattern::validateArgumentName(formatName); + if (argNumber < UMSGPAT_ARG_NAME_NOT_NUMBER) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + for (int32_t partIndex = 0; + (partIndex = nextTopLevelArgStart(partIndex)) >= 0 && U_SUCCESS(status); + ) { + if (argNameMatches(partIndex + 1, formatName, argNumber)) { + Format* new_format = newFormat.clone(); + if (new_format == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + setCustomArgStartFormat(partIndex, new_format, status); + } + } +} + +// ------------------------------------- +// Gets the format array. +const Format** +MessageFormat::getFormats(int32_t& cnt) const +{ + // This old API returns an array (which we hold) of Format* + // pointers. The array is valid up to the next call to any + // method on this object. We construct and resize an array + // on demand that contains aliases to the subformats[i].format + // pointers. + + // Get total required capacity first (it's refreshed on each call). + int32_t totalCapacity = 0; + for (int32_t partIndex = 0; (partIndex = nextTopLevelArgStart(partIndex)) >= 0; ++totalCapacity) {} + + MessageFormat* t = const_cast (this); + cnt = 0; + if (formatAliases == nullptr) { + t->formatAliasesCapacity = totalCapacity; + Format** a = (Format**) + uprv_malloc(sizeof(Format*) * formatAliasesCapacity); + if (a == nullptr) { + t->formatAliasesCapacity = 0; + return nullptr; + } + t->formatAliases = a; + } else if (totalCapacity > formatAliasesCapacity) { + Format** a = (Format**) + uprv_realloc(formatAliases, sizeof(Format*) * totalCapacity); + if (a == nullptr) { + t->formatAliasesCapacity = 0; + return nullptr; + } + t->formatAliases = a; + t->formatAliasesCapacity = totalCapacity; + } + + for (int32_t partIndex = 0; (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + t->formatAliases[cnt++] = getCachedFormatter(partIndex); + } + + return (const Format**)formatAliases; +} + + +UnicodeString MessageFormat::getArgName(int32_t partIndex) { + const MessagePattern::Part& part = msgPattern.getPart(partIndex); + return msgPattern.getSubstring(part); +} + +StringEnumeration* +MessageFormat::getFormatNames(UErrorCode& status) { + if (U_FAILURE(status)) return nullptr; + + LocalPointer formatNames(new UVector(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + formatNames->setDeleter(uprv_deleteUObject); + + for (int32_t partIndex = 0; (partIndex = nextTopLevelArgStart(partIndex)) >= 0;) { + LocalPointer name(getArgName(partIndex + 1).clone(), status); + formatNames->adoptElement(name.orphan(), status); + if (U_FAILURE(status)) return nullptr; + } + + LocalPointer nameEnumerator( + new FormatNameEnumeration(std::move(formatNames), status), status); + return U_SUCCESS(status) ? nameEnumerator.orphan() : nullptr; +} + +// ------------------------------------- +// Formats the source Formattable array and copy into the result buffer. +// Ignore the FieldPosition result for error checking. + +UnicodeString& +MessageFormat::format(const Formattable* source, + int32_t cnt, + UnicodeString& appendTo, + FieldPosition& ignore, + UErrorCode& success) const +{ + return format(source, nullptr, cnt, appendTo, &ignore, success); +} + +// ------------------------------------- +// Internally creates a MessageFormat instance based on the +// pattern and formats the arguments Formattable array and +// copy into the appendTo buffer. + +UnicodeString& +MessageFormat::format( const UnicodeString& pattern, + const Formattable* arguments, + int32_t cnt, + UnicodeString& appendTo, + UErrorCode& success) +{ + MessageFormat temp(pattern, success); + return temp.format(arguments, nullptr, cnt, appendTo, nullptr, success); +} + +// ------------------------------------- +// Formats the source Formattable object and copy into the +// appendTo buffer. The Formattable object must be an array +// of Formattable instances, returns error otherwise. + +UnicodeString& +MessageFormat::format(const Formattable& source, + UnicodeString& appendTo, + FieldPosition& ignore, + UErrorCode& success) const +{ + if (U_FAILURE(success)) + return appendTo; + if (source.getType() != Formattable::kArray) { + success = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; + } + int32_t cnt; + const Formattable* tmpPtr = source.getArray(cnt); + return format(tmpPtr, nullptr, cnt, appendTo, &ignore, success); +} + +UnicodeString& +MessageFormat::format(const UnicodeString* argumentNames, + const Formattable* arguments, + int32_t count, + UnicodeString& appendTo, + UErrorCode& success) const { + return format(arguments, argumentNames, count, appendTo, nullptr, success); +} + +// Does linear search to find the match for an ArgName. +const Formattable* MessageFormat::getArgFromListByName(const Formattable* arguments, + const UnicodeString *argumentNames, + int32_t cnt, UnicodeString& name) const { + for (int32_t i = 0; i < cnt; ++i) { + if (0 == argumentNames[i].compare(name)) { + return arguments + i; + } + } + return nullptr; +} + + +UnicodeString& +MessageFormat::format(const Formattable* arguments, + const UnicodeString *argumentNames, + int32_t cnt, + UnicodeString& appendTo, + FieldPosition* pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; + } + + UnicodeStringAppendable usapp(appendTo); + AppendableWrapper app(usapp); + format(0, nullptr, arguments, argumentNames, cnt, app, pos, status); + return appendTo; +} + +namespace { + +/** + * Mutable input/output values for the PluralSelectorProvider. + * Separate so that it is possible to make MessageFormat Freezable. + */ +class PluralSelectorContext { +public: + PluralSelectorContext(int32_t start, const UnicodeString &name, + const Formattable &num, double off, UErrorCode &errorCode) + : startIndex(start), argName(name), offset(off), + numberArgIndex(-1), formatter(nullptr), forReplaceNumber(false) { + // number needs to be set even when select() is not called. + // Keep it as a Number/Formattable: + // For format() methods, and to preserve information (e.g., BigDecimal). + if(off == 0) { + number = num; + } else { + number = num.getDouble(errorCode) - off; + } + } + + // Input values for plural selection with decimals. + int32_t startIndex; + const UnicodeString &argName; + /** argument number - plural offset */ + Formattable number; + double offset; + // Output values for plural selection with decimals. + /** -1 if REPLACE_NUMBER, 0 arg not found, >0 ARG_START index */ + int32_t numberArgIndex; + const Format *formatter; + /** formatted argument number - plural offset */ + UnicodeString numberString; + /** true if number-offset was formatted with the stock number formatter */ + UBool forReplaceNumber; +}; + +} // namespace + +// if argumentNames is nullptr, this means arguments is a numeric array. +// arguments can not be nullptr. +// We use const void *plNumber rather than const PluralSelectorContext *pluralNumber +// so that we need not declare the PluralSelectorContext in the public header file. +void MessageFormat::format(int32_t msgStart, const void *plNumber, + const Formattable* arguments, + const UnicodeString *argumentNames, + int32_t cnt, + AppendableWrapper& appendTo, + FieldPosition* ignore, + UErrorCode& success) const { + if (U_FAILURE(success)) { + return; + } + + const UnicodeString& msgString = msgPattern.getPatternString(); + int32_t prevIndex = msgPattern.getPart(msgStart).getLimit(); + for (int32_t i = msgStart + 1; U_SUCCESS(success) ; ++i) { + const MessagePattern::Part* part = &msgPattern.getPart(i); + const UMessagePatternPartType type = part->getType(); + int32_t index = part->getIndex(); + appendTo.append(msgString, prevIndex, index - prevIndex); + if (type == UMSGPAT_PART_TYPE_MSG_LIMIT) { + return; + } + prevIndex = part->getLimit(); + if (type == UMSGPAT_PART_TYPE_REPLACE_NUMBER) { + const PluralSelectorContext &pluralNumber = + *static_cast(plNumber); + if(pluralNumber.forReplaceNumber) { + // number-offset was already formatted. + appendTo.formatAndAppend(pluralNumber.formatter, + pluralNumber.number, pluralNumber.numberString, success); + } else { + const NumberFormat* nf = getDefaultNumberFormat(success); + appendTo.formatAndAppend(nf, pluralNumber.number, success); + } + continue; + } + if (type != UMSGPAT_PART_TYPE_ARG_START) { + continue; + } + int32_t argLimit = msgPattern.getLimitPartIndex(i); + UMessagePatternArgType argType = part->getArgType(); + part = &msgPattern.getPart(++i); + const Formattable* arg; + UBool noArg = false; + UnicodeString argName = msgPattern.getSubstring(*part); + if (argumentNames == nullptr) { + int32_t argNumber = part->getValue(); // ARG_NUMBER + if (0 <= argNumber && argNumber < cnt) { + arg = arguments + argNumber; + } else { + arg = nullptr; + noArg = true; + } + } else { + arg = getArgFromListByName(arguments, argumentNames, cnt, argName); + if (arg == nullptr) { + noArg = true; + } + } + ++i; + int32_t prevDestLength = appendTo.length(); + const Format* formatter = nullptr; + if (noArg) { + appendTo.append( + UnicodeString(LEFT_CURLY_BRACE).append(argName).append(RIGHT_CURLY_BRACE)); + } else if (arg == nullptr) { + appendTo.append(NULL_STRING, 4); + } else if(plNumber!=nullptr && + static_cast(plNumber)->numberArgIndex==(i-2)) { + const PluralSelectorContext &pluralNumber = + *static_cast(plNumber); + if(pluralNumber.offset == 0) { + // The number was already formatted with this formatter. + appendTo.formatAndAppend(pluralNumber.formatter, pluralNumber.number, + pluralNumber.numberString, success); + } else { + // Do not use the formatted (number-offset) string for a named argument + // that formats the number without subtracting the offset. + appendTo.formatAndAppend(pluralNumber.formatter, *arg, success); + } + } else if ((formatter = getCachedFormatter(i -2)) != 0) { + // Handles all ArgType.SIMPLE, and formatters from setFormat() and its siblings. + if (dynamic_cast(formatter) || + dynamic_cast(formatter) || + dynamic_cast(formatter)) { + // We only handle nested formats here if they were provided via + // setFormat() or its siblings. Otherwise they are not cached and instead + // handled below according to argType. + UnicodeString subMsgString; + formatter->format(*arg, subMsgString, success); + if (subMsgString.indexOf(LEFT_CURLY_BRACE) >= 0 || + (subMsgString.indexOf(SINGLE_QUOTE) >= 0 && !MessageImpl::jdkAposMode(msgPattern)) + ) { + MessageFormat subMsgFormat(subMsgString, fLocale, success); + subMsgFormat.format(0, nullptr, arguments, argumentNames, cnt, appendTo, ignore, success); + } else { + appendTo.append(subMsgString); + } + } else { + appendTo.formatAndAppend(formatter, *arg, success); + } + } else if (argType == UMSGPAT_ARG_TYPE_NONE || (cachedFormatters && uhash_iget(cachedFormatters, i - 2))) { + // We arrive here if getCachedFormatter returned nullptr, but there was actually an element in the hash table. + // This can only happen if the hash table contained a DummyFormat, so the if statement above is a check + // for the hash table containing DummyFormat. + if (arg->isNumeric()) { + const NumberFormat* nf = getDefaultNumberFormat(success); + appendTo.formatAndAppend(nf, *arg, success); + } else if (arg->getType() == Formattable::kDate) { + const DateFormat* df = getDefaultDateFormat(success); + appendTo.formatAndAppend(df, *arg, success); + } else { + appendTo.append(arg->getString(success)); + } + } else if (argType == UMSGPAT_ARG_TYPE_CHOICE) { + if (!arg->isNumeric()) { + success = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + // We must use the Formattable::getDouble() variant with the UErrorCode parameter + // because only this one converts non-double numeric types to double. + const double number = arg->getDouble(success); + int32_t subMsgStart = ChoiceFormat::findSubMessage(msgPattern, i, number); + formatComplexSubMessage(subMsgStart, nullptr, arguments, argumentNames, + cnt, appendTo, success); + } else if (UMSGPAT_ARG_TYPE_HAS_PLURAL_STYLE(argType)) { + if (!arg->isNumeric()) { + success = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + const PluralSelectorProvider &selector = + argType == UMSGPAT_ARG_TYPE_PLURAL ? pluralProvider : ordinalProvider; + // We must use the Formattable::getDouble() variant with the UErrorCode parameter + // because only this one converts non-double numeric types to double. + double offset = msgPattern.getPluralOffset(i); + PluralSelectorContext context(i, argName, *arg, offset, success); + int32_t subMsgStart = PluralFormat::findSubMessage( + msgPattern, i, selector, &context, arg->getDouble(success), success); + formatComplexSubMessage(subMsgStart, &context, arguments, argumentNames, + cnt, appendTo, success); + } else if (argType == UMSGPAT_ARG_TYPE_SELECT) { + int32_t subMsgStart = SelectFormat::findSubMessage(msgPattern, i, arg->getString(success), success); + formatComplexSubMessage(subMsgStart, nullptr, arguments, argumentNames, + cnt, appendTo, success); + } else { + // This should never happen. + success = U_INTERNAL_PROGRAM_ERROR; + return; + } + ignore = updateMetaData(appendTo, prevDestLength, ignore, arg); + prevIndex = msgPattern.getPart(argLimit).getLimit(); + i = argLimit; + } +} + + +void MessageFormat::formatComplexSubMessage(int32_t msgStart, + const void *plNumber, + const Formattable* arguments, + const UnicodeString *argumentNames, + int32_t cnt, + AppendableWrapper& appendTo, + UErrorCode& success) const { + if (U_FAILURE(success)) { + return; + } + + if (!MessageImpl::jdkAposMode(msgPattern)) { + format(msgStart, plNumber, arguments, argumentNames, cnt, appendTo, nullptr, success); + return; + } + + // JDK compatibility mode: (see JDK MessageFormat.format() API docs) + // - remove SKIP_SYNTAX; that is, remove half of the apostrophes + // - if the result string contains an open curly brace '{' then + // instantiate a temporary MessageFormat object and format again; + // otherwise just append the result string + const UnicodeString& msgString = msgPattern.getPatternString(); + UnicodeString sb; + int32_t prevIndex = msgPattern.getPart(msgStart).getLimit(); + for (int32_t i = msgStart;;) { + const MessagePattern::Part& part = msgPattern.getPart(++i); + const UMessagePatternPartType type = part.getType(); + int32_t index = part.getIndex(); + if (type == UMSGPAT_PART_TYPE_MSG_LIMIT) { + sb.append(msgString, prevIndex, index - prevIndex); + break; + } else if (type == UMSGPAT_PART_TYPE_REPLACE_NUMBER || type == UMSGPAT_PART_TYPE_SKIP_SYNTAX) { + sb.append(msgString, prevIndex, index - prevIndex); + if (type == UMSGPAT_PART_TYPE_REPLACE_NUMBER) { + const PluralSelectorContext &pluralNumber = + *static_cast(plNumber); + if(pluralNumber.forReplaceNumber) { + // number-offset was already formatted. + sb.append(pluralNumber.numberString); + } else { + const NumberFormat* nf = getDefaultNumberFormat(success); + sb.append(nf->format(pluralNumber.number, sb, success)); + } + } + prevIndex = part.getLimit(); + } else if (type == UMSGPAT_PART_TYPE_ARG_START) { + sb.append(msgString, prevIndex, index - prevIndex); + prevIndex = index; + i = msgPattern.getLimitPartIndex(i); + index = msgPattern.getPart(i).getLimit(); + MessageImpl::appendReducedApostrophes(msgString, prevIndex, index, sb); + prevIndex = index; + } + } + if (sb.indexOf(LEFT_CURLY_BRACE) >= 0) { + UnicodeString emptyPattern; // gcc 3.3.3 fails with "UnicodeString()" as the first parameter. + MessageFormat subMsgFormat(emptyPattern, fLocale, success); + subMsgFormat.applyPattern(sb, UMSGPAT_APOS_DOUBLE_REQUIRED, nullptr, success); + subMsgFormat.format(0, nullptr, arguments, argumentNames, cnt, appendTo, nullptr, success); + } else { + appendTo.append(sb); + } +} + + +UnicodeString MessageFormat::getLiteralStringUntilNextArgument(int32_t from) const { + const UnicodeString& msgString=msgPattern.getPatternString(); + int32_t prevIndex=msgPattern.getPart(from).getLimit(); + UnicodeString b; + for (int32_t i = from + 1; ; ++i) { + const MessagePattern::Part& part = msgPattern.getPart(i); + const UMessagePatternPartType type=part.getType(); + int32_t index=part.getIndex(); + b.append(msgString, prevIndex, index - prevIndex); + if(type==UMSGPAT_PART_TYPE_ARG_START || type==UMSGPAT_PART_TYPE_MSG_LIMIT) { + return b; + } + // Unexpected Part "part" in parsed message. + U_ASSERT(type==UMSGPAT_PART_TYPE_SKIP_SYNTAX || type==UMSGPAT_PART_TYPE_INSERT_CHAR); + prevIndex=part.getLimit(); + } +} + + +FieldPosition* MessageFormat::updateMetaData(AppendableWrapper& /*dest*/, int32_t /*prevLength*/, + FieldPosition* /*fp*/, const Formattable* /*argId*/) const { + // Unlike in Java, there are no field attributes defined for MessageFormat. Do nothing. + return nullptr; + /* + if (fp != nullptr && Field.ARGUMENT.equals(fp.getFieldAttribute())) { + fp->setBeginIndex(prevLength); + fp->setEndIndex(dest.get_length()); + return nullptr; + } + return fp; + */ +} + +int32_t +MessageFormat::findOtherSubMessage(int32_t partIndex) const { + int32_t count=msgPattern.countParts(); + const MessagePattern::Part *part = &msgPattern.getPart(partIndex); + if(MessagePattern::Part::hasNumericValue(part->getType())) { + ++partIndex; + } + // Iterate over (ARG_SELECTOR [ARG_INT|ARG_DOUBLE] message) tuples + // until ARG_LIMIT or end of plural-only pattern. + UnicodeString other(false, OTHER_STRING, 5); + do { + part=&msgPattern.getPart(partIndex++); + UMessagePatternPartType type=part->getType(); + if(type==UMSGPAT_PART_TYPE_ARG_LIMIT) { + break; + } + U_ASSERT(type==UMSGPAT_PART_TYPE_ARG_SELECTOR); + // part is an ARG_SELECTOR followed by an optional explicit value, and then a message + if(msgPattern.partSubstringMatches(*part, other)) { + return partIndex; + } + if(MessagePattern::Part::hasNumericValue(msgPattern.getPartType(partIndex))) { + ++partIndex; // skip the numeric-value part of "=1" etc. + } + partIndex=msgPattern.getLimitPartIndex(partIndex); + } while(++partIndex 0) { + if (!allocateArgTypes(argTypeCount, ec)) { + return; + } + uprv_memcpy(argTypes, that.argTypes, argTypeCount * sizeof(argTypes[0])); + } + if (cachedFormatters != nullptr) { + uhash_removeAll(cachedFormatters); + } + if (customFormatArgStarts != nullptr) { + uhash_removeAll(customFormatArgStarts); + } + if (that.cachedFormatters) { + if (cachedFormatters == nullptr) { + cachedFormatters=uhash_open(uhash_hashLong, uhash_compareLong, + equalFormatsForHash, &ec); + if (U_FAILURE(ec)) { + return; + } + uhash_setValueDeleter(cachedFormatters, uprv_deleteUObject); + } + + const int32_t count = uhash_count(that.cachedFormatters); + int32_t pos, idx; + for (idx = 0, pos = UHASH_FIRST; idx < count && U_SUCCESS(ec); ++idx) { + const UHashElement* cur = uhash_nextElement(that.cachedFormatters, &pos); + Format* newFormat = ((Format*)(cur->value.pointer))->clone(); + if (newFormat) { + uhash_iput(cachedFormatters, cur->key.integer, newFormat, &ec); + } else { + ec = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + } + if (that.customFormatArgStarts) { + if (customFormatArgStarts == nullptr) { + customFormatArgStarts=uhash_open(uhash_hashLong, uhash_compareLong, + nullptr, &ec); + } + const int32_t count = uhash_count(that.customFormatArgStarts); + int32_t pos, idx; + for (idx = 0, pos = UHASH_FIRST; idx < count && U_SUCCESS(ec); ++idx) { + const UHashElement* cur = uhash_nextElement(that.customFormatArgStarts, &pos); + uhash_iputi(customFormatArgStarts, cur->key.integer, cur->value.integer, &ec); + } + } +} + + +Formattable* +MessageFormat::parse(int32_t msgStart, + const UnicodeString& source, + ParsePosition& pos, + int32_t& count, + UErrorCode& ec) const { + count = 0; + if (U_FAILURE(ec)) { + pos.setErrorIndex(pos.getIndex()); + return nullptr; + } + // parse() does not work with named arguments. + if (msgPattern.hasNamedArguments()) { + ec = U_ARGUMENT_TYPE_MISMATCH; + pos.setErrorIndex(pos.getIndex()); + return nullptr; + } + LocalArray resultArray(new Formattable[argTypeCount ? argTypeCount : 1]); + const UnicodeString& msgString=msgPattern.getPatternString(); + int32_t prevIndex=msgPattern.getPart(msgStart).getLimit(); + int32_t sourceOffset = pos.getIndex(); + ParsePosition tempStatus(0); + + for(int32_t i=msgStart+1; ; ++i) { + UBool haveArgResult = false; + const MessagePattern::Part* part=&msgPattern.getPart(i); + const UMessagePatternPartType type=part->getType(); + int32_t index=part->getIndex(); + // Make sure the literal string matches. + int32_t len = index - prevIndex; + if (len == 0 || (0 == msgString.compare(prevIndex, len, source, sourceOffset, len))) { + sourceOffset += len; + prevIndex += len; + } else { + pos.setErrorIndex(sourceOffset); + return nullptr; // leave index as is to signal error + } + if(type==UMSGPAT_PART_TYPE_MSG_LIMIT) { + // Things went well! Done. + pos.setIndex(sourceOffset); + return resultArray.orphan(); + } + if(type==UMSGPAT_PART_TYPE_SKIP_SYNTAX || type==UMSGPAT_PART_TYPE_INSERT_CHAR) { + prevIndex=part->getLimit(); + continue; + } + // We do not support parsing Plural formats. (No REPLACE_NUMBER here.) + // Unexpected Part "part" in parsed message. + U_ASSERT(type==UMSGPAT_PART_TYPE_ARG_START); + int32_t argLimit=msgPattern.getLimitPartIndex(i); + + UMessagePatternArgType argType=part->getArgType(); + part=&msgPattern.getPart(++i); + int32_t argNumber = part->getValue(); // ARG_NUMBER + UnicodeString key; + ++i; + const Format* formatter = nullptr; + Formattable& argResult = resultArray[argNumber]; + + if(cachedFormatters!=nullptr && (formatter = getCachedFormatter(i - 2))!=nullptr) { + // Just parse using the formatter. + tempStatus.setIndex(sourceOffset); + formatter->parseObject(source, argResult, tempStatus); + if (tempStatus.getIndex() == sourceOffset) { + pos.setErrorIndex(sourceOffset); + return nullptr; // leave index as is to signal error + } + sourceOffset = tempStatus.getIndex(); + haveArgResult = true; + } else if( + argType==UMSGPAT_ARG_TYPE_NONE || (cachedFormatters && uhash_iget(cachedFormatters, i -2))) { + // We arrive here if getCachedFormatter returned nullptr, but there was actually an element in the hash table. + // This can only happen if the hash table contained a DummyFormat, so the if statement above is a check + // for the hash table containing DummyFormat. + + // Match as a string. + // if at end, use longest possible match + // otherwise uses first match to intervening string + // does NOT recursively try all possibilities + UnicodeString stringAfterArgument = getLiteralStringUntilNextArgument(argLimit); + int32_t next; + if (!stringAfterArgument.isEmpty()) { + next = source.indexOf(stringAfterArgument, sourceOffset); + } else { + next = source.length(); + } + if (next < 0) { + pos.setErrorIndex(sourceOffset); + return nullptr; // leave index as is to signal error + } else { + UnicodeString strValue(source.tempSubString(sourceOffset, next - sourceOffset)); + UnicodeString compValue; + compValue.append(LEFT_CURLY_BRACE); + itos(argNumber, compValue); + compValue.append(RIGHT_CURLY_BRACE); + if (0 != strValue.compare(compValue)) { + argResult.setString(strValue); + haveArgResult = true; + } + sourceOffset = next; + } + } else if(argType==UMSGPAT_ARG_TYPE_CHOICE) { + tempStatus.setIndex(sourceOffset); + double choiceResult = ChoiceFormat::parseArgument(msgPattern, i, source, tempStatus); + if (tempStatus.getIndex() == sourceOffset) { + pos.setErrorIndex(sourceOffset); + return nullptr; // leave index as is to signal error + } + argResult.setDouble(choiceResult); + haveArgResult = true; + sourceOffset = tempStatus.getIndex(); + } else if(UMSGPAT_ARG_TYPE_HAS_PLURAL_STYLE(argType) || argType==UMSGPAT_ARG_TYPE_SELECT) { + // Parsing not supported. + ec = U_UNSUPPORTED_ERROR; + return nullptr; + } else { + // This should never happen. + ec = U_INTERNAL_PROGRAM_ERROR; + return nullptr; + } + if (haveArgResult && count <= argNumber) { + count = argNumber + 1; + } + prevIndex=msgPattern.getPart(argLimit).getLimit(); + i=argLimit; + } +} +// ------------------------------------- +// Parses the source pattern and returns the Formattable objects array, +// the array count and the ending parse position. The caller of this method +// owns the array. + +Formattable* +MessageFormat::parse(const UnicodeString& source, + ParsePosition& pos, + int32_t& count) const { + UErrorCode ec = U_ZERO_ERROR; + return parse(0, source, pos, count, ec); +} + +// ------------------------------------- +// Parses the source string and returns the array of +// Formattable objects and the array count. The caller +// owns the returned array. + +Formattable* +MessageFormat::parse(const UnicodeString& source, + int32_t& cnt, + UErrorCode& success) const +{ + if (msgPattern.hasNamedArguments()) { + success = U_ARGUMENT_TYPE_MISMATCH; + return nullptr; + } + ParsePosition status(0); + // Calls the actual implementation method and starts + // from zero offset of the source text. + Formattable* result = parse(source, status, cnt); + if (status.getIndex() == 0) { + success = U_MESSAGE_PARSE_ERROR; + delete[] result; + return nullptr; + } + return result; +} + +// ------------------------------------- +// Parses the source text and copy into the result buffer. + +void +MessageFormat::parseObject( const UnicodeString& source, + Formattable& result, + ParsePosition& status) const +{ + int32_t cnt = 0; + Formattable* tmpResult = parse(source, status, cnt); + if (tmpResult != nullptr) + result.adoptArray(tmpResult, cnt); +} + +UnicodeString +MessageFormat::autoQuoteApostrophe(const UnicodeString& pattern, UErrorCode& status) { + UnicodeString result; + if (U_SUCCESS(status)) { + int32_t plen = pattern.length(); + const char16_t* pat = pattern.getBuffer(); + int32_t blen = plen * 2 + 1; // space for null termination, convenience + char16_t* buf = result.getBuffer(blen); + if (buf == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + int32_t len = umsg_autoQuoteApostrophe(pat, plen, buf, blen, &status); + result.releaseBuffer(U_SUCCESS(status) ? len : 0); + } + } + if (U_FAILURE(status)) { + result.setToBogus(); + } + return result; +} + +// ------------------------------------- + +static Format* makeRBNF(URBNFRuleSetTag tag, const Locale& locale, const UnicodeString& defaultRuleSet, UErrorCode& ec) { + RuleBasedNumberFormat* fmt = new RuleBasedNumberFormat(tag, locale, ec); + if (fmt == nullptr) { + ec = U_MEMORY_ALLOCATION_ERROR; + } else if (U_SUCCESS(ec) && defaultRuleSet.length() > 0) { + UErrorCode localStatus = U_ZERO_ERROR; // ignore unrecognized default rule set + fmt->setDefaultRuleSet(defaultRuleSet, localStatus); + } + return fmt; +} + +void MessageFormat::cacheExplicitFormats(UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + + if (cachedFormatters != nullptr) { + uhash_removeAll(cachedFormatters); + } + if (customFormatArgStarts != nullptr) { + uhash_removeAll(customFormatArgStarts); + } + + // The last two "parts" can at most be ARG_LIMIT and MSG_LIMIT + // which we need not examine. + int32_t limit = msgPattern.countParts() - 2; + argTypeCount = 0; + // We also need not look at the first two "parts" + // (at most MSG_START and ARG_START) in this loop. + // We determine the argTypeCount first so that we can allocateArgTypes + // so that the next loop can set argTypes[argNumber]. + // (This is for the C API which needs the argTypes to read its va_arg list.) + for (int32_t i = 2; i < limit && U_SUCCESS(status); ++i) { + const MessagePattern::Part& part = msgPattern.getPart(i); + if (part.getType() == UMSGPAT_PART_TYPE_ARG_NUMBER) { + const int argNumber = part.getValue(); + if (argNumber >= argTypeCount) { + argTypeCount = argNumber + 1; + } + } + } + if (!allocateArgTypes(argTypeCount, status)) { + return; + } + // Set all argTypes to kObject, as a "none" value, for lack of any better value. + // We never use kObject for real arguments. + // We use it as "no argument yet" for the check for hasArgTypeConflicts. + for (int32_t i = 0; i < argTypeCount; ++i) { + argTypes[i] = Formattable::kObject; + } + hasArgTypeConflicts = false; + + // This loop starts at part index 1 because we do need to examine + // ARG_START parts. (But we can ignore the MSG_START.) + for (int32_t i = 1; i < limit && U_SUCCESS(status); ++i) { + const MessagePattern::Part* part = &msgPattern.getPart(i); + if (part->getType() != UMSGPAT_PART_TYPE_ARG_START) { + continue; + } + UMessagePatternArgType argType = part->getArgType(); + + int32_t argNumber = -1; + part = &msgPattern.getPart(i + 1); + if (part->getType() == UMSGPAT_PART_TYPE_ARG_NUMBER) { + argNumber = part->getValue(); + } + Formattable::Type formattableType; + + switch (argType) { + case UMSGPAT_ARG_TYPE_NONE: + formattableType = Formattable::kString; + break; + case UMSGPAT_ARG_TYPE_SIMPLE: { + int32_t index = i; + i += 2; + UnicodeString explicitType = msgPattern.getSubstring(msgPattern.getPart(i++)); + UnicodeString style; + if ((part = &msgPattern.getPart(i))->getType() == UMSGPAT_PART_TYPE_ARG_STYLE) { + style = msgPattern.getSubstring(*part); + ++i; + } + UParseError parseError; + Format* formatter = createAppropriateFormat(explicitType, style, formattableType, parseError, status); + setArgStartFormat(index, formatter, status); + break; + } + case UMSGPAT_ARG_TYPE_CHOICE: + case UMSGPAT_ARG_TYPE_PLURAL: + case UMSGPAT_ARG_TYPE_SELECTORDINAL: + formattableType = Formattable::kDouble; + break; + case UMSGPAT_ARG_TYPE_SELECT: + formattableType = Formattable::kString; + break; + default: + status = U_INTERNAL_PROGRAM_ERROR; // Should be unreachable. + formattableType = Formattable::kString; + break; + } + if (argNumber != -1) { + if (argTypes[argNumber] != Formattable::kObject && argTypes[argNumber] != formattableType) { + hasArgTypeConflicts = true; + } + argTypes[argNumber] = formattableType; + } + } +} + +Format* MessageFormat::createAppropriateFormat(UnicodeString& type, UnicodeString& style, + Formattable::Type& formattableType, UParseError& parseError, + UErrorCode& ec) { + if (U_FAILURE(ec)) { + return nullptr; + } + Format* fmt = nullptr; + int32_t typeID, styleID; + DateFormat::EStyle date_style; + int32_t firstNonSpace; + + switch (typeID = findKeyword(type, TYPE_IDS)) { + case 0: // number + formattableType = Formattable::kDouble; + switch (findKeyword(style, NUMBER_STYLE_IDS)) { + case 0: // default + fmt = NumberFormat::createInstance(fLocale, ec); + break; + case 1: // currency + fmt = NumberFormat::createCurrencyInstance(fLocale, ec); + break; + case 2: // percent + fmt = NumberFormat::createPercentInstance(fLocale, ec); + break; + case 3: // integer + formattableType = Formattable::kLong; + fmt = createIntegerFormat(fLocale, ec); + break; + default: // pattern or skeleton + firstNonSpace = PatternProps::skipWhiteSpace(style, 0); + if (style.compare(firstNonSpace, 2, u"::", 0, 2) == 0) { + // Skeleton + UnicodeString skeleton = style.tempSubString(firstNonSpace + 2); + fmt = number::NumberFormatter::forSkeleton(skeleton, ec).locale(fLocale).toFormat(ec); + } else { + // Pattern + fmt = NumberFormat::createInstance(fLocale, ec); + if (fmt) { + auto* decfmt = dynamic_cast(fmt); + if (decfmt != nullptr) { + decfmt->applyPattern(style, parseError, ec); + } + } + } + break; + } + break; + + case 1: // date + case 2: // time + formattableType = Formattable::kDate; + firstNonSpace = PatternProps::skipWhiteSpace(style, 0); + if (style.compare(firstNonSpace, 2, u"::", 0, 2) == 0) { + // Skeleton + UnicodeString skeleton = style.tempSubString(firstNonSpace + 2); + fmt = DateFormat::createInstanceForSkeleton(skeleton, fLocale, ec); + } else { + // Pattern + styleID = findKeyword(style, DATE_STYLE_IDS); + date_style = (styleID >= 0) ? DATE_STYLES[styleID] : DateFormat::kDefault; + + if (typeID == 1) { + fmt = DateFormat::createDateInstance(date_style, fLocale); + } else { + fmt = DateFormat::createTimeInstance(date_style, fLocale); + } + + if (styleID < 0 && fmt != nullptr) { + SimpleDateFormat* sdtfmt = dynamic_cast(fmt); + if (sdtfmt != nullptr) { + sdtfmt->applyPattern(style); + } + } + } + break; + + case 3: // spellout + formattableType = Formattable::kDouble; + fmt = makeRBNF(URBNF_SPELLOUT, fLocale, style, ec); + break; + case 4: // ordinal + formattableType = Formattable::kDouble; + fmt = makeRBNF(URBNF_ORDINAL, fLocale, style, ec); + break; + case 5: // duration + formattableType = Formattable::kDouble; + fmt = makeRBNF(URBNF_DURATION, fLocale, style, ec); + break; + default: + formattableType = Formattable::kString; + ec = U_ILLEGAL_ARGUMENT_ERROR; + break; + } + + return fmt; +} + + +//------------------------------------- +// Finds the string, s, in the string array, list. +int32_t MessageFormat::findKeyword(const UnicodeString& s, + const char16_t * const *list) +{ + if (s.isEmpty()) { + return 0; // default + } + + int32_t length = s.length(); + const char16_t *ps = PatternProps::trimWhiteSpace(s.getBuffer(), length); + UnicodeString buffer(false, ps, length); + // Trims the space characters and turns all characters + // in s to lower case. + buffer.toLower(""); + for (int32_t i = 0; list[i]; ++i) { + if (!buffer.compare(list[i], u_strlen(list[i]))) { + return i; + } + } + return -1; +} + +/** + * Convenience method that ought to be in NumberFormat + */ +NumberFormat* +MessageFormat::createIntegerFormat(const Locale& locale, UErrorCode& status) const { + NumberFormat *temp = NumberFormat::createInstance(locale, status); + DecimalFormat *temp2; + if (temp != nullptr && (temp2 = dynamic_cast(temp)) != nullptr) { + temp2->setMaximumFractionDigits(0); + temp2->setDecimalSeparatorAlwaysShown(false); + temp2->setParseIntegerOnly(true); + } + + return temp; +} + +/** + * Return the default number format. Used to format a numeric + * argument when subformats[i].format is nullptr. Returns nullptr + * on failure. + * + * Semantically const but may modify *this. + */ +const NumberFormat* MessageFormat::getDefaultNumberFormat(UErrorCode& ec) const { + if (defaultNumberFormat == nullptr) { + MessageFormat* t = (MessageFormat*) this; + t->defaultNumberFormat = NumberFormat::createInstance(fLocale, ec); + if (U_FAILURE(ec)) { + delete t->defaultNumberFormat; + t->defaultNumberFormat = nullptr; + } else if (t->defaultNumberFormat == nullptr) { + ec = U_MEMORY_ALLOCATION_ERROR; + } + } + return defaultNumberFormat; +} + +/** + * Return the default date format. Used to format a date + * argument when subformats[i].format is nullptr. Returns nullptr + * on failure. + * + * Semantically const but may modify *this. + */ +const DateFormat* MessageFormat::getDefaultDateFormat(UErrorCode& ec) const { + if (defaultDateFormat == nullptr) { + MessageFormat* t = (MessageFormat*) this; + t->defaultDateFormat = DateFormat::createDateTimeInstance(DateFormat::kShort, DateFormat::kShort, fLocale); + if (t->defaultDateFormat == nullptr) { + ec = U_MEMORY_ALLOCATION_ERROR; + } + } + return defaultDateFormat; +} + +UBool +MessageFormat::usesNamedArguments() const { + return msgPattern.hasNamedArguments(); +} + +int32_t +MessageFormat::getArgTypeCount() const { + return argTypeCount; +} + +UBool MessageFormat::equalFormats(const void* left, const void* right) { + return *(const Format*)left==*(const Format*)right; +} + + +bool MessageFormat::DummyFormat::operator==(const Format&) const { + return true; +} + +MessageFormat::DummyFormat* MessageFormat::DummyFormat::clone() const { + return new DummyFormat(); +} + +UnicodeString& MessageFormat::DummyFormat::format(const Formattable&, + UnicodeString& appendTo, + UErrorCode& status) const { + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } + return appendTo; +} + +UnicodeString& MessageFormat::DummyFormat::format(const Formattable&, + UnicodeString& appendTo, + FieldPosition&, + UErrorCode& status) const { + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } + return appendTo; +} + +UnicodeString& MessageFormat::DummyFormat::format(const Formattable&, + UnicodeString& appendTo, + FieldPositionIterator*, + UErrorCode& status) const { + if (U_SUCCESS(status)) { + status = U_UNSUPPORTED_ERROR; + } + return appendTo; +} + +void MessageFormat::DummyFormat::parseObject(const UnicodeString&, + Formattable&, + ParsePosition& ) const { +} + + +FormatNameEnumeration::FormatNameEnumeration(LocalPointer nameList, UErrorCode& /*status*/) { + pos=0; + fFormatNames = std::move(nameList); +} + +const UnicodeString* +FormatNameEnumeration::snext(UErrorCode& status) { + if (U_SUCCESS(status) && pos < fFormatNames->size()) { + return (const UnicodeString*)fFormatNames->elementAt(pos++); + } + return nullptr; +} + +void +FormatNameEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +FormatNameEnumeration::count(UErrorCode& /*status*/) const { + return (fFormatNames==nullptr) ? 0 : fFormatNames->size(); +} + +FormatNameEnumeration::~FormatNameEnumeration() { +} + +MessageFormat::PluralSelectorProvider::PluralSelectorProvider(const MessageFormat &mf, UPluralType t) + : msgFormat(mf), rules(nullptr), type(t) { +} + +MessageFormat::PluralSelectorProvider::~PluralSelectorProvider() { + delete rules; +} + +UnicodeString MessageFormat::PluralSelectorProvider::select(void *ctx, double number, + UErrorCode& ec) const { + if (U_FAILURE(ec)) { + return UnicodeString(false, OTHER_STRING, 5); + } + MessageFormat::PluralSelectorProvider* t = const_cast(this); + if(rules == nullptr) { + t->rules = PluralRules::forLocale(msgFormat.fLocale, type, ec); + if (U_FAILURE(ec)) { + return UnicodeString(false, OTHER_STRING, 5); + } + } + // Select a sub-message according to how the number is formatted, + // which is specified in the selected sub-message. + // We avoid this circle by looking at how + // the number is formatted in the "other" sub-message + // which must always be present and usually contains the number. + // Message authors should be consistent across sub-messages. + PluralSelectorContext &context = *static_cast(ctx); + int32_t otherIndex = msgFormat.findOtherSubMessage(context.startIndex); + context.numberArgIndex = msgFormat.findFirstPluralNumberArg(otherIndex, context.argName); + if(context.numberArgIndex > 0 && msgFormat.cachedFormatters != nullptr) { + context.formatter = + (const Format*)uhash_iget(msgFormat.cachedFormatters, context.numberArgIndex); + } + if(context.formatter == nullptr) { + context.formatter = msgFormat.getDefaultNumberFormat(ec); + context.forReplaceNumber = true; + } + if (context.number.getDouble(ec) != number) { + ec = U_INTERNAL_PROGRAM_ERROR; + return UnicodeString(false, OTHER_STRING, 5); + } + context.formatter->format(context.number, context.numberString, ec); + auto* decFmt = dynamic_cast(context.formatter); + if(decFmt != nullptr) { + number::impl::DecimalQuantity dq; + decFmt->formatToDecimalQuantity(context.number, dq, ec); + if (U_FAILURE(ec)) { + return UnicodeString(false, OTHER_STRING, 5); + } + return rules->select(dq); + } else { + return rules->select(number); + } +} + +void MessageFormat::PluralSelectorProvider::reset() { + delete rules; + rules = nullptr; +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/msgfmt_impl.h b/intl/icu/source/i18n/msgfmt_impl.h new file mode 100644 index 0000000000..84344a3a25 --- /dev/null +++ b/intl/icu/source/i18n/msgfmt_impl.h @@ -0,0 +1,45 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2008, International Business Machines Corporation and +* others. All Rights Reserved. * +******************************************************************************* +* +* File MSGFMT.H +* +******************************************************************************* +*/ + +#ifndef __MSGFMT_IMPL_H__ +#define __MSGFMT_IMPL_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/msgfmt.h" +#include "uvector.h" +#include "unicode/strenum.h" + +U_NAMESPACE_BEGIN + +class FormatNameEnumeration : public StringEnumeration { +public: + FormatNameEnumeration(LocalPointer fFormatNames, UErrorCode& status); + virtual ~FormatNameEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; +private: + int32_t pos; + LocalPointer fFormatNames; +}; + +U_NAMESPACE_END + +#endif + +#endif diff --git a/intl/icu/source/i18n/name2uni.cpp b/intl/icu/source/i18n/name2uni.cpp new file mode 100644 index 0000000000..2d26dba812 --- /dev/null +++ b/intl/icu/source/i18n/name2uni.cpp @@ -0,0 +1,259 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2001-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 06/07/01 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unifilt.h" +#include "unicode/uchar.h" +#include "unicode/uniset.h" +#include "unicode/utf16.h" +#include "cmemory.h" +#include "name2uni.h" +#include "patternprops.h" +#include "uprops.h" +#include "uinvchar.h" +#include "util.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NameUnicodeTransliterator) + +static const char16_t OPEN[] = {92,78,126,123,126,0}; // "\N~{~" +static const char16_t OPEN_DELIM = 92; // '\\' first char of OPEN +static const char16_t CLOSE_DELIM = 125; // '}' +static const char16_t SPACE = 32; // ' ' + +U_CDECL_BEGIN + +// USetAdder implementation +// Does not use uset.h to reduce code dependencies +static void U_CALLCONV +_set_add(USet *set, UChar32 c) { + uset_add(set, c); +} + +// These functions aren't used. +/*static void U_CALLCONV +_set_addRange(USet *set, UChar32 start, UChar32 end) { + ((UnicodeSet *)set)->add(start, end); +} + +static void U_CALLCONV +_set_addString(USet *set, const char16_t *str, int32_t length) { + ((UnicodeSet *)set)->add(UnicodeString((UBool)(length<0), str, length)); +}*/ + +U_CDECL_END + +/** + * Constructs a transliterator with the default delimiters '{' and + * '}'. + */ +NameUnicodeTransliterator::NameUnicodeTransliterator(UnicodeFilter* adoptedFilter) : + Transliterator(UNICODE_STRING("Name-Any", 8), adoptedFilter) { + + UnicodeSet *legalPtr = &legal; + // Get the legal character set + USetAdder sa = { + (USet *)legalPtr, // USet* == UnicodeSet* + _set_add, + nullptr, // Don't need _set_addRange + nullptr, // Don't need _set_addString + nullptr, // Don't need remove() + nullptr + }; + uprv_getCharNameCharacters(&sa); +} + +/** + * Destructor. + */ +NameUnicodeTransliterator::~NameUnicodeTransliterator() {} + +/** + * Copy constructor. + */ +NameUnicodeTransliterator::NameUnicodeTransliterator(const NameUnicodeTransliterator& o) : + Transliterator(o), legal(o.legal) {} + +/** + * Assignment operator. + */ +/*NameUnicodeTransliterator& NameUnicodeTransliterator::operator=( + const NameUnicodeTransliterator& o) { + Transliterator::operator=(o); + // not necessary: the legal sets should all be the same -- legal=o.legal; + return *this; +}*/ + +/** + * Transliterator API. + */ +NameUnicodeTransliterator* NameUnicodeTransliterator::clone() const { + return new NameUnicodeTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void NameUnicodeTransliterator::handleTransliterate(Replaceable& text, UTransPosition& offsets, + UBool isIncremental) const { + // The failure mode, here and below, is to behave like Any-Null, + // if either there is no name data (max len == 0) or there is no + // memory (malloc() => nullptr). + + int32_t maxLen = uprv_getMaxCharNameLength(); + if (maxLen == 0) { + offsets.start = offsets.limit; + return; + } + + // Accommodate the longest possible name + ++maxLen; // allow for temporary trailing space + char* cbuf = (char*) uprv_malloc(maxLen); + if (cbuf == nullptr) { + offsets.start = offsets.limit; + return; + } + + UnicodeString openPat(true, OPEN, -1); + UnicodeString str, name; + + int32_t cursor = offsets.start; + int32_t limit = offsets.limit; + + // Modes: + // 0 - looking for open delimiter + // 1 - after open delimiter + int32_t mode = 0; + int32_t openPos = -1; // open delim candidate pos + + UChar32 c; + while (cursor < limit) { + c = text.char32At(cursor); + + switch (mode) { + case 0: // looking for open delimiter + if (c == OPEN_DELIM) { // quick check first + openPos = cursor; + int32_t i = + ICU_Utility::parsePattern(openPat, text, cursor, limit); + if (i >= 0 && i < limit) { + mode = 1; + name.truncate(0); + cursor = i; + continue; // *** reprocess char32At(cursor) + } + } + break; + + case 1: // after open delimiter + // Look for legal chars. If \s+ is found, convert it + // to a single space. If closeDelimiter is found, exit + // the loop. If any other character is found, exit the + // loop. If the limit is reached, exit the loop. + + // Convert \s+ => SPACE. This assumes there are no + // runs of >1 space characters in names. + if (PatternProps::isWhiteSpace(c)) { + // Ignore leading whitespace + if (name.length() > 0 && + name.charAt(name.length()-1) != SPACE) { + name.append(SPACE); + // If we are too long then abort. maxLen includes + // temporary trailing space, so use '>'. + if (name.length() > maxLen) { + mode = 0; + } + } + break; + } + + if (c == CLOSE_DELIM) { + int32_t len = name.length(); + + // Delete trailing space, if any + if (len > 0 && + name.charAt(len-1) == SPACE) { + --len; + } + + if (uprv_isInvariantUString(name.getBuffer(), len)) { + cbuf[0] = 0; + name.extract(0, len, cbuf, maxLen, US_INV); + + UErrorCode status = U_ZERO_ERROR; + c = u_charFromName(U_EXTENDED_CHAR_NAME, cbuf, &status); + if (U_SUCCESS(status)) { + // Lookup succeeded + + // assert(U16_LENGTH(CLOSE_DELIM) == 1); + cursor++; // advance over CLOSE_DELIM + + str.truncate(0); + str.append(c); + text.handleReplaceBetween(openPos, cursor, str); + + // Adjust indices for the change in the length of + // the string. Do not assume that str.length() == + // 1, in case of surrogates. + int32_t delta = cursor - openPos - str.length(); + cursor -= delta; + limit -= delta; + // assert(cursor == openPos + str.length()); + } + } + // If the lookup failed, we leave things as-is and + // still switch to mode 0 and continue. + mode = 0; + openPos = -1; // close off candidate + continue; // *** reprocess char32At(cursor) + } + + // Check if c is a legal char. We assume here that + // legal.contains(OPEN_DELIM) is false, so when we abort a + // name, we don't have to go back to openPos+1. + if (legal.contains(c)) { + name.append(c); + // If we go past the longest possible name then abort. + // maxLen includes temporary trailing space, so use '>='. + if (name.length() >= maxLen) { + mode = 0; + } + } + + // Invalid character + else { + --cursor; // Backup and reprocess this character + mode = 0; + } + + break; + } + + cursor += U16_LENGTH(c); + } + + offsets.contextLimit += limit - offsets.limit; + offsets.limit = limit; + // In incremental mode, only advance the cursor up to the last + // open delimiter candidate. + offsets.start = (isIncremental && openPos >= 0) ? openPos : cursor; + + uprv_free(cbuf); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/name2uni.h b/intl/icu/source/i18n/name2uni.h new file mode 100644 index 0000000000..6881c6bc85 --- /dev/null +++ b/intl/icu/source/i18n/name2uni.h @@ -0,0 +1,93 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 06/07/01 aliu Creation. +********************************************************************** +*/ +#ifndef NAME2UNI_H +#define NAME2UNI_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" +#include "unicode/uniset.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that performs name to character mapping. + * It recognizes the Perl syntax \N{name}. + * @author Alan Liu + */ +class NameUnicodeTransliterator : public Transliterator { +public: + + /** + * Constructs a transliterator. + * @param adoptedFilter the filter for this transliterator. + */ + NameUnicodeTransliterator(UnicodeFilter* adoptedFilter = 0); + + /** + * Destructor. + */ + virtual ~NameUnicodeTransliterator(); + + /** + * Copy constructor. + */ + NameUnicodeTransliterator(const NameUnicodeTransliterator&); + + /** + * Transliterator API. + * @return A copy of the object. + */ + virtual NameUnicodeTransliterator* clone() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + protected: + + /** + * Implements {@link Transliterator#handleTransliterate}. + * @param text the buffer holding transliterated and + * untransliterated text + * @param offset the start and limit of the text, the position + * of the cursor, and the start and limit of transliteration. + * @param incremental if true, assume more text may be coming after + * pos.contextLimit. Otherwise, assume the text is complete. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + + /** + * Set of characters which occur in Unicode character names. + */ + UnicodeSet legal; +private: + /** + * Assignment operator. + */ + NameUnicodeTransliterator& operator=(const NameUnicodeTransliterator&); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/nfrlist.h b/intl/icu/source/i18n/nfrlist.h new file mode 100644 index 0000000000..1864d4d3bd --- /dev/null +++ b/intl/icu/source/i18n/nfrlist.h @@ -0,0 +1,112 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2012, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfrlist.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#ifndef NFRLIST_H +#define NFRLIST_H + +#include "unicode/rbnf.h" + +#if U_HAVE_RBNF + +#include "unicode/uobject.h" +#include "nfrule.h" + +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +// unsafe class for internal use only. assume memory allocations succeed, indexes are valid. +// should be a template, but we can't use them + +class NFRuleList : public UMemory { +protected: + NFRule** fStuff; + uint32_t fCount; + uint32_t fCapacity; +public: + NFRuleList(uint32_t capacity = 10) + : fStuff(capacity ? (NFRule**)uprv_malloc(capacity * sizeof(NFRule*)) : nullptr) + , fCount(0) + , fCapacity(capacity) {} + ~NFRuleList() { + if (fStuff) { + for(uint32_t i = 0; i < fCount; ++i) { + delete fStuff[i]; + } + uprv_free(fStuff); + } + } + NFRule* operator[](uint32_t index) const { return fStuff != nullptr ? fStuff[index] : nullptr; } + NFRule* remove(uint32_t index) { + if (fStuff == nullptr) { + return nullptr; + } + NFRule* result = fStuff[index]; + fCount -= 1; + for (uint32_t i = index; i < fCount; ++i) { // assumes small arrays + fStuff[i] = fStuff[i+1]; + } + return result; + } + void add(NFRule* thing) { + if (fCount == fCapacity) { + fCapacity += 10; + fStuff = (NFRule**)uprv_realloc(fStuff, fCapacity * sizeof(NFRule*)); // assume success + } + if (fStuff != nullptr) { + fStuff[fCount++] = thing; + } else { + fCapacity = 0; + fCount = 0; + } + } + uint32_t size() const { return fCount; } + NFRule* last() const { return (fCount > 0 && fStuff != nullptr) ? fStuff[fCount-1] : nullptr; } + NFRule** release() { + add(nullptr); // ensure null termination + NFRule** result = fStuff; + fStuff = nullptr; + fCount = 0; + fCapacity = 0; + return result; + } + void deleteAll() { + NFRule** tmp = nullptr; + int32_t size = fCount; + if (size > 0) { + tmp = release(); + for (int32_t i = 0; i < size; i++) { + delete tmp[i]; + } + if (tmp) { + uprv_free(tmp); + } + } + } + +private: + NFRuleList(const NFRuleList &other); // forbid copying of this class + NFRuleList &operator=(const NFRuleList &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif + +// NFRLIST_H +#endif diff --git a/intl/icu/source/i18n/nfrs.cpp b/intl/icu/source/i18n/nfrs.cpp new file mode 100644 index 0000000000..1f4b9b9d29 --- /dev/null +++ b/intl/icu/source/i18n/nfrs.cpp @@ -0,0 +1,1035 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfrs.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#include "nfrs.h" + +#if U_HAVE_RBNF + +#include "unicode/uchar.h" +#include "nfrule.h" +#include "nfrlist.h" +#include "patternprops.h" +#include "putilimp.h" + +#ifdef RBNF_DEBUG +#include "cmemory.h" +#endif + +enum { + /** -x */ + NEGATIVE_RULE_INDEX = 0, + /** x.x */ + IMPROPER_FRACTION_RULE_INDEX = 1, + /** 0.x */ + PROPER_FRACTION_RULE_INDEX = 2, + /** x.0 */ + DEFAULT_RULE_INDEX = 3, + /** Inf */ + INFINITY_RULE_INDEX = 4, + /** NaN */ + NAN_RULE_INDEX = 5, + NON_NUMERICAL_RULE_LENGTH = 6 +}; + +U_NAMESPACE_BEGIN + +#if 0 +// euclid's algorithm works with doubles +// note, doubles only get us up to one quadrillion or so, which +// isn't as much range as we get with longs. We probably still +// want either 64-bit math, or BigInteger. + +static int64_t +util_lcm(int64_t x, int64_t y) +{ + x.abs(); + y.abs(); + + if (x == 0 || y == 0) { + return 0; + } else { + do { + if (x < y) { + int64_t t = x; x = y; y = t; + } + x -= y * (x/y); + } while (x != 0); + + return y; + } +} + +#else +/** + * Calculates the least common multiple of x and y. + */ +static int64_t +util_lcm(int64_t x, int64_t y) +{ + // binary gcd algorithm from Knuth, "The Art of Computer Programming," + // vol. 2, 1st ed., pp. 298-299 + int64_t x1 = x; + int64_t y1 = y; + + int p2 = 0; + while ((x1 & 1) == 0 && (y1 & 1) == 0) { + ++p2; + x1 >>= 1; + y1 >>= 1; + } + + int64_t t; + if ((x1 & 1) == 1) { + t = -y1; + } else { + t = x1; + } + + while (t != 0) { + while ((t & 1) == 0) { + t = t >> 1; + } + if (t > 0) { + x1 = t; + } else { + y1 = -t; + } + t = x1 - y1; + } + + int64_t gcd = x1 << p2; + + // x * y == gcd(x, y) * lcm(x, y) + return x / gcd * y; +} +#endif + +static const char16_t gPercent = 0x0025; +static const char16_t gColon = 0x003a; +static const char16_t gSemicolon = 0x003b; +static const char16_t gLineFeed = 0x000a; + +static const char16_t gPercentPercent[] = +{ + 0x25, 0x25, 0 +}; /* "%%" */ + +static const char16_t gNoparse[] = +{ + 0x40, 0x6E, 0x6F, 0x70, 0x61, 0x72, 0x73, 0x65, 0 +}; /* "@noparse" */ + +NFRuleSet::NFRuleSet(RuleBasedNumberFormat *_owner, UnicodeString* descriptions, int32_t index, UErrorCode& status) + : name() + , rules(0) + , owner(_owner) + , fractionRules() + , fIsFractionRuleSet(false) + , fIsPublic(false) + , fIsParseable(true) +{ + for (int32_t i = 0; i < NON_NUMERICAL_RULE_LENGTH; ++i) { + nonNumericalRules[i] = nullptr; + } + + if (U_FAILURE(status)) { + return; + } + + UnicodeString& description = descriptions[index]; // !!! make sure index is valid + + if (description.length() == 0) { + // throw new IllegalArgumentException("Empty rule set description"); + status = U_PARSE_ERROR; + return; + } + + // if the description begins with a rule set name (the rule set + // name can be omitted in formatter descriptions that consist + // of only one rule set), copy it out into our "name" member + // and delete it from the description + if (description.charAt(0) == gPercent) { + int32_t pos = description.indexOf(gColon); + if (pos == -1) { + // throw new IllegalArgumentException("Rule set name doesn't end in colon"); + status = U_PARSE_ERROR; + } else { + name.setTo(description, 0, pos); + while (pos < description.length() && PatternProps::isWhiteSpace(description.charAt(++pos))) { + } + description.remove(0, pos); + } + } else { + name.setTo(UNICODE_STRING_SIMPLE("%default")); + } + + if (description.length() == 0) { + // throw new IllegalArgumentException("Empty rule set description"); + status = U_PARSE_ERROR; + } + + fIsPublic = name.indexOf(gPercentPercent, 2, 0) != 0; + + if ( name.endsWith(gNoparse,8) ) { + fIsParseable = false; + name.truncate(name.length()-8); // remove the @noparse from the name + } + + // all of the other members of NFRuleSet are initialized + // by parseRules() +} + +void +NFRuleSet::parseRules(UnicodeString& description, UErrorCode& status) +{ + // start by creating a Vector whose elements are Strings containing + // the descriptions of the rules (one rule per element). The rules + // are separated by semicolons (there's no escape facility: ALL + // semicolons are rule delimiters) + + if (U_FAILURE(status)) { + return; + } + + // ensure we are starting with an empty rule list + rules.deleteAll(); + + // dlf - the original code kept a separate description array for no reason, + // so I got rid of it. The loop was too complex so I simplified it. + + UnicodeString currentDescription; + int32_t oldP = 0; + while (oldP < description.length()) { + int32_t p = description.indexOf(gSemicolon, oldP); + if (p == -1) { + p = description.length(); + } + currentDescription.setTo(description, oldP, p - oldP); + NFRule::makeRules(currentDescription, this, rules.last(), owner, rules, status); + oldP = p + 1; + } + + // for rules that didn't specify a base value, their base values + // were initialized to 0. Make another pass through the list and + // set all those rules' base values. We also remove any special + // rules from the list and put them into their own member variables + int64_t defaultBaseValue = 0; + + // (this isn't a for loop because we might be deleting items from + // the vector-- we want to make sure we only increment i when + // we _didn't_ delete anything from the vector) + int32_t rulesSize = rules.size(); + for (int32_t i = 0; i < rulesSize; i++) { + NFRule* rule = rules[i]; + int64_t baseValue = rule->getBaseValue(); + + if (baseValue == 0) { + // if the rule's base value is 0, fill in a default + // base value (this will be 1 plus the preceding + // rule's base value for regular rule sets, and the + // same as the preceding rule's base value in fraction + // rule sets) + rule->setBaseValue(defaultBaseValue, status); + } + else { + // if it's a regular rule that already knows its base value, + // check to make sure the rules are in order, and update + // the default base value for the next rule + if (baseValue < defaultBaseValue) { + // throw new IllegalArgumentException("Rules are not in order"); + status = U_PARSE_ERROR; + return; + } + defaultBaseValue = baseValue; + } + if (!fIsFractionRuleSet) { + ++defaultBaseValue; + } + } +} + +/** + * Set one of the non-numerical rules. + * @param rule The rule to set. + */ +void NFRuleSet::setNonNumericalRule(NFRule *rule) { + int64_t baseValue = rule->getBaseValue(); + if (baseValue == NFRule::kNegativeNumberRule) { + delete nonNumericalRules[NEGATIVE_RULE_INDEX]; + nonNumericalRules[NEGATIVE_RULE_INDEX] = rule; + } + else if (baseValue == NFRule::kImproperFractionRule) { + setBestFractionRule(IMPROPER_FRACTION_RULE_INDEX, rule, true); + } + else if (baseValue == NFRule::kProperFractionRule) { + setBestFractionRule(PROPER_FRACTION_RULE_INDEX, rule, true); + } + else if (baseValue == NFRule::kDefaultRule) { + setBestFractionRule(DEFAULT_RULE_INDEX, rule, true); + } + else if (baseValue == NFRule::kInfinityRule) { + delete nonNumericalRules[INFINITY_RULE_INDEX]; + nonNumericalRules[INFINITY_RULE_INDEX] = rule; + } + else if (baseValue == NFRule::kNaNRule) { + delete nonNumericalRules[NAN_RULE_INDEX]; + nonNumericalRules[NAN_RULE_INDEX] = rule; + } +} + +/** + * Determine the best fraction rule to use. Rules matching the decimal point from + * DecimalFormatSymbols become the main set of rules to use. + * @param originalIndex The index into nonNumericalRules + * @param newRule The new rule to consider + * @param rememberRule Should the new rule be added to fractionRules. + */ +void NFRuleSet::setBestFractionRule(int32_t originalIndex, NFRule *newRule, UBool rememberRule) { + if (rememberRule) { + fractionRules.add(newRule); + } + NFRule *bestResult = nonNumericalRules[originalIndex]; + if (bestResult == nullptr) { + nonNumericalRules[originalIndex] = newRule; + } + else { + // We have more than one. Which one is better? + const DecimalFormatSymbols *decimalFormatSymbols = owner->getDecimalFormatSymbols(); + if (decimalFormatSymbols->getSymbol(DecimalFormatSymbols::kDecimalSeparatorSymbol).charAt(0) + == newRule->getDecimalPoint()) + { + nonNumericalRules[originalIndex] = newRule; + } + // else leave it alone + } +} + +NFRuleSet::~NFRuleSet() +{ + for (int i = 0; i < NON_NUMERICAL_RULE_LENGTH; i++) { + if (i != IMPROPER_FRACTION_RULE_INDEX + && i != PROPER_FRACTION_RULE_INDEX + && i != DEFAULT_RULE_INDEX) + { + delete nonNumericalRules[i]; + } + // else it will be deleted via NFRuleList fractionRules + } +} + +static UBool +util_equalRules(const NFRule* rule1, const NFRule* rule2) +{ + if (rule1) { + if (rule2) { + return *rule1 == *rule2; + } + } else if (!rule2) { + return true; + } + return false; +} + +bool +NFRuleSet::operator==(const NFRuleSet& rhs) const +{ + if (rules.size() == rhs.rules.size() && + fIsFractionRuleSet == rhs.fIsFractionRuleSet && + name == rhs.name) { + + // ...then compare the non-numerical rule lists... + for (int i = 0; i < NON_NUMERICAL_RULE_LENGTH; i++) { + if (!util_equalRules(nonNumericalRules[i], rhs.nonNumericalRules[i])) { + return false; + } + } + + // ...then compare the rule lists... + for (uint32_t i = 0; i < rules.size(); ++i) { + if (*rules[i] != *rhs.rules[i]) { + return false; + } + } + return true; + } + return false; +} + +void +NFRuleSet::setDecimalFormatSymbols(const DecimalFormatSymbols &newSymbols, UErrorCode& status) { + for (uint32_t i = 0; i < rules.size(); ++i) { + rules[i]->setDecimalFormatSymbols(newSymbols, status); + } + // Switch the fraction rules to mirror the DecimalFormatSymbols. + for (int32_t nonNumericalIdx = IMPROPER_FRACTION_RULE_INDEX; nonNumericalIdx <= DEFAULT_RULE_INDEX; nonNumericalIdx++) { + if (nonNumericalRules[nonNumericalIdx]) { + for (uint32_t fIdx = 0; fIdx < fractionRules.size(); fIdx++) { + NFRule *fractionRule = fractionRules[fIdx]; + if (nonNumericalRules[nonNumericalIdx]->getBaseValue() == fractionRule->getBaseValue()) { + setBestFractionRule(nonNumericalIdx, fractionRule, false); + } + } + } + } + + for (uint32_t nnrIdx = 0; nnrIdx < NON_NUMERICAL_RULE_LENGTH; nnrIdx++) { + NFRule *rule = nonNumericalRules[nnrIdx]; + if (rule) { + rule->setDecimalFormatSymbols(newSymbols, status); + } + } +} + +#define RECURSION_LIMIT 64 + +void +NFRuleSet::format(int64_t number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const +{ + if (recursionCount >= RECURSION_LIMIT) { + // stop recursion + status = U_INVALID_STATE_ERROR; + return; + } + const NFRule *rule = findNormalRule(number); + if (rule) { // else error, but can't report it + rule->doFormat(number, toAppendTo, pos, ++recursionCount, status); + } +} + +void +NFRuleSet::format(double number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const +{ + if (recursionCount >= RECURSION_LIMIT) { + // stop recursion + status = U_INVALID_STATE_ERROR; + return; + } + const NFRule *rule = findDoubleRule(number); + if (rule) { // else error, but can't report it + rule->doFormat(number, toAppendTo, pos, ++recursionCount, status); + } +} + +const NFRule* +NFRuleSet::findDoubleRule(double number) const +{ + // if this is a fraction rule set, use findFractionRuleSetRule() + if (isFractionRuleSet()) { + return findFractionRuleSetRule(number); + } + + if (uprv_isNaN(number)) { + const NFRule *rule = nonNumericalRules[NAN_RULE_INDEX]; + if (!rule) { + rule = owner->getDefaultNaNRule(); + } + return rule; + } + + // if the number is negative, return the negative number rule + // (if there isn't a negative-number rule, we pretend it's a + // positive number) + if (number < 0) { + if (nonNumericalRules[NEGATIVE_RULE_INDEX]) { + return nonNumericalRules[NEGATIVE_RULE_INDEX]; + } else { + number = -number; + } + } + + if (uprv_isInfinite(number)) { + const NFRule *rule = nonNumericalRules[INFINITY_RULE_INDEX]; + if (!rule) { + rule = owner->getDefaultInfinityRule(); + } + return rule; + } + + // if the number isn't an integer, we use one of the fraction rules... + if (number != uprv_floor(number)) { + // if the number is between 0 and 1, return the proper + // fraction rule + if (number < 1 && nonNumericalRules[PROPER_FRACTION_RULE_INDEX]) { + return nonNumericalRules[PROPER_FRACTION_RULE_INDEX]; + } + // otherwise, return the improper fraction rule + else if (nonNumericalRules[IMPROPER_FRACTION_RULE_INDEX]) { + return nonNumericalRules[IMPROPER_FRACTION_RULE_INDEX]; + } + } + + // if there's a default rule, use it to format the number + if (nonNumericalRules[DEFAULT_RULE_INDEX]) { + return nonNumericalRules[DEFAULT_RULE_INDEX]; + } + + // and if we haven't yet returned a rule, use findNormalRule() + // to find the applicable rule + int64_t r = util64_fromDouble(number + 0.5); + return findNormalRule(r); +} + +const NFRule * +NFRuleSet::findNormalRule(int64_t number) const +{ + // if this is a fraction rule set, use findFractionRuleSetRule() + // to find the rule (we should only go into this clause if the + // value is 0) + if (fIsFractionRuleSet) { + return findFractionRuleSetRule((double)number); + } + + // if the number is negative, return the negative-number rule + // (if there isn't one, pretend the number is positive) + if (number < 0) { + if (nonNumericalRules[NEGATIVE_RULE_INDEX]) { + return nonNumericalRules[NEGATIVE_RULE_INDEX]; + } else { + number = -number; + } + } + + // we have to repeat the preceding two checks, even though we + // do them in findRule(), because the version of format() that + // takes a long bypasses findRule() and goes straight to this + // function. This function does skip the fraction rules since + // we know the value is an integer (it also skips the default + // rule, since it's considered a fraction rule. Skipping the + // default rule in this function is also how we avoid infinite + // recursion) + + // {dlf} unfortunately this fails if there are no rules except + // special rules. If there are no rules, use the default rule. + + // binary-search the rule list for the applicable rule + // (a rule is used for all values from its base value to + // the next rule's base value) + int32_t hi = rules.size(); + if (hi > 0) { + int32_t lo = 0; + + while (lo < hi) { + int32_t mid = (lo + hi) / 2; + if (rules[mid]->getBaseValue() == number) { + return rules[mid]; + } + else if (rules[mid]->getBaseValue() > number) { + hi = mid; + } + else { + lo = mid + 1; + } + } + if (hi == 0) { // bad rule set, minimum base > 0 + return nullptr; // want to throw exception here + } + + NFRule *result = rules[hi - 1]; + + // use shouldRollBack() to see whether we need to invoke the + // rollback rule (see shouldRollBack()'s documentation for + // an explanation of the rollback rule). If we do, roll back + // one rule and return that one instead of the one we'd normally + // return + if (result->shouldRollBack(number)) { + if (hi == 1) { // bad rule set, no prior rule to rollback to from this base + return nullptr; + } + result = rules[hi - 2]; + } + return result; + } + // else use the default rule + return nonNumericalRules[DEFAULT_RULE_INDEX]; +} + +/** + * If this rule is a fraction rule set, this function is used by + * findRule() to select the most appropriate rule for formatting + * the number. Basically, the base value of each rule in the rule + * set is treated as the denominator of a fraction. Whichever + * denominator can produce the fraction closest in value to the + * number passed in is the result. If there's a tie, the earlier + * one in the list wins. (If there are two rules in a row with the + * same base value, the first one is used when the numerator of the + * fraction would be 1, and the second rule is used the rest of the + * time. + * @param number The number being formatted (which will always be + * a number between 0 and 1) + * @return The rule to use to format this number + */ +const NFRule* +NFRuleSet::findFractionRuleSetRule(double number) const +{ + // the obvious way to do this (multiply the value being formatted + // by each rule's base value until you get an integral result) + // doesn't work because of rounding error. This method is more + // accurate + + // find the least common multiple of the rules' base values + // and multiply this by the number being formatted. This is + // all the precision we need, and we can do all of the rest + // of the math using integer arithmetic + int64_t leastCommonMultiple = rules[0]->getBaseValue(); + int64_t numerator; + { + for (uint32_t i = 1; i < rules.size(); ++i) { + leastCommonMultiple = util_lcm(leastCommonMultiple, rules[i]->getBaseValue()); + } + numerator = util64_fromDouble(number * (double)leastCommonMultiple + 0.5); + } + // for each rule, do the following... + int64_t tempDifference; + int64_t difference = util64_fromDouble(uprv_maxMantissa()); + int32_t winner = 0; + for (uint32_t i = 0; i < rules.size(); ++i) { + // "numerator" is the numerator of the fraction if the + // denominator is the LCD. The numerator if the rule's + // base value is the denominator is "numerator" times the + // base value divided bythe LCD. Here we check to see if + // that's an integer, and if not, how close it is to being + // an integer. + tempDifference = numerator * rules[i]->getBaseValue() % leastCommonMultiple; + + + // normalize the result of the above calculation: we want + // the numerator's distance from the CLOSEST multiple + // of the LCD + if (leastCommonMultiple - tempDifference < tempDifference) { + tempDifference = leastCommonMultiple - tempDifference; + } + + // if this is as close as we've come, keep track of how close + // that is, and the line number of the rule that did it. If + // we've scored a direct hit, we don't have to look at any more + // rules + if (tempDifference < difference) { + difference = tempDifference; + winner = i; + if (difference == 0) { + break; + } + } + } + + // if we have two successive rules that both have the winning base + // value, then the first one (the one we found above) is used if + // the numerator of the fraction is 1 and the second one is used if + // the numerator of the fraction is anything else (this lets us + // do things like "one third"/"two thirds" without having to define + // a whole bunch of extra rule sets) + if ((unsigned)(winner + 1) < rules.size() && + rules[winner + 1]->getBaseValue() == rules[winner]->getBaseValue()) { + double n = ((double)rules[winner]->getBaseValue()) * number; + if (n < 0.5 || n >= 2) { + ++winner; + } + } + + // finally, return the winning rule + return rules[winner]; +} + +/** + * Parses a string. Matches the string to be parsed against each + * of its rules (with a base value less than upperBound) and returns + * the value produced by the rule that matched the most characters + * in the source string. + * @param text The string to parse + * @param parsePosition The initial position is ignored and assumed + * to be 0. On exit, this object has been updated to point to the + * first character position this rule set didn't consume. + * @param upperBound Limits the rules that can be allowed to match. + * Only rules whose base values are strictly less than upperBound + * are considered. + * @return The numerical result of parsing this string. This will + * be the matching rule's base value, composed appropriately with + * the results of matching any of its substitutions. The object + * will be an instance of Long if it's an integral value; otherwise, + * it will be an instance of Double. This function always returns + * a valid object: If nothing matched the input string at all, + * this function returns new Long(0), and the parse position is + * left unchanged. + */ +#ifdef RBNF_DEBUG +#include + +static void dumpUS(FILE* f, const UnicodeString& us) { + int len = us.length(); + char* buf = (char *)uprv_malloc((len+1)*sizeof(char)); //new char[len+1]; + if (buf != nullptr) { + us.extract(0, len, buf); + buf[len] = 0; + fprintf(f, "%s", buf); + uprv_free(buf); //delete[] buf; + } +} +#endif + +UBool +NFRuleSet::parse(const UnicodeString& text, ParsePosition& pos, double upperBound, uint32_t nonNumericalExecutedRuleMask, Formattable& result) const +{ + // try matching each rule in the rule set against the text being + // parsed. Whichever one matches the most characters is the one + // that determines the value we return. + + result.setLong(0); + + // dump out if there's no text to parse + if (text.length() == 0) { + return 0; + } + + ParsePosition highWaterMark; + ParsePosition workingPos = pos; + +#ifdef RBNF_DEBUG + fprintf(stderr, " %x '", this); + dumpUS(stderr, name); + fprintf(stderr, "' text '"); + dumpUS(stderr, text); + fprintf(stderr, "'\n"); + fprintf(stderr, " parse negative: %d\n", this, negativeNumberRule != 0); +#endif + // Try each of the negative rules, fraction rules, infinity rules and NaN rules + for (int i = 0; i < NON_NUMERICAL_RULE_LENGTH; i++) { + if (nonNumericalRules[i] && ((nonNumericalExecutedRuleMask >> i) & 1) == 0) { + // Mark this rule as being executed so that we don't try to execute it again. + nonNumericalExecutedRuleMask |= 1 << i; + + Formattable tempResult; + UBool success = nonNumericalRules[i]->doParse(text, workingPos, 0, upperBound, nonNumericalExecutedRuleMask, tempResult); + if (success && (workingPos.getIndex() > highWaterMark.getIndex())) { + result = tempResult; + highWaterMark = workingPos; + } + workingPos = pos; + } + } +#ifdef RBNF_DEBUG + fprintf(stderr, " continue other with text '"); + dumpUS(stderr, text); + fprintf(stderr, "' hwm: %d\n", highWaterMark.getIndex()); +#endif + + // finally, go through the regular rules one at a time. We start + // at the end of the list because we want to try matching the most + // sigificant rule first (this helps ensure that we parse + // "five thousand three hundred six" as + // "(five thousand) (three hundred) (six)" rather than + // "((five thousand three) hundred) (six)"). Skip rules whose + // base values are higher than the upper bound (again, this helps + // limit ambiguity by making sure the rules that match a rule's + // are less significant than the rule containing the substitutions)/ + { + int64_t ub = util64_fromDouble(upperBound); +#ifdef RBNF_DEBUG + { + char ubstr[64]; + util64_toa(ub, ubstr, 64); + char ubstrhex[64]; + util64_toa(ub, ubstrhex, 64, 16); + fprintf(stderr, "ub: %g, i64: %s (%s)\n", upperBound, ubstr, ubstrhex); + } +#endif + for (int32_t i = rules.size(); --i >= 0 && highWaterMark.getIndex() < text.length();) { + if ((!fIsFractionRuleSet) && (rules[i]->getBaseValue() >= ub)) { + continue; + } + Formattable tempResult; + UBool success = rules[i]->doParse(text, workingPos, fIsFractionRuleSet, upperBound, nonNumericalExecutedRuleMask, tempResult); + if (success && workingPos.getIndex() > highWaterMark.getIndex()) { + result = tempResult; + highWaterMark = workingPos; + } + workingPos = pos; + } + } +#ifdef RBNF_DEBUG + fprintf(stderr, " exit\n"); +#endif + // finally, update the parse position we were passed to point to the + // first character we didn't use, and return the result that + // corresponds to that string of characters + pos = highWaterMark; + + return 1; +} + +void +NFRuleSet::appendRules(UnicodeString& result) const +{ + uint32_t i; + + // the rule set name goes first... + result.append(name); + result.append(gColon); + result.append(gLineFeed); + + // followed by the regular rules... + for (i = 0; i < rules.size(); i++) { + rules[i]->_appendRuleText(result); + result.append(gLineFeed); + } + + // followed by the special rules (if they exist) + for (i = 0; i < NON_NUMERICAL_RULE_LENGTH; ++i) { + NFRule *rule = nonNumericalRules[i]; + if (nonNumericalRules[i]) { + if (rule->getBaseValue() == NFRule::kImproperFractionRule + || rule->getBaseValue() == NFRule::kProperFractionRule + || rule->getBaseValue() == NFRule::kDefaultRule) + { + for (uint32_t fIdx = 0; fIdx < fractionRules.size(); fIdx++) { + NFRule *fractionRule = fractionRules[fIdx]; + if (fractionRule->getBaseValue() == rule->getBaseValue()) { + fractionRule->_appendRuleText(result); + result.append(gLineFeed); + } + } + } + else { + rule->_appendRuleText(result); + result.append(gLineFeed); + } + } + } +} + +// utility functions + +int64_t util64_fromDouble(double d) { + int64_t result = 0; + if (!uprv_isNaN(d)) { + double mant = uprv_maxMantissa(); + if (d < -mant) { + d = -mant; + } else if (d > mant) { + d = mant; + } + UBool neg = d < 0; + if (neg) { + d = -d; + } + result = (int64_t)uprv_floor(d); + if (neg) { + result = -result; + } + } + return result; +} + +uint64_t util64_pow(uint32_t base, uint16_t exponent) { + if (base == 0) { + return 0; + } + uint64_t result = 1; + uint64_t pow = base; + while (true) { + if ((exponent & 1) == 1) { + result *= pow; + } + exponent >>= 1; + if (exponent == 0) { + break; + } + pow *= pow; + } + return result; +} + +static const uint8_t asciiDigits[] = { + 0x30u, 0x31u, 0x32u, 0x33u, 0x34u, 0x35u, 0x36u, 0x37u, + 0x38u, 0x39u, 0x61u, 0x62u, 0x63u, 0x64u, 0x65u, 0x66u, + 0x67u, 0x68u, 0x69u, 0x6au, 0x6bu, 0x6cu, 0x6du, 0x6eu, + 0x6fu, 0x70u, 0x71u, 0x72u, 0x73u, 0x74u, 0x75u, 0x76u, + 0x77u, 0x78u, 0x79u, 0x7au, +}; + +static const char16_t kUMinus = (char16_t)0x002d; + +#ifdef RBNF_DEBUG +static const char kMinus = '-'; + +static const uint8_t digitInfo[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0x80u, 0x81u, 0x82u, 0x83u, 0x84u, 0x85u, 0x86u, 0x87u, + 0x88u, 0x89u, 0, 0, 0, 0, 0, 0, + 0, 0x8au, 0x8bu, 0x8cu, 0x8du, 0x8eu, 0x8fu, 0x90u, + 0x91u, 0x92u, 0x93u, 0x94u, 0x95u, 0x96u, 0x97u, 0x98u, + 0x99u, 0x9au, 0x9bu, 0x9cu, 0x9du, 0x9eu, 0x9fu, 0xa0u, + 0xa1u, 0xa2u, 0xa3u, 0, 0, 0, 0, 0, + 0, 0x8au, 0x8bu, 0x8cu, 0x8du, 0x8eu, 0x8fu, 0x90u, + 0x91u, 0x92u, 0x93u, 0x94u, 0x95u, 0x96u, 0x97u, 0x98u, + 0x99u, 0x9au, 0x9bu, 0x9cu, 0x9du, 0x9eu, 0x9fu, 0xa0u, + 0xa1u, 0xa2u, 0xa3u, 0, 0, 0, 0, 0, +}; + +int64_t util64_atoi(const char* str, uint32_t radix) +{ + if (radix > 36) { + radix = 36; + } else if (radix < 2) { + radix = 2; + } + int64_t lradix = radix; + + int neg = 0; + if (*str == kMinus) { + ++str; + neg = 1; + } + int64_t result = 0; + uint8_t b; + while ((b = digitInfo[*str++]) && ((b &= 0x7f) < radix)) { + result *= lradix; + result += (int32_t)b; + } + if (neg) { + result = -result; + } + return result; +} + +int64_t util64_utoi(const char16_t* str, uint32_t radix) +{ + if (radix > 36) { + radix = 36; + } else if (radix < 2) { + radix = 2; + } + int64_t lradix = radix; + + int neg = 0; + if (*str == kUMinus) { + ++str; + neg = 1; + } + int64_t result = 0; + char16_t c; + uint8_t b; + while (((c = *str++) < 0x0080) && (b = digitInfo[c]) && ((b &= 0x7f) < radix)) { + result *= lradix; + result += (int32_t)b; + } + if (neg) { + result = -result; + } + return result; +} + +uint32_t util64_toa(int64_t w, char* buf, uint32_t len, uint32_t radix, UBool raw) +{ + if (radix > 36) { + radix = 36; + } else if (radix < 2) { + radix = 2; + } + int64_t base = radix; + + char* p = buf; + if (len && (w < 0) && (radix == 10) && !raw) { + w = -w; + *p++ = kMinus; + --len; + } else if (len && (w == 0)) { + *p++ = (char)raw ? 0 : asciiDigits[0]; + --len; + } + + while (len && w != 0) { + int64_t n = w / base; + int64_t m = n * base; + int32_t d = (int32_t)(w-m); + *p++ = raw ? (char)d : asciiDigits[d]; + w = n; + --len; + } + if (len) { + *p = 0; // null terminate if room for caller convenience + } + + len = p - buf; + if (*buf == kMinus) { + ++buf; + } + while (--p > buf) { + char c = *p; + *p = *buf; + *buf = c; + ++buf; + } + + return len; +} +#endif + +uint32_t util64_tou(int64_t w, char16_t* buf, uint32_t len, uint32_t radix, UBool raw) +{ + if (radix > 36) { + radix = 36; + } else if (radix < 2) { + radix = 2; + } + int64_t base = radix; + + char16_t* p = buf; + if (len && (w < 0) && (radix == 10) && !raw) { + w = -w; + *p++ = kUMinus; + --len; + } else if (len && (w == 0)) { + *p++ = (char16_t)raw ? 0 : asciiDigits[0]; + --len; + } + + while (len && (w != 0)) { + int64_t n = w / base; + int64_t m = n * base; + int32_t d = (int32_t)(w-m); + *p++ = (char16_t)(raw ? d : asciiDigits[d]); + w = n; + --len; + } + if (len) { + *p = 0; // null terminate if room for caller convenience + } + + len = (uint32_t)(p - buf); + if (*buf == kUMinus) { + ++buf; + } + while (--p > buf) { + char16_t c = *p; + *p = *buf; + *buf = c; + ++buf; + } + + return len; +} + + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif diff --git a/intl/icu/source/i18n/nfrs.h b/intl/icu/source/i18n/nfrs.h new file mode 100644 index 0000000000..a1beedda17 --- /dev/null +++ b/intl/icu/source/i18n/nfrs.h @@ -0,0 +1,111 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfrs.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#ifndef NFRS_H +#define NFRS_H + +#include "unicode/uobject.h" +#include "unicode/rbnf.h" + +#if U_HAVE_RBNF + +#include "unicode/utypes.h" +#include "unicode/umisc.h" + +#include "nfrlist.h" + +U_NAMESPACE_BEGIN + +class NFRuleSet : public UMemory { +public: + NFRuleSet(RuleBasedNumberFormat *owner, UnicodeString* descriptions, int32_t index, UErrorCode& status); + void parseRules(UnicodeString& rules, UErrorCode& status); + void setNonNumericalRule(NFRule *rule); + void setBestFractionRule(int32_t originalIndex, NFRule *newRule, UBool rememberRule); + void makeIntoFractionRuleSet() { fIsFractionRuleSet = true; } + + ~NFRuleSet(); + + bool operator==(const NFRuleSet& rhs) const; + bool operator!=(const NFRuleSet& rhs) const { return !operator==(rhs); } + + UBool isPublic() const { return fIsPublic; } + + UBool isParseable() const { return fIsParseable; } + + UBool isFractionRuleSet() const { return fIsFractionRuleSet; } + + void getName(UnicodeString& result) const { result.setTo(name); } + UBool isNamed(const UnicodeString& _name) const { return this->name == _name; } + + void format(int64_t number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + void format(double number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + + UBool parse(const UnicodeString& text, ParsePosition& pos, double upperBound, uint32_t nonNumericalExecutedRuleMask, Formattable& result) const; + + void appendRules(UnicodeString& result) const; // toString + + void setDecimalFormatSymbols(const DecimalFormatSymbols &newSymbols, UErrorCode& status); + + const RuleBasedNumberFormat *getOwner() const { return owner; } +private: + const NFRule * findNormalRule(int64_t number) const; + const NFRule * findDoubleRule(double number) const; + const NFRule * findFractionRuleSetRule(double number) const; + + friend class NFSubstitution; + +private: + UnicodeString name; + NFRuleList rules; + NFRule *nonNumericalRules[6]; + RuleBasedNumberFormat *owner; + NFRuleList fractionRules; + UBool fIsFractionRuleSet; + UBool fIsPublic; + UBool fIsParseable; + + NFRuleSet(const NFRuleSet &other); // forbid copying of this class + NFRuleSet &operator=(const NFRuleSet &other); // forbid copying of this class +}; + +// utilities from old llong.h +// convert mantissa portion of double to int64 +int64_t util64_fromDouble(double d); + +// raise radix to the power exponent, only non-negative exponents +// Arithmetic is performed in unsigned space since overflow in +// signed space is undefined behavior. +uint64_t util64_pow(uint32_t radix, uint16_t exponent); + +// convert n to digit string in buffer, return length of string +uint32_t util64_tou(int64_t n, char16_t* buffer, uint32_t buflen, uint32_t radix = 10, UBool raw = false); + +#ifdef RBNF_DEBUG +int64_t util64_utoi(const char16_t* str, uint32_t radix = 10); +uint32_t util64_toa(int64_t n, char* buffer, uint32_t buflen, uint32_t radix = 10, UBool raw = false); +int64_t util64_atoi(const char* str, uint32_t radix); +#endif + + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif + +// NFRS_H +#endif diff --git a/intl/icu/source/i18n/nfrule.cpp b/intl/icu/source/i18n/nfrule.cpp new file mode 100644 index 0000000000..51bd4c974f --- /dev/null +++ b/intl/icu/source/i18n/nfrule.cpp @@ -0,0 +1,1632 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfrule.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#include "nfrule.h" + +#if U_HAVE_RBNF + +#include "unicode/localpointer.h" +#include "unicode/rbnf.h" +#include "unicode/tblcoll.h" +#include "unicode/plurfmt.h" +#include "unicode/upluralrules.h" +#include "unicode/coleitr.h" +#include "unicode/uchar.h" +#include "nfrs.h" +#include "nfrlist.h" +#include "nfsubs.h" +#include "patternprops.h" +#include "putilimp.h" + +U_NAMESPACE_BEGIN + +NFRule::NFRule(const RuleBasedNumberFormat* _rbnf, const UnicodeString &_ruleText, UErrorCode &status) + : baseValue((int32_t)0) + , radix(10) + , exponent(0) + , decimalPoint(0) + , fRuleText(_ruleText) + , sub1(nullptr) + , sub2(nullptr) + , formatter(_rbnf) + , rulePatternFormat(nullptr) +{ + if (!fRuleText.isEmpty()) { + parseRuleDescriptor(fRuleText, status); + } +} + +NFRule::~NFRule() +{ + if (sub1 != sub2) { + delete sub2; + sub2 = nullptr; + } + delete sub1; + sub1 = nullptr; + delete rulePatternFormat; + rulePatternFormat = nullptr; +} + +static const char16_t gLeftBracket = 0x005b; +static const char16_t gRightBracket = 0x005d; +static const char16_t gColon = 0x003a; +static const char16_t gZero = 0x0030; +static const char16_t gNine = 0x0039; +static const char16_t gSpace = 0x0020; +static const char16_t gSlash = 0x002f; +static const char16_t gGreaterThan = 0x003e; +static const char16_t gLessThan = 0x003c; +static const char16_t gComma = 0x002c; +static const char16_t gDot = 0x002e; +static const char16_t gTick = 0x0027; +//static const char16_t gMinus = 0x002d; +static const char16_t gSemicolon = 0x003b; +static const char16_t gX = 0x0078; + +static const char16_t gMinusX[] = {0x2D, 0x78, 0}; /* "-x" */ +static const char16_t gInf[] = {0x49, 0x6E, 0x66, 0}; /* "Inf" */ +static const char16_t gNaN[] = {0x4E, 0x61, 0x4E, 0}; /* "NaN" */ + +static const char16_t gDollarOpenParenthesis[] = {0x24, 0x28, 0}; /* "$(" */ +static const char16_t gClosedParenthesisDollar[] = {0x29, 0x24, 0}; /* ")$" */ + +static const char16_t gLessLess[] = {0x3C, 0x3C, 0}; /* "<<" */ +static const char16_t gLessPercent[] = {0x3C, 0x25, 0}; /* "<%" */ +static const char16_t gLessHash[] = {0x3C, 0x23, 0}; /* "<#" */ +static const char16_t gLessZero[] = {0x3C, 0x30, 0}; /* "<0" */ +static const char16_t gGreaterGreater[] = {0x3E, 0x3E, 0}; /* ">>" */ +static const char16_t gGreaterPercent[] = {0x3E, 0x25, 0}; /* ">%" */ +static const char16_t gGreaterHash[] = {0x3E, 0x23, 0}; /* ">#" */ +static const char16_t gGreaterZero[] = {0x3E, 0x30, 0}; /* ">0" */ +static const char16_t gEqualPercent[] = {0x3D, 0x25, 0}; /* "=%" */ +static const char16_t gEqualHash[] = {0x3D, 0x23, 0}; /* "=#" */ +static const char16_t gEqualZero[] = {0x3D, 0x30, 0}; /* "=0" */ +static const char16_t gGreaterGreaterGreater[] = {0x3E, 0x3E, 0x3E, 0}; /* ">>>" */ + +static const char16_t * const RULE_PREFIXES[] = { + gLessLess, gLessPercent, gLessHash, gLessZero, + gGreaterGreater, gGreaterPercent,gGreaterHash, gGreaterZero, + gEqualPercent, gEqualHash, gEqualZero, nullptr +}; + +void +NFRule::makeRules(UnicodeString& description, + NFRuleSet *owner, + const NFRule *predecessor, + const RuleBasedNumberFormat *rbnf, + NFRuleList& rules, + UErrorCode& status) +{ + // we know we're making at least one rule, so go ahead and + // new it up and initialize its basevalue and divisor + // (this also strips the rule descriptor, if any, off the + // description string) + NFRule* rule1 = new NFRule(rbnf, description, status); + /* test for nullptr */ + if (rule1 == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + description = rule1->fRuleText; + + // check the description to see whether there's text enclosed + // in brackets + int32_t brack1 = description.indexOf(gLeftBracket); + int32_t brack2 = brack1 < 0 ? -1 : description.indexOf(gRightBracket); + + // if the description doesn't contain a matched pair of brackets, + // or if it's of a type that doesn't recognize bracketed text, + // then leave the description alone, initialize the rule's + // rule text and substitutions, and return that rule + if (brack2 < 0 || brack1 > brack2 + || rule1->getType() == kProperFractionRule + || rule1->getType() == kNegativeNumberRule + || rule1->getType() == kInfinityRule + || rule1->getType() == kNaNRule) + { + rule1->extractSubstitutions(owner, description, predecessor, status); + } + else { + // if the description does contain a matched pair of brackets, + // then it's really shorthand for two rules (with one exception) + NFRule* rule2 = nullptr; + UnicodeString sbuf; + + // we'll actually only split the rule into two rules if its + // base value is an even multiple of its divisor (or it's one + // of the special rules) + if ((rule1->baseValue > 0 + && (rule1->baseValue % util64_pow(rule1->radix, rule1->exponent)) == 0) + || rule1->getType() == kImproperFractionRule + || rule1->getType() == kDefaultRule) { + + // if it passes that test, new up the second rule. If the + // rule set both rules will belong to is a fraction rule + // set, they both have the same base value; otherwise, + // increment the original rule's base value ("rule1" actually + // goes SECOND in the rule set's rule list) + rule2 = new NFRule(rbnf, UnicodeString(), status); + /* test for nullptr */ + if (rule2 == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (rule1->baseValue >= 0) { + rule2->baseValue = rule1->baseValue; + if (!owner->isFractionRuleSet()) { + ++rule1->baseValue; + } + } + + // if the description began with "x.x" and contains bracketed + // text, it describes both the improper fraction rule and + // the proper fraction rule + else if (rule1->getType() == kImproperFractionRule) { + rule2->setType(kProperFractionRule); + } + + // if the description began with "x.0" and contains bracketed + // text, it describes both the default rule and the + // improper fraction rule + else if (rule1->getType() == kDefaultRule) { + rule2->baseValue = rule1->baseValue; + rule1->setType(kImproperFractionRule); + } + + // both rules have the same radix and exponent (i.e., the + // same divisor) + rule2->radix = rule1->radix; + rule2->exponent = rule1->exponent; + + // rule2's rule text omits the stuff in brackets: initialize + // its rule text and substitutions accordingly + sbuf.append(description, 0, brack1); + if (brack2 + 1 < description.length()) { + sbuf.append(description, brack2 + 1, description.length() - brack2 - 1); + } + rule2->extractSubstitutions(owner, sbuf, predecessor, status); + } + + // rule1's text includes the text in the brackets but omits + // the brackets themselves: initialize _its_ rule text and + // substitutions accordingly + sbuf.setTo(description, 0, brack1); + sbuf.append(description, brack1 + 1, brack2 - brack1 - 1); + if (brack2 + 1 < description.length()) { + sbuf.append(description, brack2 + 1, description.length() - brack2 - 1); + } + rule1->extractSubstitutions(owner, sbuf, predecessor, status); + + // if we only have one rule, return it; if we have two, return + // a two-element array containing them (notice that rule2 goes + // BEFORE rule1 in the list: in all cases, rule2 OMITS the + // material in the brackets and rule1 INCLUDES the material + // in the brackets) + if (rule2 != nullptr) { + if (rule2->baseValue >= kNoBase) { + rules.add(rule2); + } + else { + owner->setNonNumericalRule(rule2); + } + } + } + if (rule1->baseValue >= kNoBase) { + rules.add(rule1); + } + else { + owner->setNonNumericalRule(rule1); + } +} + +/** + * This function parses the rule's rule descriptor (i.e., the base + * value and/or other tokens that precede the rule's rule text + * in the description) and sets the rule's base value, radix, and + * exponent according to the descriptor. (If the description doesn't + * include a rule descriptor, then this function sets everything to + * default values and the rule set sets the rule's real base value). + * @param description The rule's description + * @return If "description" included a rule descriptor, this is + * "description" with the descriptor and any trailing whitespace + * stripped off. Otherwise; it's "descriptor" unchangd. + */ +void +NFRule::parseRuleDescriptor(UnicodeString& description, UErrorCode& status) +{ + // the description consists of a rule descriptor and a rule body, + // separated by a colon. The rule descriptor is optional. If + // it's omitted, just set the base value to 0. + int32_t p = description.indexOf(gColon); + if (p != -1) { + // copy the descriptor out into its own string and strip it, + // along with any trailing whitespace, out of the original + // description + UnicodeString descriptor; + descriptor.setTo(description, 0, p); + + ++p; + while (p < description.length() && PatternProps::isWhiteSpace(description.charAt(p))) { + ++p; + } + description.removeBetween(0, p); + + // check first to see if the rule descriptor matches the token + // for one of the special rules. If it does, set the base + // value to the correct identifier value + int descriptorLength = descriptor.length(); + char16_t firstChar = descriptor.charAt(0); + char16_t lastChar = descriptor.charAt(descriptorLength - 1); + if (firstChar >= gZero && firstChar <= gNine && lastChar != gX) { + // if the rule descriptor begins with a digit, it's a descriptor + // for a normal rule + // since we don't have Long.parseLong, and this isn't much work anyway, + // just build up the value as we encounter the digits. + int64_t val = 0; + p = 0; + char16_t c = gSpace; + + // begin parsing the descriptor: copy digits + // into "tempValue", skip periods, commas, and spaces, + // stop on a slash or > sign (or at the end of the string), + // and throw an exception on any other character + int64_t ll_10 = 10; + while (p < descriptorLength) { + c = descriptor.charAt(p); + if (c >= gZero && c <= gNine) { + val = val * ll_10 + (int32_t)(c - gZero); + } + else if (c == gSlash || c == gGreaterThan) { + break; + } + else if (PatternProps::isWhiteSpace(c) || c == gComma || c == gDot) { + } + else { + // throw new IllegalArgumentException("Illegal character in rule descriptor"); + status = U_PARSE_ERROR; + return; + } + ++p; + } + + // we have the base value, so set it + setBaseValue(val, status); + + // if we stopped the previous loop on a slash, we're + // now parsing the rule's radix. Again, accumulate digits + // in tempValue, skip punctuation, stop on a > mark, and + // throw an exception on anything else + if (c == gSlash) { + val = 0; + ++p; + ll_10 = 10; + while (p < descriptorLength) { + c = descriptor.charAt(p); + if (c >= gZero && c <= gNine) { + val = val * ll_10 + (int32_t)(c - gZero); + } + else if (c == gGreaterThan) { + break; + } + else if (PatternProps::isWhiteSpace(c) || c == gComma || c == gDot) { + } + else { + // throw new IllegalArgumentException("Illegal character is rule descriptor"); + status = U_PARSE_ERROR; + return; + } + ++p; + } + + // tempValue now contain's the rule's radix. Set it + // accordingly, and recalculate the rule's exponent + radix = (int32_t)val; + if (radix == 0) { + // throw new IllegalArgumentException("Rule can't have radix of 0"); + status = U_PARSE_ERROR; + } + + exponent = expectedExponent(); + } + + // if we stopped the previous loop on a > sign, then continue + // for as long as we still see > signs. For each one, + // decrement the exponent (unless the exponent is already 0). + // If we see another character before reaching the end of + // the descriptor, that's also a syntax error. + if (c == gGreaterThan) { + while (p < descriptor.length()) { + c = descriptor.charAt(p); + if (c == gGreaterThan && exponent > 0) { + --exponent; + } else { + // throw new IllegalArgumentException("Illegal character in rule descriptor"); + status = U_PARSE_ERROR; + return; + } + ++p; + } + } + } + else if (0 == descriptor.compare(gMinusX, 2)) { + setType(kNegativeNumberRule); + } + else if (descriptorLength == 3) { + if (firstChar == gZero && lastChar == gX) { + setBaseValue(kProperFractionRule, status); + decimalPoint = descriptor.charAt(1); + } + else if (firstChar == gX && lastChar == gX) { + setBaseValue(kImproperFractionRule, status); + decimalPoint = descriptor.charAt(1); + } + else if (firstChar == gX && lastChar == gZero) { + setBaseValue(kDefaultRule, status); + decimalPoint = descriptor.charAt(1); + } + else if (descriptor.compare(gNaN, 3) == 0) { + setBaseValue(kNaNRule, status); + } + else if (descriptor.compare(gInf, 3) == 0) { + setBaseValue(kInfinityRule, status); + } + } + } + // else use the default base value for now. + + // finally, if the rule body begins with an apostrophe, strip it off + // (this is generally used to put whitespace at the beginning of + // a rule's rule text) + if (description.length() > 0 && description.charAt(0) == gTick) { + description.removeBetween(0, 1); + } + + // return the description with all the stuff we've just waded through + // stripped off the front. It now contains just the rule body. + // return description; +} + +/** +* Searches the rule's rule text for the substitution tokens, +* creates the substitutions, and removes the substitution tokens +* from the rule's rule text. +* @param owner The rule set containing this rule +* @param predecessor The rule preseding this one in "owners" rule list +* @param ownersOwner The RuleBasedFormat that owns this rule +*/ +void +NFRule::extractSubstitutions(const NFRuleSet* ruleSet, + const UnicodeString &ruleText, + const NFRule* predecessor, + UErrorCode& status) +{ + if (U_FAILURE(status)) { + return; + } + fRuleText = ruleText; + sub1 = extractSubstitution(ruleSet, predecessor, status); + if (sub1 == nullptr) { + // Small optimization. There is no need to create a redundant NullSubstitution. + sub2 = nullptr; + } + else { + sub2 = extractSubstitution(ruleSet, predecessor, status); + } + int32_t pluralRuleStart = fRuleText.indexOf(gDollarOpenParenthesis, -1, 0); + int32_t pluralRuleEnd = (pluralRuleStart >= 0 ? fRuleText.indexOf(gClosedParenthesisDollar, -1, pluralRuleStart) : -1); + if (pluralRuleEnd >= 0) { + int32_t endType = fRuleText.indexOf(gComma, pluralRuleStart); + if (endType < 0) { + status = U_PARSE_ERROR; + return; + } + UnicodeString type(fRuleText.tempSubString(pluralRuleStart + 2, endType - pluralRuleStart - 2)); + UPluralType pluralType; + if (type.startsWith(UNICODE_STRING_SIMPLE("cardinal"))) { + pluralType = UPLURAL_TYPE_CARDINAL; + } + else if (type.startsWith(UNICODE_STRING_SIMPLE("ordinal"))) { + pluralType = UPLURAL_TYPE_ORDINAL; + } + else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + rulePatternFormat = formatter->createPluralFormat(pluralType, + fRuleText.tempSubString(endType + 1, pluralRuleEnd - endType - 1), status); + } +} + +/** +* Searches the rule's rule text for the first substitution token, +* creates a substitution based on it, and removes the token from +* the rule's rule text. +* @param owner The rule set containing this rule +* @param predecessor The rule preceding this one in the rule set's +* rule list +* @param ownersOwner The RuleBasedNumberFormat that owns this rule +* @return The newly-created substitution. This is never null; if +* the rule text doesn't contain any substitution tokens, this will +* be a NullSubstitution. +*/ +NFSubstitution * +NFRule::extractSubstitution(const NFRuleSet* ruleSet, + const NFRule* predecessor, + UErrorCode& status) +{ + NFSubstitution* result = nullptr; + + // search the rule's rule text for the first two characters of + // a substitution token + int32_t subStart = indexOfAnyRulePrefix(); + int32_t subEnd = subStart; + + // if we didn't find one, create a null substitution positioned + // at the end of the rule text + if (subStart == -1) { + return nullptr; + } + + // special-case the ">>>" token, since searching for the > at the + // end will actually find the > in the middle + if (fRuleText.indexOf(gGreaterGreaterGreater, 3, 0) == subStart) { + subEnd = subStart + 2; + + // otherwise the substitution token ends with the same character + // it began with + } else { + char16_t c = fRuleText.charAt(subStart); + subEnd = fRuleText.indexOf(c, subStart + 1); + // special case for '<%foo<<' + if (c == gLessThan && subEnd != -1 && subEnd < fRuleText.length() - 1 && fRuleText.charAt(subEnd+1) == c) { + // ordinals use "=#,##0==%abbrev=" as their rule. Notice that the '==' in the middle + // occurs because of the juxtaposition of two different rules. The check for '<' is a hack + // to get around this. Having the duplicate at the front would cause problems with + // rules like "<<%" to format, say, percents... + ++subEnd; + } + } + + // if we don't find the end of the token (i.e., if we're on a single, + // unmatched token character), create a null substitution positioned + // at the end of the rule + if (subEnd == -1) { + return nullptr; + } + + // if we get here, we have a real substitution token (or at least + // some text bounded by substitution token characters). Use + // makeSubstitution() to create the right kind of substitution + UnicodeString subToken; + subToken.setTo(fRuleText, subStart, subEnd + 1 - subStart); + result = NFSubstitution::makeSubstitution(subStart, this, predecessor, ruleSet, + this->formatter, subToken, status); + + // remove the substitution from the rule text + fRuleText.removeBetween(subStart, subEnd+1); + + return result; +} + +/** + * Sets the rule's base value, and causes the radix and exponent + * to be recalculated. This is used during construction when we + * don't know the rule's base value until after it's been + * constructed. It should be used at any other time. + * @param The new base value for the rule. + */ +void +NFRule::setBaseValue(int64_t newBaseValue, UErrorCode& status) +{ + // set the base value + baseValue = newBaseValue; + radix = 10; + + // if this isn't a special rule, recalculate the radix and exponent + // (the radix always defaults to 10; if it's supposed to be something + // else, it's cleaned up by the caller and the exponent is + // recalculated again-- the only function that does this is + // NFRule.parseRuleDescriptor() ) + if (baseValue >= 1) { + exponent = expectedExponent(); + + // this function gets called on a fully-constructed rule whose + // description didn't specify a base value. This means it + // has substitutions, and some substitutions hold on to copies + // of the rule's divisor. Fix their copies of the divisor. + if (sub1 != nullptr) { + sub1->setDivisor(radix, exponent, status); + } + if (sub2 != nullptr) { + sub2->setDivisor(radix, exponent, status); + } + + // if this is a special rule, its radix and exponent are basically + // ignored. Set them to "safe" default values + } else { + exponent = 0; + } +} + +/** +* This calculates the rule's exponent based on its radix and base +* value. This will be the highest power the radix can be raised to +* and still produce a result less than or equal to the base value. +*/ +int16_t +NFRule::expectedExponent() const +{ + // since the log of 0, or the log base 0 of something, causes an + // error, declare the exponent in these cases to be 0 (we also + // deal with the special-rule identifiers here) + if (radix == 0 || baseValue < 1) { + return 0; + } + + // we get rounding error in some cases-- for example, log 1000 / log 10 + // gives us 1.9999999996 instead of 2. The extra logic here is to take + // that into account + int16_t tempResult = (int16_t)(uprv_log((double)baseValue) / uprv_log((double)radix)); + int64_t temp = util64_pow(radix, tempResult + 1); + if (temp <= baseValue) { + tempResult += 1; + } + return tempResult; +} + +/** + * Searches the rule's rule text for any of the specified strings. + * @return The index of the first match in the rule's rule text + * (i.e., the first substring in the rule's rule text that matches + * _any_ of the strings in "strings"). If none of the strings in + * "strings" is found in the rule's rule text, returns -1. + */ +int32_t +NFRule::indexOfAnyRulePrefix() const +{ + int result = -1; + for (int i = 0; RULE_PREFIXES[i]; i++) { + int32_t pos = fRuleText.indexOf(*RULE_PREFIXES[i]); + if (pos != -1 && (result == -1 || pos < result)) { + result = pos; + } + } + return result; +} + +//----------------------------------------------------------------------- +// boilerplate +//----------------------------------------------------------------------- + +static UBool +util_equalSubstitutions(const NFSubstitution* sub1, const NFSubstitution* sub2) +{ + if (sub1) { + if (sub2) { + return *sub1 == *sub2; + } + } else if (!sub2) { + return true; + } + return false; +} + +/** +* Tests two rules for equality. +* @param that The rule to compare this one against +* @return True is the two rules are functionally equivalent +*/ +bool +NFRule::operator==(const NFRule& rhs) const +{ + return baseValue == rhs.baseValue + && radix == rhs.radix + && exponent == rhs.exponent + && fRuleText == rhs.fRuleText + && util_equalSubstitutions(sub1, rhs.sub1) + && util_equalSubstitutions(sub2, rhs.sub2); +} + +/** +* Returns a textual representation of the rule. This won't +* necessarily be the same as the description that this rule +* was created with, but it will produce the same result. +* @return A textual description of the rule +*/ +static void util_append64(UnicodeString& result, int64_t n) +{ + char16_t buffer[256]; + int32_t len = util64_tou(n, buffer, sizeof(buffer)); + UnicodeString temp(buffer, len); + result.append(temp); +} + +void +NFRule::_appendRuleText(UnicodeString& result) const +{ + switch (getType()) { + case kNegativeNumberRule: result.append(gMinusX, 2); break; + case kImproperFractionRule: result.append(gX).append(decimalPoint == 0 ? gDot : decimalPoint).append(gX); break; + case kProperFractionRule: result.append(gZero).append(decimalPoint == 0 ? gDot : decimalPoint).append(gX); break; + case kDefaultRule: result.append(gX).append(decimalPoint == 0 ? gDot : decimalPoint).append(gZero); break; + case kInfinityRule: result.append(gInf, 3); break; + case kNaNRule: result.append(gNaN, 3); break; + default: + // for a normal rule, write out its base value, and if the radix is + // something other than 10, write out the radix (with the preceding + // slash, of course). Then calculate the expected exponent and if + // if isn't the same as the actual exponent, write an appropriate + // number of > signs. Finally, terminate the whole thing with + // a colon. + util_append64(result, baseValue); + if (radix != 10) { + result.append(gSlash); + util_append64(result, radix); + } + int numCarets = expectedExponent() - exponent; + for (int i = 0; i < numCarets; i++) { + result.append(gGreaterThan); + } + break; + } + result.append(gColon); + result.append(gSpace); + + // if the rule text begins with a space, write an apostrophe + // (whitespace after the rule descriptor is ignored; the + // apostrophe is used to make the whitespace significant) + if (fRuleText.charAt(0) == gSpace && (sub1 == nullptr || sub1->getPos() != 0)) { + result.append(gTick); + } + + // now, write the rule's rule text, inserting appropriate + // substitution tokens in the appropriate places + UnicodeString ruleTextCopy; + ruleTextCopy.setTo(fRuleText); + + UnicodeString temp; + if (sub2 != nullptr) { + sub2->toString(temp); + ruleTextCopy.insert(sub2->getPos(), temp); + } + if (sub1 != nullptr) { + sub1->toString(temp); + ruleTextCopy.insert(sub1->getPos(), temp); + } + + result.append(ruleTextCopy); + + // and finally, top the whole thing off with a semicolon and + // return the result + result.append(gSemicolon); +} + +int64_t NFRule::getDivisor() const +{ + return util64_pow(radix, exponent); +} + + +//----------------------------------------------------------------------- +// formatting +//----------------------------------------------------------------------- + +/** +* Formats the number, and inserts the resulting text into +* toInsertInto. +* @param number The number being formatted +* @param toInsertInto The string where the resultant text should +* be inserted +* @param pos The position in toInsertInto where the resultant text +* should be inserted +*/ +void +NFRule::doFormat(int64_t number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const +{ + // first, insert the rule's rule text into toInsertInto at the + // specified position, then insert the results of the substitutions + // into the right places in toInsertInto (notice we do the + // substitutions in reverse order so that the offsets don't get + // messed up) + int32_t pluralRuleStart = fRuleText.length(); + int32_t lengthOffset = 0; + if (!rulePatternFormat) { + toInsertInto.insert(pos, fRuleText); + } + else { + pluralRuleStart = fRuleText.indexOf(gDollarOpenParenthesis, -1, 0); + int pluralRuleEnd = fRuleText.indexOf(gClosedParenthesisDollar, -1, pluralRuleStart); + int initialLength = toInsertInto.length(); + if (pluralRuleEnd < fRuleText.length() - 1) { + toInsertInto.insert(pos, fRuleText.tempSubString(pluralRuleEnd + 2)); + } + toInsertInto.insert(pos, + rulePatternFormat->format((int32_t)(number/util64_pow(radix, exponent)), status)); + if (pluralRuleStart > 0) { + toInsertInto.insert(pos, fRuleText.tempSubString(0, pluralRuleStart)); + } + lengthOffset = fRuleText.length() - (toInsertInto.length() - initialLength); + } + + if (sub2 != nullptr) { + sub2->doSubstitution(number, toInsertInto, pos - (sub2->getPos() > pluralRuleStart ? lengthOffset : 0), recursionCount, status); + } + if (sub1 != nullptr) { + sub1->doSubstitution(number, toInsertInto, pos - (sub1->getPos() > pluralRuleStart ? lengthOffset : 0), recursionCount, status); + } +} + +/** +* Formats the number, and inserts the resulting text into +* toInsertInto. +* @param number The number being formatted +* @param toInsertInto The string where the resultant text should +* be inserted +* @param pos The position in toInsertInto where the resultant text +* should be inserted +*/ +void +NFRule::doFormat(double number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const +{ + // first, insert the rule's rule text into toInsertInto at the + // specified position, then insert the results of the substitutions + // into the right places in toInsertInto + // [again, we have two copies of this routine that do the same thing + // so that we don't sacrifice precision in a long by casting it + // to a double] + int32_t pluralRuleStart = fRuleText.length(); + int32_t lengthOffset = 0; + if (!rulePatternFormat) { + toInsertInto.insert(pos, fRuleText); + } + else { + pluralRuleStart = fRuleText.indexOf(gDollarOpenParenthesis, -1, 0); + int pluralRuleEnd = fRuleText.indexOf(gClosedParenthesisDollar, -1, pluralRuleStart); + int initialLength = toInsertInto.length(); + if (pluralRuleEnd < fRuleText.length() - 1) { + toInsertInto.insert(pos, fRuleText.tempSubString(pluralRuleEnd + 2)); + } + double pluralVal = number; + if (0 <= pluralVal && pluralVal < 1) { + // We're in a fractional rule, and we have to match the NumeratorSubstitution behavior. + // 2.3 can become 0.2999999999999998 for the fraction due to rounding errors. + pluralVal = uprv_round(pluralVal * util64_pow(radix, exponent)); + } + else { + pluralVal = pluralVal / util64_pow(radix, exponent); + } + toInsertInto.insert(pos, rulePatternFormat->format((int32_t)(pluralVal), status)); + if (pluralRuleStart > 0) { + toInsertInto.insert(pos, fRuleText.tempSubString(0, pluralRuleStart)); + } + lengthOffset = fRuleText.length() - (toInsertInto.length() - initialLength); + } + + if (sub2 != nullptr) { + sub2->doSubstitution(number, toInsertInto, pos - (sub2->getPos() > pluralRuleStart ? lengthOffset : 0), recursionCount, status); + } + if (sub1 != nullptr) { + sub1->doSubstitution(number, toInsertInto, pos - (sub1->getPos() > pluralRuleStart ? lengthOffset : 0), recursionCount, status); + } +} + +/** +* Used by the owning rule set to determine whether to invoke the +* rollback rule (i.e., whether this rule or the one that precedes +* it in the rule set's list should be used to format the number) +* @param The number being formatted +* @return True if the rule set should use the rule that precedes +* this one in its list; false if it should use this rule +*/ +UBool +NFRule::shouldRollBack(int64_t number) const +{ + // we roll back if the rule contains a modulus substitution, + // the number being formatted is an even multiple of the rule's + // divisor, and the rule's base value is NOT an even multiple + // of its divisor + // In other words, if the original description had + // 100: << hundred[ >>]; + // that expands into + // 100: << hundred; + // 101: << hundred >>; + // internally. But when we're formatting 200, if we use the rule + // at 101, which would normally apply, we get "two hundred zero". + // To prevent this, we roll back and use the rule at 100 instead. + // This is the logic that makes this happen: the rule at 101 has + // a modulus substitution, its base value isn't an even multiple + // of 100, and the value we're trying to format _is_ an even + // multiple of 100. This is called the "rollback rule." + if ((sub1 != nullptr && sub1->isModulusSubstitution()) || (sub2 != nullptr && sub2->isModulusSubstitution())) { + int64_t re = util64_pow(radix, exponent); + return (number % re) == 0 && (baseValue % re) != 0; + } + return false; +} + +//----------------------------------------------------------------------- +// parsing +//----------------------------------------------------------------------- + +/** +* Attempts to parse the string with this rule. +* @param text The string being parsed +* @param parsePosition On entry, the value is ignored and assumed to +* be 0. On exit, this has been updated with the position of the first +* character not consumed by matching the text against this rule +* (if this rule doesn't match the text at all, the parse position +* if left unchanged (presumably at 0) and the function returns +* new Long(0)). +* @param isFractionRule True if this rule is contained within a +* fraction rule set. This is only used if the rule has no +* substitutions. +* @return If this rule matched the text, this is the rule's base value +* combined appropriately with the results of parsing the substitutions. +* If nothing matched, this is new Long(0) and the parse position is +* left unchanged. The result will be an instance of Long if the +* result is an integer and Double otherwise. The result is never null. +*/ +#ifdef RBNF_DEBUG +#include + +static void dumpUS(FILE* f, const UnicodeString& us) { + int len = us.length(); + char* buf = (char *)uprv_malloc((len+1)*sizeof(char)); //new char[len+1]; + if (buf != nullptr) { + us.extract(0, len, buf); + buf[len] = 0; + fprintf(f, "%s", buf); + uprv_free(buf); //delete[] buf; + } +} +#endif +UBool +NFRule::doParse(const UnicodeString& text, + ParsePosition& parsePosition, + UBool isFractionRule, + double upperBound, + uint32_t nonNumericalExecutedRuleMask, + Formattable& resVal) const +{ + // internally we operate on a copy of the string being parsed + // (because we're going to change it) and use our own ParsePosition + ParsePosition pp; + UnicodeString workText(text); + + int32_t sub1Pos = sub1 != nullptr ? sub1->getPos() : fRuleText.length(); + int32_t sub2Pos = sub2 != nullptr ? sub2->getPos() : fRuleText.length(); + + // check to see whether the text before the first substitution + // matches the text at the beginning of the string being + // parsed. If it does, strip that off the front of workText; + // otherwise, dump out with a mismatch + UnicodeString prefix; + prefix.setTo(fRuleText, 0, sub1Pos); + +#ifdef RBNF_DEBUG + fprintf(stderr, "doParse %p ", this); + { + UnicodeString rt; + _appendRuleText(rt); + dumpUS(stderr, rt); + } + + fprintf(stderr, " text: '"); + dumpUS(stderr, text); + fprintf(stderr, "' prefix: '"); + dumpUS(stderr, prefix); +#endif + stripPrefix(workText, prefix, pp); + int32_t prefixLength = text.length() - workText.length(); + +#ifdef RBNF_DEBUG + fprintf(stderr, "' pl: %d ppi: %d s1p: %d\n", prefixLength, pp.getIndex(), sub1Pos); +#endif + + if (pp.getIndex() == 0 && sub1Pos != 0) { + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + parsePosition.setErrorIndex(pp.getErrorIndex()); + resVal.setLong(0); + return true; + } + if (baseValue == kInfinityRule) { + // If you match this, don't try to perform any calculations on it. + parsePosition.setIndex(pp.getIndex()); + resVal.setDouble(uprv_getInfinity()); + return true; + } + if (baseValue == kNaNRule) { + // If you match this, don't try to perform any calculations on it. + parsePosition.setIndex(pp.getIndex()); + resVal.setDouble(uprv_getNaN()); + return true; + } + + // this is the fun part. The basic guts of the rule-matching + // logic is matchToDelimiter(), which is called twice. The first + // time it searches the input string for the rule text BETWEEN + // the substitutions and tries to match the intervening text + // in the input string with the first substitution. If that + // succeeds, it then calls it again, this time to look for the + // rule text after the second substitution and to match the + // intervening input text against the second substitution. + // + // For example, say we have a rule that looks like this: + // first << middle >> last; + // and input text that looks like this: + // first one middle two last + // First we use stripPrefix() to match "first " in both places and + // strip it off the front, leaving + // one middle two last + // Then we use matchToDelimiter() to match " middle " and try to + // match "one" against a substitution. If it's successful, we now + // have + // two last + // We use matchToDelimiter() a second time to match " last" and + // try to match "two" against a substitution. If "two" matches + // the substitution, we have a successful parse. + // + // Since it's possible in many cases to find multiple instances + // of each of these pieces of rule text in the input string, + // we need to try all the possible combinations of these + // locations. This prevents us from prematurely declaring a mismatch, + // and makes sure we match as much input text as we can. + int highWaterMark = 0; + double result = 0; + int start = 0; + double tempBaseValue = (double)(baseValue <= 0 ? 0 : baseValue); + + UnicodeString temp; + do { + // our partial parse result starts out as this rule's base + // value. If it finds a successful match, matchToDelimiter() + // will compose this in some way with what it gets back from + // the substitution, giving us a new partial parse result + pp.setIndex(0); + + temp.setTo(fRuleText, sub1Pos, sub2Pos - sub1Pos); + double partialResult = matchToDelimiter(workText, start, tempBaseValue, + temp, pp, sub1, + nonNumericalExecutedRuleMask, + upperBound); + + // if we got a successful match (or were trying to match a + // null substitution), pp is now pointing at the first unmatched + // character. Take note of that, and try matchToDelimiter() + // on the input text again + if (pp.getIndex() != 0 || sub1 == nullptr) { + start = pp.getIndex(); + + UnicodeString workText2; + workText2.setTo(workText, pp.getIndex(), workText.length() - pp.getIndex()); + ParsePosition pp2; + + // the second matchToDelimiter() will compose our previous + // partial result with whatever it gets back from its + // substitution if there's a successful match, giving us + // a real result + temp.setTo(fRuleText, sub2Pos, fRuleText.length() - sub2Pos); + partialResult = matchToDelimiter(workText2, 0, partialResult, + temp, pp2, sub2, + nonNumericalExecutedRuleMask, + upperBound); + + // if we got a successful match on this second + // matchToDelimiter() call, update the high-water mark + // and result (if necessary) + if (pp2.getIndex() != 0 || sub2 == nullptr) { + if (prefixLength + pp.getIndex() + pp2.getIndex() > highWaterMark) { + highWaterMark = prefixLength + pp.getIndex() + pp2.getIndex(); + result = partialResult; + } + } + else { + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + int32_t i_temp = pp2.getErrorIndex() + sub1Pos + pp.getIndex(); + if (i_temp> parsePosition.getErrorIndex()) { + parsePosition.setErrorIndex(i_temp); + } + } + } + else { + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + int32_t i_temp = sub1Pos + pp.getErrorIndex(); + if (i_temp > parsePosition.getErrorIndex()) { + parsePosition.setErrorIndex(i_temp); + } + } + // keep trying to match things until the outer matchToDelimiter() + // call fails to make a match (each time, it picks up where it + // left off the previous time) + } while (sub1Pos != sub2Pos + && pp.getIndex() > 0 + && pp.getIndex() < workText.length() + && pp.getIndex() != start); + + // update the caller's ParsePosition with our high-water mark + // (i.e., it now points at the first character this function + // didn't match-- the ParsePosition is therefore unchanged if + // we didn't match anything) + parsePosition.setIndex(highWaterMark); + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + if (highWaterMark > 0) { + parsePosition.setErrorIndex(0); + } + + // this is a hack for one unusual condition: Normally, whether this + // rule belong to a fraction rule set or not is handled by its + // substitutions. But if that rule HAS NO substitutions, then + // we have to account for it here. By definition, if the matching + // rule in a fraction rule set has no substitutions, its numerator + // is 1, and so the result is the reciprocal of its base value. + if (isFractionRule && highWaterMark > 0 && sub1 == nullptr) { + result = 1 / result; + } + + resVal.setDouble(result); + return true; // ??? do we need to worry if it is a long or a double? +} + +/** +* This function is used by parse() to match the text being parsed +* against a possible prefix string. This function +* matches characters from the beginning of the string being parsed +* to characters from the prospective prefix. If they match, pp is +* updated to the first character not matched, and the result is +* the unparsed part of the string. If they don't match, the whole +* string is returned, and pp is left unchanged. +* @param text The string being parsed +* @param prefix The text to match against +* @param pp On entry, ignored and assumed to be 0. On exit, points +* to the first unmatched character (assuming the whole prefix matched), +* or is unchanged (if the whole prefix didn't match). +* @return If things match, this is the unparsed part of "text"; +* if they didn't match, this is "text". +*/ +void +NFRule::stripPrefix(UnicodeString& text, const UnicodeString& prefix, ParsePosition& pp) const +{ + // if the prefix text is empty, dump out without doing anything + if (prefix.length() != 0) { + UErrorCode status = U_ZERO_ERROR; + // use prefixLength() to match the beginning of + // "text" against "prefix". This function returns the + // number of characters from "text" that matched (or 0 if + // we didn't match the whole prefix) + int32_t pfl = prefixLength(text, prefix, status); + if (U_FAILURE(status)) { // Memory allocation error. + return; + } + if (pfl != 0) { + // if we got a successful match, update the parse position + // and strip the prefix off of "text" + pp.setIndex(pp.getIndex() + pfl); + text.remove(0, pfl); + } + } +} + +/** +* Used by parse() to match a substitution and any following text. +* "text" is searched for instances of "delimiter". For each instance +* of delimiter, the intervening text is tested to see whether it +* matches the substitution. The longest match wins. +* @param text The string being parsed +* @param startPos The position in "text" where we should start looking +* for "delimiter". +* @param baseValue A partial parse result (often the rule's base value), +* which is combined with the result from matching the substitution +* @param delimiter The string to search "text" for. +* @param pp Ignored and presumed to be 0 on entry. If there's a match, +* on exit this will point to the first unmatched character. +* @param sub If we find "delimiter" in "text", this substitution is used +* to match the text between the beginning of the string and the +* position of "delimiter." (If "delimiter" is the empty string, then +* this function just matches against this substitution and updates +* everything accordingly.) +* @param upperBound When matching the substitution, it will only +* consider rules with base values lower than this value. +* @return If there's a match, this is the result of composing +* baseValue with the result of matching the substitution. Otherwise, +* this is new Long(0). It's never null. If the result is an integer, +* this will be an instance of Long; otherwise, it's an instance of +* Double. +* +* !!! note {dlf} in point of fact, in the java code the caller always converts +* the result to a double, so we might as well return one. +*/ +double +NFRule::matchToDelimiter(const UnicodeString& text, + int32_t startPos, + double _baseValue, + const UnicodeString& delimiter, + ParsePosition& pp, + const NFSubstitution* sub, + uint32_t nonNumericalExecutedRuleMask, + double upperBound) const +{ + UErrorCode status = U_ZERO_ERROR; + // if "delimiter" contains real (i.e., non-ignorable) text, search + // it for "delimiter" beginning at "start". If that succeeds, then + // use "sub"'s doParse() method to match the text before the + // instance of "delimiter" we just found. + if (!allIgnorable(delimiter, status)) { + if (U_FAILURE(status)) { //Memory allocation error. + return 0; + } + ParsePosition tempPP; + Formattable result; + + // use findText() to search for "delimiter". It returns a two- + // element array: element 0 is the position of the match, and + // element 1 is the number of characters that matched + // "delimiter". + int32_t dLen; + int32_t dPos = findText(text, delimiter, startPos, &dLen); + + // if findText() succeeded, isolate the text preceding the + // match, and use "sub" to match that text + while (dPos >= 0) { + UnicodeString subText; + subText.setTo(text, 0, dPos); + if (subText.length() > 0) { + UBool success = sub->doParse(subText, tempPP, _baseValue, upperBound, +#if UCONFIG_NO_COLLATION + false, +#else + formatter->isLenient(), +#endif + nonNumericalExecutedRuleMask, + result); + + // if the substitution could match all the text up to + // where we found "delimiter", then this function has + // a successful match. Bump the caller's parse position + // to point to the first character after the text + // that matches "delimiter", and return the result + // we got from parsing the substitution. + if (success && tempPP.getIndex() == dPos) { + pp.setIndex(dPos + dLen); + return result.getDouble(); + } + else { + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + if (tempPP.getErrorIndex() > 0) { + pp.setErrorIndex(tempPP.getErrorIndex()); + } else { + pp.setErrorIndex(tempPP.getIndex()); + } + } + } + + // if we didn't match the substitution, search for another + // copy of "delimiter" in "text" and repeat the loop if + // we find it + tempPP.setIndex(0); + dPos = findText(text, delimiter, dPos + dLen, &dLen); + } + // if we make it here, this was an unsuccessful match, and we + // leave pp unchanged and return 0 + pp.setIndex(0); + return 0; + + // if "delimiter" is empty, or consists only of ignorable characters + // (i.e., is semantically empty), thwe we obviously can't search + // for "delimiter". Instead, just use "sub" to parse as much of + // "text" as possible. + } + else if (sub == nullptr) { + return _baseValue; + } + else { + ParsePosition tempPP; + Formattable result; + + // try to match the whole string against the substitution + UBool success = sub->doParse(text, tempPP, _baseValue, upperBound, +#if UCONFIG_NO_COLLATION + false, +#else + formatter->isLenient(), +#endif + nonNumericalExecutedRuleMask, + result); + if (success && (tempPP.getIndex() != 0)) { + // if there's a successful match (or it's a null + // substitution), update pp to point to the first + // character we didn't match, and pass the result from + // sub.doParse() on through to the caller + pp.setIndex(tempPP.getIndex()); + return result.getDouble(); + } + else { + // commented out because ParsePosition doesn't have error index in 1.1.x + // restored for ICU4C port + pp.setErrorIndex(tempPP.getErrorIndex()); + } + + // and if we get to here, then nothing matched, so we return + // 0 and leave pp alone + return 0; + } +} + +/** +* Used by stripPrefix() to match characters. If lenient parse mode +* is off, this just calls startsWith(). If lenient parse mode is on, +* this function uses CollationElementIterators to match characters in +* the strings (only primary-order differences are significant in +* determining whether there's a match). +* @param str The string being tested +* @param prefix The text we're hoping to see at the beginning +* of "str" +* @return If "prefix" is found at the beginning of "str", this +* is the number of characters in "str" that were matched (this +* isn't necessarily the same as the length of "prefix" when matching +* text with a collator). If there's no match, this is 0. +*/ +int32_t +NFRule::prefixLength(const UnicodeString& str, const UnicodeString& prefix, UErrorCode& status) const +{ + // if we're looking for an empty prefix, it obviously matches + // zero characters. Just go ahead and return 0. + if (prefix.length() == 0) { + return 0; + } + +#if !UCONFIG_NO_COLLATION + // go through all this grief if we're in lenient-parse mode + if (formatter->isLenient()) { + // Check if non-lenient rule finds the text before call lenient parsing + if (str.startsWith(prefix)) { + return prefix.length(); + } + // get the formatter's collator and use it to create two + // collation element iterators, one over the target string + // and another over the prefix (right now, we'll throw an + // exception if the collator we get back from the formatter + // isn't a RuleBasedCollator, because RuleBasedCollator defines + // the CollationElementIterator protocol. Hopefully, this + // will change someday.) + const RuleBasedCollator* collator = formatter->getCollator(); + if (collator == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + LocalPointer strIter(collator->createCollationElementIterator(str)); + LocalPointer prefixIter(collator->createCollationElementIterator(prefix)); + // Check for memory allocation error. + if (strIter.isNull() || prefixIter.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + + UErrorCode err = U_ZERO_ERROR; + + // The original code was problematic. Consider this match: + // prefix = "fifty-" + // string = " fifty-7" + // The intent is to match string up to the '7', by matching 'fifty-' at position 1 + // in the string. Unfortunately, we were getting a match, and then computing where + // the match terminated by rematching the string. The rematch code was using as an + // initial guess the substring of string between 0 and prefix.length. Because of + // the leading space and trailing hyphen (both ignorable) this was succeeding, leaving + // the position before the hyphen in the string. Recursing down, we then parsed the + // remaining string '-7' as numeric. The resulting number turned out as 43 (50 - 7). + // This was not pretty, especially since the string "fifty-7" parsed just fine. + // + // We have newer APIs now, so we can use calls on the iterator to determine what we + // matched up to. If we terminate because we hit the last element in the string, + // our match terminates at this length. If we terminate because we hit the last element + // in the target, our match terminates at one before the element iterator position. + + // match collation elements between the strings + int32_t oStr = strIter->next(err); + int32_t oPrefix = prefixIter->next(err); + + while (oPrefix != CollationElementIterator::NULLORDER) { + // skip over ignorable characters in the target string + while (CollationElementIterator::primaryOrder(oStr) == 0 + && oStr != CollationElementIterator::NULLORDER) { + oStr = strIter->next(err); + } + + // skip over ignorable characters in the prefix + while (CollationElementIterator::primaryOrder(oPrefix) == 0 + && oPrefix != CollationElementIterator::NULLORDER) { + oPrefix = prefixIter->next(err); + } + + // dlf: move this above following test, if we consume the + // entire target, aren't we ok even if the source was also + // entirely consumed? + + // if skipping over ignorables brought to the end of + // the prefix, we DID match: drop out of the loop + if (oPrefix == CollationElementIterator::NULLORDER) { + break; + } + + // if skipping over ignorables brought us to the end + // of the target string, we didn't match and return 0 + if (oStr == CollationElementIterator::NULLORDER) { + return 0; + } + + // match collation elements from the two strings + // (considering only primary differences). If we + // get a mismatch, dump out and return 0 + if (CollationElementIterator::primaryOrder(oStr) + != CollationElementIterator::primaryOrder(oPrefix)) { + return 0; + + // otherwise, advance to the next character in each string + // and loop (we drop out of the loop when we exhaust + // collation elements in the prefix) + } else { + oStr = strIter->next(err); + oPrefix = prefixIter->next(err); + } + } + + int32_t result = strIter->getOffset(); + if (oStr != CollationElementIterator::NULLORDER) { + --result; // back over character that we don't want to consume; + } + +#ifdef RBNF_DEBUG + fprintf(stderr, "prefix length: %d\n", result); +#endif + return result; +#if 0 + //---------------------------------------------------------------- + // JDK 1.2-specific API call + // return strIter.getOffset(); + //---------------------------------------------------------------- + // JDK 1.1 HACK (take out for 1.2-specific code) + + // if we make it to here, we have a successful match. Now we + // have to find out HOW MANY characters from the target string + // matched the prefix (there isn't necessarily a one-to-one + // mapping between collation elements and characters). + // In JDK 1.2, there's a simple getOffset() call we can use. + // In JDK 1.1, on the other hand, we have to go through some + // ugly contortions. First, use the collator to compare the + // same number of characters from the prefix and target string. + // If they're equal, we're done. + collator->setStrength(Collator::PRIMARY); + if (str.length() >= prefix.length()) { + UnicodeString temp; + temp.setTo(str, 0, prefix.length()); + if (collator->equals(temp, prefix)) { +#ifdef RBNF_DEBUG + fprintf(stderr, "returning: %d\n", prefix.length()); +#endif + return prefix.length(); + } + } + + // if they're not equal, then we have to compare successively + // larger and larger substrings of the target string until we + // get to one that matches the prefix. At that point, we know + // how many characters matched the prefix, and we can return. + int32_t p = 1; + while (p <= str.length()) { + UnicodeString temp; + temp.setTo(str, 0, p); + if (collator->equals(temp, prefix)) { + return p; + } else { + ++p; + } + } + + // SHOULD NEVER GET HERE!!! + return 0; + //---------------------------------------------------------------- +#endif + + // If lenient parsing is turned off, forget all that crap above. + // Just use String.startsWith() and be done with it. + } else +#endif + { + if (str.startsWith(prefix)) { + return prefix.length(); + } else { + return 0; + } + } +} + +/** +* Searches a string for another string. If lenient parsing is off, +* this just calls indexOf(). If lenient parsing is on, this function +* uses CollationElementIterator to match characters, and only +* primary-order differences are significant in determining whether +* there's a match. +* @param str The string to search +* @param key The string to search "str" for +* @param startingAt The index into "str" where the search is to +* begin +* @return A two-element array of ints. Element 0 is the position +* of the match, or -1 if there was no match. Element 1 is the +* number of characters in "str" that matched (which isn't necessarily +* the same as the length of "key") +*/ +int32_t +NFRule::findText(const UnicodeString& str, + const UnicodeString& key, + int32_t startingAt, + int32_t* length) const +{ + if (rulePatternFormat) { + Formattable result; + FieldPosition position(UNUM_INTEGER_FIELD); + position.setBeginIndex(startingAt); + rulePatternFormat->parseType(str, this, result, position); + int start = position.getBeginIndex(); + if (start >= 0) { + int32_t pluralRuleStart = fRuleText.indexOf(gDollarOpenParenthesis, -1, 0); + int32_t pluralRuleSuffix = fRuleText.indexOf(gClosedParenthesisDollar, -1, pluralRuleStart) + 2; + int32_t matchLen = position.getEndIndex() - start; + UnicodeString prefix(fRuleText.tempSubString(0, pluralRuleStart)); + UnicodeString suffix(fRuleText.tempSubString(pluralRuleSuffix)); + if (str.compare(start - prefix.length(), prefix.length(), prefix, 0, prefix.length()) == 0 + && str.compare(start + matchLen, suffix.length(), suffix, 0, suffix.length()) == 0) + { + *length = matchLen + prefix.length() + suffix.length(); + return start - prefix.length(); + } + } + *length = 0; + return -1; + } + if (!formatter->isLenient()) { + // if lenient parsing is turned off, this is easy: just call + // String.indexOf() and we're done + *length = key.length(); + return str.indexOf(key, startingAt); + } + else { + // Check if non-lenient rule finds the text before call lenient parsing + *length = key.length(); + int32_t pos = str.indexOf(key, startingAt); + if(pos >= 0) { + return pos; + } else { + // but if lenient parsing is turned ON, we've got some work ahead of us + return findTextLenient(str, key, startingAt, length); + } + } +} + +int32_t +NFRule::findTextLenient(const UnicodeString& str, + const UnicodeString& key, + int32_t startingAt, + int32_t* length) const +{ + //---------------------------------------------------------------- + // JDK 1.1 HACK (take out of 1.2-specific code) + + // in JDK 1.2, CollationElementIterator provides us with an + // API to map between character offsets and collation elements + // and we can do this by marching through the string comparing + // collation elements. We can't do that in JDK 1.1. Instead, + // we have to go through this horrible slow mess: + int32_t p = startingAt; + int32_t keyLen = 0; + + // basically just isolate smaller and smaller substrings of + // the target string (each running to the end of the string, + // and with the first one running from startingAt to the end) + // and then use prefixLength() to see if the search key is at + // the beginning of each substring. This is excruciatingly + // slow, but it will locate the key and tell use how long the + // matching text was. + UnicodeString temp; + UErrorCode status = U_ZERO_ERROR; + while (p < str.length() && keyLen == 0) { + temp.setTo(str, p, str.length() - p); + keyLen = prefixLength(temp, key, status); + if (U_FAILURE(status)) { + break; + } + if (keyLen != 0) { + *length = keyLen; + return p; + } + ++p; + } + // if we make it to here, we didn't find it. Return -1 for the + // location. The length should be ignored, but set it to 0, + // which should be "safe" + *length = 0; + return -1; +} + +/** +* Checks to see whether a string consists entirely of ignorable +* characters. +* @param str The string to test. +* @return true if the string is empty of consists entirely of +* characters that the number formatter's collator says are +* ignorable at the primary-order level. false otherwise. +*/ +UBool +NFRule::allIgnorable(const UnicodeString& str, UErrorCode& status) const +{ + // if the string is empty, we can just return true + if (str.length() == 0) { + return true; + } + +#if !UCONFIG_NO_COLLATION + // if lenient parsing is turned on, walk through the string with + // a collation element iterator and make sure each collation + // element is 0 (ignorable) at the primary level + if (formatter->isLenient()) { + const RuleBasedCollator* collator = formatter->getCollator(); + if (collator == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + LocalPointer iter(collator->createCollationElementIterator(str)); + + // Memory allocation error check. + if (iter.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + + UErrorCode err = U_ZERO_ERROR; + int32_t o = iter->next(err); + while (o != CollationElementIterator::NULLORDER + && CollationElementIterator::primaryOrder(o) == 0) { + o = iter->next(err); + } + + return o == CollationElementIterator::NULLORDER; + } +#endif + + // if lenient parsing is turned off, there is no such thing as + // an ignorable character: return true only if the string is empty + return false; +} + +void +NFRule::setDecimalFormatSymbols(const DecimalFormatSymbols& newSymbols, UErrorCode& status) { + if (sub1 != nullptr) { + sub1->setDecimalFormatSymbols(newSymbols, status); + } + if (sub2 != nullptr) { + sub2->setDecimalFormatSymbols(newSymbols, status); + } +} + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif diff --git a/intl/icu/source/i18n/nfrule.h b/intl/icu/source/i18n/nfrule.h new file mode 100644 index 0000000000..fda74fabf2 --- /dev/null +++ b/intl/icu/source/i18n/nfrule.h @@ -0,0 +1,129 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +*/ + +#ifndef NFRULE_H +#define NFRULE_H + +#include "unicode/rbnf.h" + +#if U_HAVE_RBNF + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" + +U_NAMESPACE_BEGIN + +class FieldPosition; +class Formattable; +class NFRuleList; +class NFRuleSet; +class NFSubstitution; +class ParsePosition; +class PluralFormat; +class RuleBasedNumberFormat; +class UnicodeString; + +class NFRule : public UMemory { +public: + + enum ERuleType { + kNoBase = 0, + kNegativeNumberRule = -1, + kImproperFractionRule = -2, + kProperFractionRule = -3, + kDefaultRule = -4, + kInfinityRule = -5, + kNaNRule = -6, + kOtherRule = -7 + }; + + static void makeRules(UnicodeString& definition, + NFRuleSet* ruleSet, + const NFRule* predecessor, + const RuleBasedNumberFormat* rbnf, + NFRuleList& ruleList, + UErrorCode& status); + + NFRule(const RuleBasedNumberFormat* rbnf, const UnicodeString &ruleText, UErrorCode &status); + ~NFRule(); + + bool operator==(const NFRule& rhs) const; + bool operator!=(const NFRule& rhs) const { return !operator==(rhs); } + + ERuleType getType() const { return (ERuleType)(baseValue <= kNoBase ? (ERuleType)baseValue : kOtherRule); } + void setType(ERuleType ruleType) { baseValue = (int32_t)ruleType; } + + int64_t getBaseValue() const { return baseValue; } + void setBaseValue(int64_t value, UErrorCode& status); + + char16_t getDecimalPoint() const { return decimalPoint; } + + int64_t getDivisor() const; + + void doFormat(int64_t number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + void doFormat(double number, UnicodeString& toAppendTo, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + + UBool doParse(const UnicodeString& text, + ParsePosition& pos, + UBool isFractional, + double upperBound, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const; + + UBool shouldRollBack(int64_t number) const; + + void _appendRuleText(UnicodeString& result) const; + + int32_t findTextLenient(const UnicodeString& str, const UnicodeString& key, + int32_t startingAt, int32_t* resultCount) const; + + void setDecimalFormatSymbols(const DecimalFormatSymbols &newSymbols, UErrorCode& status); + +private: + void parseRuleDescriptor(UnicodeString& descriptor, UErrorCode& status); + void extractSubstitutions(const NFRuleSet* ruleSet, const UnicodeString &ruleText, const NFRule* predecessor, UErrorCode& status); + NFSubstitution* extractSubstitution(const NFRuleSet* ruleSet, const NFRule* predecessor, UErrorCode& status); + + int16_t expectedExponent() const; + int32_t indexOfAnyRulePrefix() const; + double matchToDelimiter(const UnicodeString& text, int32_t startPos, double baseValue, + const UnicodeString& delimiter, ParsePosition& pp, const NFSubstitution* sub, + uint32_t nonNumericalExecutedRuleMask, + double upperBound) const; + void stripPrefix(UnicodeString& text, const UnicodeString& prefix, ParsePosition& pp) const; + + int32_t prefixLength(const UnicodeString& str, const UnicodeString& prefix, UErrorCode& status) const; + UBool allIgnorable(const UnicodeString& str, UErrorCode& status) const; + int32_t findText(const UnicodeString& str, const UnicodeString& key, + int32_t startingAt, int32_t* resultCount) const; + +private: + int64_t baseValue; + int32_t radix; + int16_t exponent; + char16_t decimalPoint; + UnicodeString fRuleText; + NFSubstitution* sub1; + NFSubstitution* sub2; + const RuleBasedNumberFormat* formatter; + const PluralFormat* rulePatternFormat; + + NFRule(const NFRule &other); // forbid copying of this class + NFRule &operator=(const NFRule &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif + +// NFRULE_H +#endif + diff --git a/intl/icu/source/i18n/nfsubs.cpp b/intl/icu/source/i18n/nfsubs.cpp new file mode 100644 index 0000000000..4f3247ce50 --- /dev/null +++ b/intl/icu/source/i18n/nfsubs.cpp @@ -0,0 +1,1343 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfsubs.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#include +#include "utypeinfo.h" // for 'typeid' to work + +#include "nfsubs.h" +#include "fmtableimp.h" +#include "putilimp.h" +#include "number_decimalquantity.h" + +#if U_HAVE_RBNF + +static const char16_t gLessThan = 0x003c; +static const char16_t gEquals = 0x003d; +static const char16_t gGreaterThan = 0x003e; +static const char16_t gPercent = 0x0025; +static const char16_t gPound = 0x0023; +static const char16_t gZero = 0x0030; +static const char16_t gSpace = 0x0020; + +static const char16_t gEqualsEquals[] = +{ + 0x3D, 0x3D, 0 +}; /* "==" */ +static const char16_t gGreaterGreaterGreaterThan[] = +{ + 0x3E, 0x3E, 0x3E, 0 +}; /* ">>>" */ +static const char16_t gGreaterGreaterThan[] = +{ + 0x3E, 0x3E, 0 +}; /* ">>" */ + +U_NAMESPACE_BEGIN + +using number::impl::DecimalQuantity; + +class SameValueSubstitution : public NFSubstitution { +public: + SameValueSubstitution(int32_t pos, + const NFRuleSet* ruleset, + const UnicodeString& description, + UErrorCode& status); + virtual ~SameValueSubstitution(); + + virtual int64_t transformNumber(int64_t number) const override { return number; } + virtual double transformNumber(double number) const override { return number; } + virtual double composeRuleValue(double newRuleValue, double /*oldRuleValue*/) const override { return newRuleValue; } + virtual double calcUpperBound(double oldUpperBound) const override { return oldUpperBound; } + virtual char16_t tokenChar() const override { return (char16_t)0x003d; } // '=' + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +SameValueSubstitution::~SameValueSubstitution() {} + +class MultiplierSubstitution : public NFSubstitution { + int64_t divisor; + +public: + MultiplierSubstitution(int32_t _pos, + const NFRule *rule, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, description, status), divisor(rule->getDivisor()) + { + if (divisor == 0) { + status = U_PARSE_ERROR; + } + } + virtual ~MultiplierSubstitution(); + + virtual void setDivisor(int32_t radix, int16_t exponent, UErrorCode& status) override { + divisor = util64_pow(radix, exponent); + + if(divisor == 0) { + status = U_PARSE_ERROR; + } + } + + virtual bool operator==(const NFSubstitution& rhs) const override; + + virtual int64_t transformNumber(int64_t number) const override { + return number / divisor; + } + + virtual double transformNumber(double number) const override { + if (getRuleSet()) { + return uprv_floor(number / divisor); + } else { + return number / divisor; + } + } + + virtual double composeRuleValue(double newRuleValue, double /*oldRuleValue*/) const override { + return newRuleValue * divisor; + } + + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return static_cast(divisor); } + + virtual char16_t tokenChar() const override { return (char16_t)0x003c; } // '<' + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +MultiplierSubstitution::~MultiplierSubstitution() {} + +class ModulusSubstitution : public NFSubstitution { + int64_t divisor; + const NFRule* ruleToUse; +public: + ModulusSubstitution(int32_t pos, + const NFRule* rule, + const NFRule* rulePredecessor, + const NFRuleSet* ruleSet, + const UnicodeString& description, + UErrorCode& status); + virtual ~ModulusSubstitution(); + + virtual void setDivisor(int32_t radix, int16_t exponent, UErrorCode& status) override { + divisor = util64_pow(radix, exponent); + + if (divisor == 0) { + status = U_PARSE_ERROR; + } + } + + virtual bool operator==(const NFSubstitution& rhs) const override; + + virtual void doSubstitution(int64_t number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const override; + virtual void doSubstitution(double number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const override; + + virtual int64_t transformNumber(int64_t number) const override { return number % divisor; } + virtual double transformNumber(double number) const override { return uprv_fmod(number, static_cast(divisor)); } + + virtual UBool doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const override; + + virtual double composeRuleValue(double newRuleValue, double oldRuleValue) const override { + return oldRuleValue - uprv_fmod(oldRuleValue, static_cast(divisor)) + newRuleValue; + } + + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return static_cast(divisor); } + + virtual UBool isModulusSubstitution() const override { return true; } + + virtual char16_t tokenChar() const override { return (char16_t)0x003e; } // '>' + + virtual void toString(UnicodeString& result) const override; + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +ModulusSubstitution::~ModulusSubstitution() {} + +class IntegralPartSubstitution : public NFSubstitution { +public: + IntegralPartSubstitution(int32_t _pos, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, description, status) {} + virtual ~IntegralPartSubstitution(); + + virtual int64_t transformNumber(int64_t number) const override { return number; } + virtual double transformNumber(double number) const override { return uprv_floor(number); } + virtual double composeRuleValue(double newRuleValue, double oldRuleValue) const override { return newRuleValue + oldRuleValue; } + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return DBL_MAX; } + virtual char16_t tokenChar() const override { return (char16_t)0x003c; } // '<' + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +IntegralPartSubstitution::~IntegralPartSubstitution() {} + +class FractionalPartSubstitution : public NFSubstitution { + UBool byDigits; + UBool useSpaces; + enum { kMaxDecimalDigits = 8 }; +public: + FractionalPartSubstitution(int32_t pos, + const NFRuleSet* ruleSet, + const UnicodeString& description, + UErrorCode& status); + virtual ~FractionalPartSubstitution(); + + virtual bool operator==(const NFSubstitution& rhs) const override; + + virtual void doSubstitution(double number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const override; + virtual void doSubstitution(int64_t /*number*/, UnicodeString& /*toInsertInto*/, int32_t /*_pos*/, int32_t /*recursionCount*/, UErrorCode& /*status*/) const override {} + virtual int64_t transformNumber(int64_t /*number*/) const override { return 0; } + virtual double transformNumber(double number) const override { return number - uprv_floor(number); } + + virtual UBool doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const override; + + virtual double composeRuleValue(double newRuleValue, double oldRuleValue) const override { return newRuleValue + oldRuleValue; } + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return 0.0; } + virtual char16_t tokenChar() const override { return (char16_t)0x003e; } // '>' + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +FractionalPartSubstitution::~FractionalPartSubstitution() {} + +class AbsoluteValueSubstitution : public NFSubstitution { +public: + AbsoluteValueSubstitution(int32_t _pos, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, description, status) {} + virtual ~AbsoluteValueSubstitution(); + + virtual int64_t transformNumber(int64_t number) const override { return number >= 0 ? number : -number; } + virtual double transformNumber(double number) const override { return uprv_fabs(number); } + virtual double composeRuleValue(double newRuleValue, double /*oldRuleValue*/) const override { return -newRuleValue; } + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return DBL_MAX; } + virtual char16_t tokenChar() const override { return (char16_t)0x003e; } // '>' + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +AbsoluteValueSubstitution::~AbsoluteValueSubstitution() {} + +class NumeratorSubstitution : public NFSubstitution { + double denominator; + int64_t ldenominator; + UBool withZeros; +public: + static inline UnicodeString fixdesc(const UnicodeString& desc) { + if (desc.endsWith(LTLT, 2)) { + UnicodeString result(desc, 0, desc.length()-1); + return result; + } + return desc; + } + NumeratorSubstitution(int32_t _pos, + double _denominator, + NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, fixdesc(description), status), denominator(_denominator) + { + ldenominator = util64_fromDouble(denominator); + withZeros = description.endsWith(LTLT, 2); + } + virtual ~NumeratorSubstitution(); + + virtual bool operator==(const NFSubstitution& rhs) const override; + + virtual int64_t transformNumber(int64_t number) const override { return number * ldenominator; } + virtual double transformNumber(double number) const override { return uprv_round(number * denominator); } + + virtual void doSubstitution(int64_t /*number*/, UnicodeString& /*toInsertInto*/, int32_t /*_pos*/, int32_t /*recursionCount*/, UErrorCode& /*status*/) const override {} + virtual void doSubstitution(double number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const override; + virtual UBool doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool /*lenientParse*/, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const override; + + virtual double composeRuleValue(double newRuleValue, double oldRuleValue) const override { return newRuleValue / oldRuleValue; } + virtual double calcUpperBound(double /*oldUpperBound*/) const override { return denominator; } + virtual char16_t tokenChar() const override { return (char16_t)0x003c; } // '<' +private: + static const char16_t LTLT[2]; + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +NumeratorSubstitution::~NumeratorSubstitution() {} + +NFSubstitution* +NFSubstitution::makeSubstitution(int32_t pos, + const NFRule* rule, + const NFRule* predecessor, + const NFRuleSet* ruleSet, + const RuleBasedNumberFormat* formatter, + const UnicodeString& description, + UErrorCode& status) +{ + // if the description is empty, return a NullSubstitution + if (description.length() == 0) { + return nullptr; + } + + switch (description.charAt(0)) { + // if the description begins with '<'... + case gLessThan: + // throw an exception if the rule is a negative number + // rule + if (rule->getBaseValue() == NFRule::kNegativeNumberRule) { + // throw new IllegalArgumentException("<< not allowed in negative-number rule"); + status = U_PARSE_ERROR; + return nullptr; + } + + // if the rule is a fraction rule, return an + // IntegralPartSubstitution + else if (rule->getBaseValue() == NFRule::kImproperFractionRule + || rule->getBaseValue() == NFRule::kProperFractionRule + || rule->getBaseValue() == NFRule::kDefaultRule) { + return new IntegralPartSubstitution(pos, ruleSet, description, status); + } + + // if the rule set containing the rule is a fraction + // rule set, return a NumeratorSubstitution + else if (ruleSet->isFractionRuleSet()) { + return new NumeratorSubstitution(pos, (double)rule->getBaseValue(), + formatter->getDefaultRuleSet(), description, status); + } + + // otherwise, return a MultiplierSubstitution + else { + return new MultiplierSubstitution(pos, rule, ruleSet, + description, status); + } + + // if the description begins with '>'... + case gGreaterThan: + // if the rule is a negative-number rule, return + // an AbsoluteValueSubstitution + if (rule->getBaseValue() == NFRule::kNegativeNumberRule) { + return new AbsoluteValueSubstitution(pos, ruleSet, description, status); + } + + // if the rule is a fraction rule, return a + // FractionalPartSubstitution + else if (rule->getBaseValue() == NFRule::kImproperFractionRule + || rule->getBaseValue() == NFRule::kProperFractionRule + || rule->getBaseValue() == NFRule::kDefaultRule) { + return new FractionalPartSubstitution(pos, ruleSet, description, status); + } + + // if the rule set owning the rule is a fraction rule set, + // throw an exception + else if (ruleSet->isFractionRuleSet()) { + // throw new IllegalArgumentException(">> not allowed in fraction rule set"); + status = U_PARSE_ERROR; + return nullptr; + } + + // otherwise, return a ModulusSubstitution + else { + return new ModulusSubstitution(pos, rule, predecessor, + ruleSet, description, status); + } + + // if the description begins with '=', always return a + // SameValueSubstitution + case gEquals: + return new SameValueSubstitution(pos, ruleSet, description, status); + + // and if it's anything else, throw an exception + default: + // throw new IllegalArgumentException("Illegal substitution character"); + status = U_PARSE_ERROR; + } + return nullptr; +} + +NFSubstitution::NFSubstitution(int32_t _pos, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : pos(_pos), ruleSet(nullptr), numberFormat(nullptr) +{ + // the description should begin and end with the same character. + // If it doesn't that's a syntax error. Otherwise, + // makeSubstitution() was the only thing that needed to know + // about these characters, so strip them off + UnicodeString workingDescription(description); + if (description.length() >= 2 + && description.charAt(0) == description.charAt(description.length() - 1)) + { + workingDescription.remove(description.length() - 1, 1); + workingDescription.remove(0, 1); + } + else if (description.length() != 0) { + // throw new IllegalArgumentException("Illegal substitution syntax"); + status = U_PARSE_ERROR; + return; + } + + if (workingDescription.length() == 0) { + // if the description was just two paired token characters + // (i.e., "<<" or ">>"), it uses the rule set it belongs to to + // format its result + this->ruleSet = _ruleSet; + } + else if (workingDescription.charAt(0) == gPercent) { + // if the description contains a rule set name, that's the rule + // set we use to format the result: get a reference to the + // names rule set + this->ruleSet = _ruleSet->getOwner()->findRuleSet(workingDescription, status); + } + else if (workingDescription.charAt(0) == gPound || workingDescription.charAt(0) ==gZero) { + // if the description begins with 0 or #, treat it as a + // DecimalFormat pattern, and initialize a DecimalFormat with + // that pattern (then set it to use the DecimalFormatSymbols + // belonging to our formatter) + const DecimalFormatSymbols* sym = _ruleSet->getOwner()->getDecimalFormatSymbols(); + if (!sym) { + status = U_MISSING_RESOURCE_ERROR; + return; + } + DecimalFormat *tempNumberFormat = new DecimalFormat(workingDescription, *sym, status); + /* test for nullptr */ + if (!tempNumberFormat) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (U_FAILURE(status)) { + delete tempNumberFormat; + return; + } + this->numberFormat = tempNumberFormat; + } + else if (workingDescription.charAt(0) == gGreaterThan) { + // if the description is ">>>", this substitution bypasses the + // usual rule-search process and always uses the rule that precedes + // it in its own rule set's rule list (this is used for place-value + // notations: formats where you want to see a particular part of + // a number even when it's 0) + + // this causes problems when >>> is used in a frationalPartSubstitution + // this->ruleSet = nullptr; + this->ruleSet = _ruleSet; + this->numberFormat = nullptr; + } + else { + // and of the description is none of these things, it's a syntax error + + // throw new IllegalArgumentException("Illegal substitution syntax"); + status = U_PARSE_ERROR; + } +} + +NFSubstitution::~NFSubstitution() +{ + delete numberFormat; + numberFormat = nullptr; +} + +/** + * Set's the substitution's divisor. Used by NFRule.setBaseValue(). + * A no-op for all substitutions except multiplier and modulus + * substitutions. + * @param radix The radix of the divisor + * @param exponent The exponent of the divisor + */ +void +NFSubstitution::setDivisor(int32_t /*radix*/, int16_t /*exponent*/, UErrorCode& /*status*/) { + // a no-op for all substitutions except multiplier and modulus substitutions +} + +void +NFSubstitution::setDecimalFormatSymbols(const DecimalFormatSymbols &newSymbols, UErrorCode& /*status*/) { + if (numberFormat != nullptr) { + numberFormat->setDecimalFormatSymbols(newSymbols); + } +} + +//----------------------------------------------------------------------- +// boilerplate +//----------------------------------------------------------------------- + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NFSubstitution) + +/** + * Compares two substitutions for equality + * @param The substitution to compare this one to + * @return true if the two substitutions are functionally equivalent + */ +bool +NFSubstitution::operator==(const NFSubstitution& rhs) const +{ + // compare class and all of the fields all substitutions have + // in common + // this should be called by subclasses before their own equality tests + return typeid(*this) == typeid(rhs) + && pos == rhs.pos + && (ruleSet == nullptr) == (rhs.ruleSet == nullptr) + // && ruleSet == rhs.ruleSet causes circularity, other checks to make instead? + && (numberFormat == nullptr + ? (rhs.numberFormat == nullptr) + : (*numberFormat == *rhs.numberFormat)); +} + +/** + * Returns a textual description of the substitution + * @return A textual description of the substitution. This might + * not be identical to the description it was created from, but + * it'll produce the same result. + */ +void +NFSubstitution::toString(UnicodeString& text) const +{ + // use tokenChar() to get the character at the beginning and + // end of the substitutin token. In between them will go + // either the name of the rule set it uses, or the pattern of + // the DecimalFormat it uses + text.remove(); + text.append(tokenChar()); + + UnicodeString temp; + if (ruleSet != nullptr) { + ruleSet->getName(temp); + } else if (numberFormat != nullptr) { + numberFormat->toPattern(temp); + } + text.append(temp); + text.append(tokenChar()); +} + +//----------------------------------------------------------------------- +// formatting +//----------------------------------------------------------------------- + +/** + * Performs a mathematical operation on the number, formats it using + * either ruleSet or decimalFormat, and inserts the result into + * toInsertInto. + * @param number The number being formatted. + * @param toInsertInto The string we insert the result into + * @param pos The position in toInsertInto where the owning rule's + * rule text begins (this value is added to this substitution's + * position to determine exactly where to insert the new text) + */ +void +NFSubstitution::doSubstitution(int64_t number, UnicodeString& toInsertInto, int32_t _pos, int32_t recursionCount, UErrorCode& status) const +{ + if (ruleSet != nullptr) { + // Perform a transformation on the number that is dependent + // on the type of substitution this is, then just call its + // rule set's format() method to format the result + ruleSet->format(transformNumber(number), toInsertInto, _pos + this->pos, recursionCount, status); + } else if (numberFormat != nullptr) { + if (number <= MAX_INT64_IN_DOUBLE) { + // or perform the transformation on the number (preserving + // the result's fractional part if the formatter it set + // to show it), then use that formatter's format() method + // to format the result + double numberToFormat = transformNumber((double)number); + if (numberFormat->getMaximumFractionDigits() == 0) { + numberToFormat = uprv_floor(numberToFormat); + } + + UnicodeString temp; + numberFormat->format(numberToFormat, temp, status); + toInsertInto.insert(_pos + this->pos, temp); + } + else { + // We have gone beyond double precision. Something has to give. + // We're favoring accuracy of the large number over potential rules + // that round like a CompactDecimalFormat, which is not a common use case. + // + // Perform a transformation on the number that is dependent + // on the type of substitution this is, then just call its + // rule set's format() method to format the result + int64_t numberToFormat = transformNumber(number); + UnicodeString temp; + numberFormat->format(numberToFormat, temp, status); + toInsertInto.insert(_pos + this->pos, temp); + } + } +} + +/** + * Performs a mathematical operation on the number, formats it using + * either ruleSet or decimalFormat, and inserts the result into + * toInsertInto. + * @param number The number being formatted. + * @param toInsertInto The string we insert the result into + * @param pos The position in toInsertInto where the owning rule's + * rule text begins (this value is added to this substitution's + * position to determine exactly where to insert the new text) + */ +void +NFSubstitution::doSubstitution(double number, UnicodeString& toInsertInto, int32_t _pos, int32_t recursionCount, UErrorCode& status) const { + // perform a transformation on the number being formatted that + // is dependent on the type of substitution this is + double numberToFormat = transformNumber(number); + + if (uprv_isInfinite(numberToFormat)) { + // This is probably a minus rule. Combine it with an infinite rule. + const NFRule *infiniteRule = ruleSet->findDoubleRule(uprv_getInfinity()); + infiniteRule->doFormat(numberToFormat, toInsertInto, _pos + this->pos, recursionCount, status); + return; + } + + // if the result is an integer, from here on out we work in integer + // space (saving time and memory and preserving accuracy) + if (numberToFormat == uprv_floor(numberToFormat) && ruleSet != nullptr) { + ruleSet->format(util64_fromDouble(numberToFormat), toInsertInto, _pos + this->pos, recursionCount, status); + + // if the result isn't an integer, then call either our rule set's + // format() method or our DecimalFormat's format() method to + // format the result + } else { + if (ruleSet != nullptr) { + ruleSet->format(numberToFormat, toInsertInto, _pos + this->pos, recursionCount, status); + } else if (numberFormat != nullptr) { + UnicodeString temp; + numberFormat->format(numberToFormat, temp); + toInsertInto.insert(_pos + this->pos, temp); + } + } +} + + + //----------------------------------------------------------------------- + // parsing + //----------------------------------------------------------------------- + +#ifdef RBNF_DEBUG +#include +#endif + +/** + * Parses a string using the rule set or DecimalFormat belonging + * to this substitution. If there's a match, a mathematical + * operation (the inverse of the one used in formatting) is + * performed on the result of the parse and the value passed in + * and returned as the result. The parse position is updated to + * point to the first unmatched character in the string. + * @param text The string to parse + * @param parsePosition On entry, ignored, but assumed to be 0. + * On exit, this is updated to point to the first unmatched + * character (or 0 if the substitution didn't match) + * @param baseValue A partial parse result that should be + * combined with the result of this parse + * @param upperBound When searching the rule set for a rule + * matching the string passed in, only rules with base values + * lower than this are considered + * @param lenientParse If true and matching against rules fails, + * the substitution will also try matching the text against + * numerals using a default-costructed NumberFormat. If false, + * no extra work is done. (This value is false whenever the + * formatter isn't in lenient-parse mode, but is also false + * under some conditions even when the formatter _is_ in + * lenient-parse mode.) + * @return If there's a match, this is the result of composing + * baseValue with whatever was returned from matching the + * characters. This will be either a Long or a Double. If there's + * no match this is new Long(0) (not null), and parsePosition + * is left unchanged. + */ +UBool +NFSubstitution::doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const +{ +#ifdef RBNF_DEBUG + fprintf(stderr, " %x bv: %g ub: %g\n", this, baseValue, upperBound); +#endif + // figure out the highest base value a rule can have and match + // the text being parsed (this varies according to the type of + // substitutions: multiplier, modulus, and numerator substitutions + // restrict the search to rules with base values lower than their + // own; same-value substitutions leave the upper bound wherever + // it was, and the others allow any rule to match + upperBound = calcUpperBound(upperBound); + + // use our rule set to parse the text. If that fails and + // lenient parsing is enabled (this is always false if the + // formatter's lenient-parsing mode is off, but it may also + // be false even when the formatter's lenient-parse mode is + // on), then also try parsing the text using a default- + // constructed NumberFormat + if (ruleSet != nullptr) { + ruleSet->parse(text, parsePosition, upperBound, nonNumericalExecutedRuleMask, result); + if (lenientParse && !ruleSet->isFractionRuleSet() && parsePosition.getIndex() == 0) { + UErrorCode status = U_ZERO_ERROR; + NumberFormat* fmt = NumberFormat::createInstance(status); + if (U_SUCCESS(status)) { + fmt->parse(text, result, parsePosition); + } + delete fmt; + } + + // ...or use our DecimalFormat to parse the text + } else if (numberFormat != nullptr) { + numberFormat->parse(text, result, parsePosition); + } + + // if the parse was successful, we've already advanced the caller's + // parse position (this is the one function that doesn't have one + // of its own). Derive a parse result and return it as a Long, + // if possible, or a Double + if (parsePosition.getIndex() != 0) { + UErrorCode status = U_ZERO_ERROR; + double tempResult = result.getDouble(status); + + // composeRuleValue() produces a full parse result from + // the partial parse result passed to this function from + // the caller (this is either the owning rule's base value + // or the partial result obtained from composing the + // owning rule's base value with its other substitution's + // parse result) and the partial parse result obtained by + // matching the substitution (which will be the same value + // the caller would get by parsing just this part of the + // text with RuleBasedNumberFormat.parse() ). How the two + // values are used to derive the full parse result depends + // on the types of substitutions: For a regular rule, the + // ultimate result is its multiplier substitution's result + // times the rule's divisor (or the rule's base value) plus + // the modulus substitution's result (which will actually + // supersede part of the rule's base value). For a negative- + // number rule, the result is the negative of its substitution's + // result. For a fraction rule, it's the sum of its two + // substitution results. For a rule in a fraction rule set, + // it's the numerator substitution's result divided by + // the rule's base value. Results from same-value substitutions + // propagate back upard, and null substitutions don't affect + // the result. + tempResult = composeRuleValue(tempResult, baseValue); + result.setDouble(tempResult); + return true; + // if the parse was UNsuccessful, return 0 + } else { + result.setLong(0); + return false; + } +} + + /** + * Returns true if this is a modulus substitution. (We didn't do this + * with instanceof partially because it causes source files to + * proliferate and partially because we have to port this to C++.) + * @return true if this object is an instance of ModulusSubstitution + */ +UBool +NFSubstitution::isModulusSubstitution() const { + return false; +} + +//=================================================================== +// SameValueSubstitution +//=================================================================== + +/** + * A substitution that passes the value passed to it through unchanged. + * Represented by == in rule descriptions. + */ +SameValueSubstitution::SameValueSubstitution(int32_t _pos, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) +: NFSubstitution(_pos, _ruleSet, description, status) +{ + if (0 == description.compare(gEqualsEquals, 2)) { + // throw new IllegalArgumentException("== is not a legal token"); + status = U_PARSE_ERROR; + } +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SameValueSubstitution) + +//=================================================================== +// MultiplierSubstitution +//=================================================================== + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(MultiplierSubstitution) + +bool MultiplierSubstitution::operator==(const NFSubstitution& rhs) const +{ + return NFSubstitution::operator==(rhs) && + divisor == ((const MultiplierSubstitution*)&rhs)->divisor; +} + + +//=================================================================== +// ModulusSubstitution +//=================================================================== + +/** + * A substitution that divides the number being formatted by the its rule's + * divisor and formats the remainder. Represented by ">>" in a + * regular rule. + */ +ModulusSubstitution::ModulusSubstitution(int32_t _pos, + const NFRule* rule, + const NFRule* predecessor, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, description, status) + , divisor(rule->getDivisor()) + , ruleToUse(nullptr) +{ + // the owning rule's divisor controls the behavior of this + // substitution: rather than keeping a backpointer to the rule, + // we keep a copy of the divisor + + if (divisor == 0) { + status = U_PARSE_ERROR; + } + + if (0 == description.compare(gGreaterGreaterGreaterThan, 3)) { + // the >>> token doesn't alter how this substitution calculates the + // values it uses for formatting and parsing, but it changes + // what's done with that value after it's obtained: >>> short- + // circuits the rule-search process and goes straight to the + // specified rule to format the substitution value + ruleToUse = predecessor; + } +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ModulusSubstitution) + +bool ModulusSubstitution::operator==(const NFSubstitution& rhs) const +{ + return NFSubstitution::operator==(rhs) && + divisor == ((const ModulusSubstitution*)&rhs)->divisor && + ruleToUse == ((const ModulusSubstitution*)&rhs)->ruleToUse; +} + +//----------------------------------------------------------------------- +// formatting +//----------------------------------------------------------------------- + + +/** + * If this is a >>> substitution, use ruleToUse to fill in + * the substitution. Otherwise, just use the superclass function. + * @param number The number being formatted + * @toInsertInto The string to insert the result of this substitution + * into + * @param pos The position of the rule text in toInsertInto + */ +void +ModulusSubstitution::doSubstitution(int64_t number, UnicodeString& toInsertInto, int32_t _pos, int32_t recursionCount, UErrorCode& status) const +{ + // if this isn't a >>> substitution, just use the inherited version + // of this function (which uses either a rule set or a DecimalFormat + // to format its substitution value) + if (ruleToUse == nullptr) { + NFSubstitution::doSubstitution(number, toInsertInto, _pos, recursionCount, status); + + // a >>> substitution goes straight to a particular rule to + // format the substitution value + } else { + int64_t numberToFormat = transformNumber(number); + ruleToUse->doFormat(numberToFormat, toInsertInto, _pos + getPos(), recursionCount, status); + } +} + +/** +* If this is a >>> substitution, use ruleToUse to fill in +* the substitution. Otherwise, just use the superclass function. +* @param number The number being formatted +* @toInsertInto The string to insert the result of this substitution +* into +* @param pos The position of the rule text in toInsertInto +*/ +void +ModulusSubstitution::doSubstitution(double number, UnicodeString& toInsertInto, int32_t _pos, int32_t recursionCount, UErrorCode& status) const +{ + // if this isn't a >>> substitution, just use the inherited version + // of this function (which uses either a rule set or a DecimalFormat + // to format its substitution value) + if (ruleToUse == nullptr) { + NFSubstitution::doSubstitution(number, toInsertInto, _pos, recursionCount, status); + + // a >>> substitution goes straight to a particular rule to + // format the substitution value + } else { + double numberToFormat = transformNumber(number); + + ruleToUse->doFormat(numberToFormat, toInsertInto, _pos + getPos(), recursionCount, status); + } +} + +//----------------------------------------------------------------------- +// parsing +//----------------------------------------------------------------------- + +/** + * If this is a >>> substitution, match only against ruleToUse. + * Otherwise, use the superclass function. + * @param text The string to parse + * @param parsePosition Ignored on entry, updated on exit to point to + * the first unmatched character. + * @param baseValue The partial parse result prior to calling this + * routine. + */ +UBool +ModulusSubstitution::doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const +{ + // if this isn't a >>> substitution, we can just use the + // inherited parse() routine to do the parsing + if (ruleToUse == nullptr) { + return NFSubstitution::doParse(text, parsePosition, baseValue, upperBound, lenientParse, nonNumericalExecutedRuleMask, result); + + // but if it IS a >>> substitution, we have to do it here: we + // use the specific rule's doParse() method, and then we have to + // do some of the other work of NFRuleSet.parse() + } else { + ruleToUse->doParse(text, parsePosition, false, upperBound, nonNumericalExecutedRuleMask, result); + + if (parsePosition.getIndex() != 0) { + UErrorCode status = U_ZERO_ERROR; + double tempResult = result.getDouble(status); + tempResult = composeRuleValue(tempResult, baseValue); + result.setDouble(tempResult); + } + + return true; + } +} +/** + * Returns a textual description of the substitution + * @return A textual description of the substitution. This might + * not be identical to the description it was created from, but + * it'll produce the same result. + */ +void +ModulusSubstitution::toString(UnicodeString& text) const +{ + // use tokenChar() to get the character at the beginning and + // end of the substitutin token. In between them will go + // either the name of the rule set it uses, or the pattern of + // the DecimalFormat it uses + + if ( ruleToUse != nullptr ) { // Must have been a >>> substitution. + text.remove(); + text.append(tokenChar()); + text.append(tokenChar()); + text.append(tokenChar()); + } else { // Otherwise just use the super-class function. + NFSubstitution::toString(text); + } +} +//=================================================================== +// IntegralPartSubstitution +//=================================================================== + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(IntegralPartSubstitution) + + +//=================================================================== +// FractionalPartSubstitution +//=================================================================== + + + /** + * Constructs a FractionalPartSubstitution. This object keeps a flag + * telling whether it should format by digits or not. In addition, + * it marks the rule set it calls (if any) as a fraction rule set. + */ +FractionalPartSubstitution::FractionalPartSubstitution(int32_t _pos, + const NFRuleSet* _ruleSet, + const UnicodeString& description, + UErrorCode& status) + : NFSubstitution(_pos, _ruleSet, description, status) + , byDigits(false) + , useSpaces(true) + +{ + // akk, ruleSet can change in superclass constructor + if (0 == description.compare(gGreaterGreaterThan, 2) || + 0 == description.compare(gGreaterGreaterGreaterThan, 3) || + _ruleSet == getRuleSet()) { + byDigits = true; + if (0 == description.compare(gGreaterGreaterGreaterThan, 3)) { + useSpaces = false; + } + } else { + // cast away const + ((NFRuleSet*)getRuleSet())->makeIntoFractionRuleSet(); + } +} + +//----------------------------------------------------------------------- +// formatting +//----------------------------------------------------------------------- + +/** + * If in "by digits" mode, fills in the substitution one decimal digit + * at a time using the rule set containing this substitution. + * Otherwise, uses the superclass function. + * @param number The number being formatted + * @param toInsertInto The string to insert the result of formatting + * the substitution into + * @param pos The position of the owning rule's rule text in + * toInsertInto + */ +void +FractionalPartSubstitution::doSubstitution(double number, UnicodeString& toInsertInto, + int32_t _pos, int32_t recursionCount, UErrorCode& status) const +{ + // if we're not in "byDigits" mode, just use the inherited + // doSubstitution() routine + if (!byDigits) { + NFSubstitution::doSubstitution(number, toInsertInto, _pos, recursionCount, status); + + // if we're in "byDigits" mode, transform the value into an integer + // by moving the decimal point eight places to the right and + // pulling digits off the right one at a time, formatting each digit + // as an integer using this substitution's owning rule set + // (this is slower, but more accurate, than doing it from the + // other end) + } else { + // int32_t numberToFormat = (int32_t)uprv_round(transformNumber(number) * uprv_pow(10, kMaxDecimalDigits)); + // // this flag keeps us from formatting trailing zeros. It starts + // // out false because we're pulling from the right, and switches + // // to true the first time we encounter a non-zero digit + // UBool doZeros = false; + // for (int32_t i = 0; i < kMaxDecimalDigits; i++) { + // int64_t digit = numberToFormat % 10; + // if (digit != 0 || doZeros) { + // if (doZeros && useSpaces) { + // toInsertInto.insert(_pos + getPos(), gSpace); + // } + // doZeros = true; + // getRuleSet()->format(digit, toInsertInto, _pos + getPos()); + // } + // numberToFormat /= 10; + // } + + DecimalQuantity dl; + dl.setToDouble(number); + dl.roundToMagnitude(-20, UNUM_ROUND_HALFEVEN, status); // round to 20 fraction digits. + + UBool pad = false; + for (int32_t didx = dl.getLowerDisplayMagnitude(); didx<0; didx++) { + // Loop iterates over fraction digits, starting with the LSD. + // include both real digits from the number, and zeros + // to the left of the MSD but to the right of the decimal point. + if (pad && useSpaces) { + toInsertInto.insert(_pos + getPos(), gSpace); + } else { + pad = true; + } + int64_t digit = dl.getDigit(didx); + getRuleSet()->format(digit, toInsertInto, _pos + getPos(), recursionCount, status); + } + + if (!pad) { + // hack around lack of precision in digitlist. if we would end up with + // "foo point" make sure we add a " zero" to the end. + getRuleSet()->format((int64_t)0, toInsertInto, _pos + getPos(), recursionCount, status); + } + } +} + +//----------------------------------------------------------------------- +// parsing +//----------------------------------------------------------------------- + +/** + * If in "by digits" mode, parses the string as if it were a string + * of individual digits; otherwise, uses the superclass function. + * @param text The string to parse + * @param parsePosition Ignored on entry, but updated on exit to point + * to the first unmatched character + * @param baseValue The partial parse result prior to entering this + * function + * @param upperBound Only consider rules with base values lower than + * this when filling in the substitution + * @param lenientParse If true, try matching the text as numerals if + * matching as words doesn't work + * @return If the match was successful, the current partial parse + * result; otherwise new Long(0). The result is either a Long or + * a Double. + */ + +UBool +FractionalPartSubstitution::doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double /*upperBound*/, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& resVal) const +{ + // if we're not in byDigits mode, we can just use the inherited + // doParse() + if (!byDigits) { + return NFSubstitution::doParse(text, parsePosition, baseValue, 0, lenientParse, nonNumericalExecutedRuleMask, resVal); + + // if we ARE in byDigits mode, parse the text one digit at a time + // using this substitution's owning rule set (we do this by setting + // upperBound to 10 when calling doParse() ) until we reach + // nonmatching text + } else { + UnicodeString workText(text); + ParsePosition workPos(1); + double result = 0; + int32_t digit; +// double p10 = 0.1; + + DecimalQuantity dl; + int32_t totalDigits = 0; + NumberFormat* fmt = nullptr; + while (workText.length() > 0 && workPos.getIndex() != 0) { + workPos.setIndex(0); + Formattable temp; + getRuleSet()->parse(workText, workPos, 10, nonNumericalExecutedRuleMask, temp); + UErrorCode status = U_ZERO_ERROR; + digit = temp.getLong(status); +// digit = temp.getType() == Formattable::kLong ? +// temp.getLong() : +// (int32_t)temp.getDouble(); + + if (lenientParse && workPos.getIndex() == 0) { + if (!fmt) { + status = U_ZERO_ERROR; + fmt = NumberFormat::createInstance(status); + if (U_FAILURE(status)) { + delete fmt; + fmt = nullptr; + } + } + if (fmt) { + fmt->parse(workText, temp, workPos); + digit = temp.getLong(status); + } + } + + if (workPos.getIndex() != 0) { + dl.appendDigit(static_cast(digit), 0, true); + totalDigits++; +// result += digit * p10; +// p10 /= 10; + parsePosition.setIndex(parsePosition.getIndex() + workPos.getIndex()); + workText.removeBetween(0, workPos.getIndex()); + while (workText.length() > 0 && workText.charAt(0) == gSpace) { + workText.removeBetween(0, 1); + parsePosition.setIndex(parsePosition.getIndex() + 1); + } + } + } + delete fmt; + + dl.adjustMagnitude(-totalDigits); + result = dl.toDouble(); + result = composeRuleValue(result, baseValue); + resVal.setDouble(result); + return true; + } +} + +bool +FractionalPartSubstitution::operator==(const NFSubstitution& rhs) const +{ + return NFSubstitution::operator==(rhs) && + ((const FractionalPartSubstitution*)&rhs)->byDigits == byDigits; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(FractionalPartSubstitution) + + +//=================================================================== +// AbsoluteValueSubstitution +//=================================================================== + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(AbsoluteValueSubstitution) + +//=================================================================== +// NumeratorSubstitution +//=================================================================== + +void +NumeratorSubstitution::doSubstitution(double number, UnicodeString& toInsertInto, int32_t apos, int32_t recursionCount, UErrorCode& status) const { + // perform a transformation on the number being formatted that + // is dependent on the type of substitution this is + + double numberToFormat = transformNumber(number); + int64_t longNF = util64_fromDouble(numberToFormat); + + const NFRuleSet* aruleSet = getRuleSet(); + if (withZeros && aruleSet != nullptr) { + // if there are leading zeros in the decimal expansion then emit them + int64_t nf =longNF; + int32_t len = toInsertInto.length(); + while ((nf *= 10) < denominator) { + toInsertInto.insert(apos + getPos(), gSpace); + aruleSet->format((int64_t)0, toInsertInto, apos + getPos(), recursionCount, status); + } + apos += toInsertInto.length() - len; + } + + // if the result is an integer, from here on out we work in integer + // space (saving time and memory and preserving accuracy) + if (numberToFormat == longNF && aruleSet != nullptr) { + aruleSet->format(longNF, toInsertInto, apos + getPos(), recursionCount, status); + + // if the result isn't an integer, then call either our rule set's + // format() method or our DecimalFormat's format() method to + // format the result + } else { + if (aruleSet != nullptr) { + aruleSet->format(numberToFormat, toInsertInto, apos + getPos(), recursionCount, status); + } else { + UnicodeString temp; + getNumberFormat()->format(numberToFormat, temp, status); + toInsertInto.insert(apos + getPos(), temp); + } + } +} + +UBool +NumeratorSubstitution::doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool /*lenientParse*/, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const +{ + // we don't have to do anything special to do the parsing here, + // but we have to turn lenient parsing off-- if we leave it on, + // it SERIOUSLY messes up the algorithm + + // if withZeros is true, we need to count the zeros + // and use that to adjust the parse result + UErrorCode status = U_ZERO_ERROR; + int32_t zeroCount = 0; + UnicodeString workText(text); + + if (withZeros) { + ParsePosition workPos(1); + Formattable temp; + + while (workText.length() > 0 && workPos.getIndex() != 0) { + workPos.setIndex(0); + getRuleSet()->parse(workText, workPos, 1, nonNumericalExecutedRuleMask, temp); // parse zero or nothing at all + if (workPos.getIndex() == 0) { + // we failed, either there were no more zeros, or the number was formatted with digits + // either way, we're done + break; + } + + ++zeroCount; + parsePosition.setIndex(parsePosition.getIndex() + workPos.getIndex()); + workText.remove(0, workPos.getIndex()); + while (workText.length() > 0 && workText.charAt(0) == gSpace) { + workText.remove(0, 1); + parsePosition.setIndex(parsePosition.getIndex() + 1); + } + } + + workText = text; + workText.remove(0, (int32_t)parsePosition.getIndex()); + parsePosition.setIndex(0); + } + + // we've parsed off the zeros, now let's parse the rest from our current position + NFSubstitution::doParse(workText, parsePosition, withZeros ? 1 : baseValue, upperBound, false, nonNumericalExecutedRuleMask, result); + + if (withZeros) { + // any base value will do in this case. is there a way to + // force this to not bother trying all the base values? + + // compute the 'effective' base and prescale the value down + int64_t n = result.getLong(status); // force conversion! + int64_t d = 1; + while (d <= n) { + d *= 10; + } + // now add the zeros + while (zeroCount > 0) { + d *= 10; + --zeroCount; + } + // d is now our true denominator + result.setDouble((double)n/(double)d); + } + + return true; +} + +bool +NumeratorSubstitution::operator==(const NFSubstitution& rhs) const +{ + return NFSubstitution::operator==(rhs) && + denominator == ((const NumeratorSubstitution*)&rhs)->denominator; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NumeratorSubstitution) + +const char16_t NumeratorSubstitution::LTLT[] = { 0x003c, 0x003c }; + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif + diff --git a/intl/icu/source/i18n/nfsubs.h b/intl/icu/source/i18n/nfsubs.h new file mode 100644 index 0000000000..d252f86499 --- /dev/null +++ b/intl/icu/source/i18n/nfsubs.h @@ -0,0 +1,262 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 1997-2015, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* file name: nfsubs.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* Modification history +* Date Name Comments +* 10/11/2001 Doug Ported from ICU4J +*/ + +#ifndef NFSUBS_H +#define NFSUBS_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "nfrule.h" + +#if U_HAVE_RBNF + +#include "unicode/utypes.h" +#include "unicode/decimfmt.h" +#include "nfrs.h" +#include + +U_NAMESPACE_BEGIN + +class NFSubstitution : public UObject { + int32_t pos; + const NFRuleSet* ruleSet; + DecimalFormat* numberFormat; + +protected: + NFSubstitution(int32_t pos, + const NFRuleSet* ruleSet, + const UnicodeString& description, + UErrorCode& status); + + /** + * Get the Ruleset of the object. + * @return the Ruleset of the object. + */ + const NFRuleSet* getRuleSet() const { return ruleSet; } + + /** + * get the NumberFormat of this object. + * @return the numberformat of this object. + */ + const DecimalFormat* getNumberFormat() const { return numberFormat; } + +public: + static NFSubstitution* makeSubstitution(int32_t pos, + const NFRule* rule, + const NFRule* predecessor, + const NFRuleSet* ruleSet, + const RuleBasedNumberFormat* rbnf, + const UnicodeString& description, + UErrorCode& status); + + /** + * Destructor. + */ + virtual ~NFSubstitution(); + + /** + * Return true if the given Format objects are semantically equal. + * Objects of different subclasses are considered unequal. + * @param rhs the object to be compared with. + * @return true if the given Format objects are semantically equal. + */ + virtual bool operator==(const NFSubstitution& rhs) const; + + /** + * Return true if the given Format objects are semantically unequal. + * Objects of different subclasses are considered unequal. + * @param rhs the object to be compared with. + * @return true if the given Format objects are semantically unequal. + */ + bool operator!=(const NFSubstitution& rhs) const { return !operator==(rhs); } + + /** + * Sets the substitution's divisor. Used by NFRule.setBaseValue(). + * A no-op for all substitutions except multiplier and modulus + * substitutions. + * @param radix The radix of the divisor + * @param exponent The exponent of the divisor + */ + virtual void setDivisor(int32_t radix, int16_t exponent, UErrorCode& status); + + /** + * Replaces result with the string describing the substitution. + * @param result Output param which will receive the string. + */ + virtual void toString(UnicodeString& result) const; + + void setDecimalFormatSymbols(const DecimalFormatSymbols &newSymbols, UErrorCode& status); + + //----------------------------------------------------------------------- + // formatting + //----------------------------------------------------------------------- + + /** + * Performs a mathematical operation on the number, formats it using + * either ruleSet or decimalFormat, and inserts the result into + * toInsertInto. + * @param number The number being formatted. + * @param toInsertInto The string we insert the result into + * @param pos The position in toInsertInto where the owning rule's + * rule text begins (this value is added to this substitution's + * position to determine exactly where to insert the new text) + */ + virtual void doSubstitution(int64_t number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + + /** + * Performs a mathematical operation on the number, formats it using + * either ruleSet or decimalFormat, and inserts the result into + * toInsertInto. + * @param number The number being formatted. + * @param toInsertInto The string we insert the result into + * @param pos The position in toInsertInto where the owning rule's + * rule text begins (this value is added to this substitution's + * position to determine exactly where to insert the new text) + */ + virtual void doSubstitution(double number, UnicodeString& toInsertInto, int32_t pos, int32_t recursionCount, UErrorCode& status) const; + +protected: + /** + * Subclasses override this function to perform some kind of + * mathematical operation on the number. The result of this operation + * is formatted using the rule set or DecimalFormat that this + * substitution refers to, and the result is inserted into the result + * string. + * @param The number being formatted + * @return The result of performing the opreration on the number + */ + virtual int64_t transformNumber(int64_t number) const = 0; + + /** + * Subclasses override this function to perform some kind of + * mathematical operation on the number. The result of this operation + * is formatted using the rule set or DecimalFormat that this + * substitution refers to, and the result is inserted into the result + * string. + * @param The number being formatted + * @return The result of performing the opreration on the number + */ + virtual double transformNumber(double number) const = 0; + +public: + //----------------------------------------------------------------------- + // parsing + //----------------------------------------------------------------------- + + /** + * Parses a string using the rule set or DecimalFormat belonging + * to this substitution. If there's a match, a mathematical + * operation (the inverse of the one used in formatting) is + * performed on the result of the parse and the value passed in + * and returned as the result. The parse position is updated to + * point to the first unmatched character in the string. + * @param text The string to parse + * @param parsePosition On entry, ignored, but assumed to be 0. + * On exit, this is updated to point to the first unmatched + * character (or 0 if the substitution didn't match) + * @param baseValue A partial parse result that should be + * combined with the result of this parse + * @param upperBound When searching the rule set for a rule + * matching the string passed in, only rules with base values + * lower than this are considered + * @param lenientParse If true and matching against rules fails, + * the substitution will also try matching the text against + * numerals using a default-costructed NumberFormat. If false, + * no extra work is done. (This value is false whenever the + * formatter isn't in lenient-parse mode, but is also false + * under some conditions even when the formatter _is_ in + * lenient-parse mode.) + * @return If there's a match, this is the result of composing + * baseValue with whatever was returned from matching the + * characters. This will be either a Long or a Double. If there's + * no match this is new Long(0) (not null), and parsePosition + * is left unchanged. + */ + virtual UBool doParse(const UnicodeString& text, + ParsePosition& parsePosition, + double baseValue, + double upperBound, + UBool lenientParse, + uint32_t nonNumericalExecutedRuleMask, + Formattable& result) const; + + /** + * Derives a new value from the two values passed in. The two values + * are typically either the base values of two rules (the one containing + * the substitution and the one matching the substitution) or partial + * parse results derived in some other way. The operation is generally + * the inverse of the operation performed by transformNumber(). + * @param newRuleValue The value produced by matching this substitution + * @param oldRuleValue The value that was passed to the substitution + * by the rule that owns it + * @return A third value derived from the other two, representing a + * partial parse result + */ + virtual double composeRuleValue(double newRuleValue, double oldRuleValue) const = 0; + + /** + * Calculates an upper bound when searching for a rule that matches + * this substitution. Rules with base values greater than or equal + * to upperBound are not considered. + * @param oldUpperBound The current upper-bound setting. The new + * upper bound can't be any higher. + * @return the upper bound when searching for a rule that matches + * this substitution. + */ + virtual double calcUpperBound(double oldUpperBound) const = 0; + + //----------------------------------------------------------------------- + // simple accessors + //----------------------------------------------------------------------- + + /** + * Returns the substitution's position in the rule that owns it. + * @return The substitution's position in the rule that owns it. + */ + int32_t getPos() const { return pos; } + + /** + * Returns the character used in the textual representation of + * substitutions of this type. Used by toString(). + * @return This substitution's token character. + */ + virtual char16_t tokenChar() const = 0; + + /** + * Returns true if this is a modulus substitution. (We didn't do this + * with instanceof partially because it causes source files to + * proliferate and partially because we have to port this to C++.) + * @return true if this object is an instance of ModulusSubstitution + */ + virtual UBool isModulusSubstitution() const; + +private: + NFSubstitution(const NFSubstitution &other) = delete; // forbid copying of this class + NFSubstitution &operator=(const NFSubstitution &other) = delete; // forbid copying of this class + +public: + static UClassID getStaticClassID(); + virtual UClassID getDynamicClassID() const override; +}; + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif + +// NFSUBS_H +#endif diff --git a/intl/icu/source/i18n/nortrans.cpp b/intl/icu/source/i18n/nortrans.cpp new file mode 100644 index 0000000000..d793433b3d --- /dev/null +++ b/intl/icu/source/i18n/nortrans.cpp @@ -0,0 +1,178 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2001-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 07/03/01 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/normalizer2.h" +#include "unicode/utf16.h" +#include "cstring.h" +#include "nortrans.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NormalizationTransliterator) + +static inline Transliterator::Token cstrToken(const char *s) { + return Transliterator::pointerToken((void *)s); +} + +/** + * System registration hook. + */ +void NormalizationTransliterator::registerIDs() { + // In the Token, the byte after the NUL is the UNormalization2Mode. + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFC"), + _create, cstrToken("nfc\0\0")); + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFKC"), + _create, cstrToken("nfkc\0\0")); + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFD"), + _create, cstrToken("nfc\0\1")); + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFKD"), + _create, cstrToken("nfkc\0\1")); + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-FCD"), + _create, cstrToken("nfc\0\2")); + Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-FCC"), + _create, cstrToken("nfc\0\3")); + Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("NFC"), + UNICODE_STRING_SIMPLE("NFD"), true); + Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("NFKC"), + UNICODE_STRING_SIMPLE("NFKD"), true); + Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("FCC"), + UNICODE_STRING_SIMPLE("NFD"), false); + Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("FCD"), + UNICODE_STRING_SIMPLE("FCD"), false); +} + +/** + * Factory methods + */ +Transliterator* NormalizationTransliterator::_create(const UnicodeString& ID, + Token context) { + const char *name = (const char *)context.pointer; + UNormalization2Mode mode = (UNormalization2Mode)uprv_strchr(name, 0)[1]; + UErrorCode errorCode = U_ZERO_ERROR; + const Normalizer2 *norm2 = Normalizer2::getInstance(nullptr, name, mode, errorCode); + if(U_SUCCESS(errorCode)) { + return new NormalizationTransliterator(ID, *norm2); + } else { + return nullptr; + } +} + +/** + * Constructs a transliterator. + */ +NormalizationTransliterator::NormalizationTransliterator(const UnicodeString& id, + const Normalizer2 &norm2) : + Transliterator(id, 0), fNorm2(norm2) {} + +/** + * Destructor. + */ +NormalizationTransliterator::~NormalizationTransliterator() { +} + +/** + * Copy constructor. + */ +NormalizationTransliterator::NormalizationTransliterator(const NormalizationTransliterator& o) : + Transliterator(o), fNorm2(o.fNorm2) {} + +/** + * Transliterator API. + */ +NormalizationTransliterator* NormalizationTransliterator::clone() const { + return new NormalizationTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void NormalizationTransliterator::handleTransliterate(Replaceable& text, UTransPosition& offsets, + UBool isIncremental) const { + // start and limit of the input range + int32_t start = offsets.start; + int32_t limit = offsets.limit; + if(start >= limit) { + return; + } + + /* + * Normalize as short chunks at a time as possible even in + * bulk mode, so that styled text is minimally disrupted. + * In incremental mode, a chunk that ends with offsets.limit + * must not be normalized. + * + * If it was known that the input text is not styled, then + * a bulk mode normalization could look like this: + + UnicodeString input, normalized; + int32_t length = limit - start; + _Replaceable_extractBetween(text, start, limit, input.getBuffer(length)); + input.releaseBuffer(length); + + UErrorCode status = U_ZERO_ERROR; + fNorm2.normalize(input, normalized, status); + + text.handleReplaceBetween(start, limit, normalized); + + int32_t delta = normalized.length() - length; + offsets.contextLimit += delta; + offsets.limit += delta; + offsets.start = limit + delta; + + */ + UErrorCode errorCode = U_ZERO_ERROR; + UnicodeString segment; + UnicodeString normalized; + UChar32 c = text.char32At(start); + do { + int32_t prev = start; + // Skip at least one character so we make progress. + // c holds the character at start. + segment.remove(); + do { + segment.append(c); + start += U16_LENGTH(c); + } while(start < limit && !fNorm2.hasBoundaryBefore(c = text.char32At(start))); + if(start == limit && isIncremental && !fNorm2.hasBoundaryAfter(c)) { + // stop in incremental mode when we reach the input limit + // in case there are additional characters that could change the + // normalization result + start=prev; + break; + } + fNorm2.normalize(segment, normalized, errorCode); + if(U_FAILURE(errorCode)) { + break; + } + if(segment != normalized) { + // replace the input chunk with its normalized form + text.handleReplaceBetween(prev, start, normalized); + + // update all necessary indexes accordingly + int32_t delta = normalized.length() - (start - prev); + start += delta; + limit += delta; + } + } while(start < limit); + + offsets.start = start; + offsets.contextLimit += limit - offsets.limit; + offsets.limit = limit; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/nortrans.h b/intl/icu/source/i18n/nortrans.h new file mode 100644 index 0000000000..01cb97ab38 --- /dev/null +++ b/intl/icu/source/i18n/nortrans.h @@ -0,0 +1,102 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2001-2010, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 07/03/01 aliu Creation. +********************************************************************** +*/ +#ifndef NORTRANS_H +#define NORTRANS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" +#include "unicode/normalizer2.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that performs normalization. + * @author Alan Liu + */ +class NormalizationTransliterator : public Transliterator { + const Normalizer2 &fNorm2; + + public: + + /** + * Destructor. + */ + virtual ~NormalizationTransliterator(); + + /** + * Copy constructor. + */ + NormalizationTransliterator(const NormalizationTransliterator&); + + /** + * Transliterator API. + * @return A copy of the object. + */ + virtual NormalizationTransliterator* clone() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + protected: + + /** + * Implements {@link Transliterator#handleTransliterate}. + * @param text the buffer holding transliterated and + * untransliterated text + * @param offset the start and limit of the text, the position + * of the cursor, and the start and limit of transliteration. + * @param incremental if true, assume more text may be coming after + * pos.contextLimit. Otherwise, assume the text is complete. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + public: + + /** + * System registration hook. Public to Transliterator only. + */ + static void registerIDs(); + + private: + + // Transliterator::Factory methods + static Transliterator* _create(const UnicodeString& ID, + Token context); + + /** + * Constructs a transliterator. This method is private. + * Public users must use the factory method createInstance(). + */ + NormalizationTransliterator(const UnicodeString& id, const Normalizer2 &norm2); + +private: + /** + * Assignment operator. + */ + NormalizationTransliterator& operator=(const NormalizationTransliterator&); +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/nultrans.cpp b/intl/icu/source/i18n/nultrans.cpp new file mode 100644 index 0000000000..439cc55d38 --- /dev/null +++ b/intl/icu/source/i18n/nultrans.cpp @@ -0,0 +1,38 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2000-2005, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 01/11/2000 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "nultrans.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NullTransliterator) + +NullTransliterator::NullTransliterator() : Transliterator(UNICODE_STRING_SIMPLE("Any-Null"), 0) {} + +NullTransliterator::~NullTransliterator() {} + +NullTransliterator* NullTransliterator::clone() const { + return new NullTransliterator(); +} + +void NullTransliterator::handleTransliterate(Replaceable& /*text*/, UTransPosition& offsets, + UBool /*isIncremental*/) const { + offsets.start = offsets.limit; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/nultrans.h b/intl/icu/source/i18n/nultrans.h new file mode 100644 index 0000000000..f5f2fbc911 --- /dev/null +++ b/intl/icu/source/i18n/nultrans.h @@ -0,0 +1,73 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 01/11/2000 aliu Creation. +********************************************************************** +*/ +#ifndef NULTRANS_H +#define NULTRANS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that leaves text unchanged. + * @author Alan Liu + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ +class NullTransliterator : public Transliterator { + +public: + + /** + * Constructs a transliterator. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + NullTransliterator(); + + /** + * Destructor. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual ~NullTransliterator(); + + /** + * Transliterator API. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual NullTransliterator* clone() const override; + + /** + * Implements {@link Transliterator#handleTransliterate}. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/number_affixutils.cpp b/intl/icu/source/i18n/number_affixutils.cpp new file mode 100644 index 0000000000..5f5ff4c3a6 --- /dev/null +++ b/intl/icu/source/i18n/number_affixutils.cpp @@ -0,0 +1,444 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "number_affixutils.h" +#include "unicode/utf16.h" +#include "unicode/uniset.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +TokenConsumer::~TokenConsumer() = default; +SymbolProvider::~SymbolProvider() = default; + +int32_t AffixUtils::estimateLength(const UnicodeString &patternString, UErrorCode &status) { + AffixPatternState state = STATE_BASE; + int32_t offset = 0; + int32_t length = 0; + for (; offset < patternString.length();) { + UChar32 cp = patternString.char32At(offset); + + switch (state) { + case STATE_BASE: + if (cp == u'\'') { + // First quote + state = STATE_FIRST_QUOTE; + } else { + // Unquoted symbol + length++; + } + break; + case STATE_FIRST_QUOTE: + if (cp == u'\'') { + // Repeated quote + length++; + state = STATE_BASE; + } else { + // Quoted code point + length++; + state = STATE_INSIDE_QUOTE; + } + break; + case STATE_INSIDE_QUOTE: + if (cp == u'\'') { + // End of quoted sequence + state = STATE_AFTER_QUOTE; + } else { + // Quoted code point + length++; + } + break; + case STATE_AFTER_QUOTE: + if (cp == u'\'') { + // Double quote inside of quoted sequence + length++; + state = STATE_INSIDE_QUOTE; + } else { + // Unquoted symbol + length++; + } + break; + default: + UPRV_UNREACHABLE_EXIT; + } + + offset += U16_LENGTH(cp); + } + + switch (state) { + case STATE_FIRST_QUOTE: + case STATE_INSIDE_QUOTE: + status = U_ILLEGAL_ARGUMENT_ERROR; + break; + default: + break; + } + + return length; +} + +UnicodeString AffixUtils::escape(const UnicodeString &input) { + AffixPatternState state = STATE_BASE; + int32_t offset = 0; + UnicodeString output; + for (; offset < input.length();) { + UChar32 cp = input.char32At(offset); + + switch (cp) { + case u'\'': + output.append(u"''", -1); + break; + + case u'-': + case u'+': + case u'%': + case u'‰': + case u'¤': + if (state == STATE_BASE) { + output.append(u'\''); + output.append(cp); + state = STATE_INSIDE_QUOTE; + } else { + output.append(cp); + } + break; + + default: + if (state == STATE_INSIDE_QUOTE) { + output.append(u'\''); + output.append(cp); + state = STATE_BASE; + } else { + output.append(cp); + } + break; + } + offset += U16_LENGTH(cp); + } + + if (state == STATE_INSIDE_QUOTE) { + output.append(u'\''); + } + + return output; +} + +Field AffixUtils::getFieldForType(AffixPatternType type) { + switch (type) { + case TYPE_MINUS_SIGN: + return {UFIELD_CATEGORY_NUMBER, UNUM_SIGN_FIELD}; + case TYPE_PLUS_SIGN: + return {UFIELD_CATEGORY_NUMBER, UNUM_SIGN_FIELD}; + case TYPE_APPROXIMATELY_SIGN: + return {UFIELD_CATEGORY_NUMBER, UNUM_APPROXIMATELY_SIGN_FIELD}; + case TYPE_PERCENT: + return {UFIELD_CATEGORY_NUMBER, UNUM_PERCENT_FIELD}; + case TYPE_PERMILLE: + return {UFIELD_CATEGORY_NUMBER, UNUM_PERMILL_FIELD}; + case TYPE_CURRENCY_SINGLE: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + case TYPE_CURRENCY_DOUBLE: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + case TYPE_CURRENCY_TRIPLE: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + case TYPE_CURRENCY_QUAD: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + case TYPE_CURRENCY_QUINT: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + case TYPE_CURRENCY_OVERFLOW: + return {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +int32_t +AffixUtils::unescape(const UnicodeString &affixPattern, FormattedStringBuilder &output, int32_t position, + const SymbolProvider &provider, Field field, UErrorCode &status) { + int32_t length = 0; + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return length; } + if (tag.type == TYPE_CURRENCY_OVERFLOW) { + // Don't go to the provider for this special case + length += output.insertCodePoint( + position + length, + 0xFFFD, + {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}, + status); + } else if (tag.type < 0) { + length += output.insert( + position + length, provider.getSymbol(tag.type), getFieldForType(tag.type), status); + } else { + length += output.insertCodePoint(position + length, tag.codePoint, field, status); + } + } + return length; +} + +int32_t AffixUtils::unescapedCodePointCount(const UnicodeString &affixPattern, + const SymbolProvider &provider, UErrorCode &status) { + int32_t length = 0; + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return length; } + if (tag.type == TYPE_CURRENCY_OVERFLOW) { + length += 1; + } else if (tag.type < 0) { + length += provider.getSymbol(tag.type).length(); + } else { + length += U16_LENGTH(tag.codePoint); + } + } + return length; +} + +bool +AffixUtils::containsType(const UnicodeString &affixPattern, AffixPatternType type, UErrorCode &status) { + if (affixPattern.length() == 0) { + return false; + } + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return false; } + if (tag.type == type) { + return true; + } + } + return false; +} + +bool AffixUtils::hasCurrencySymbols(const UnicodeString &affixPattern, UErrorCode &status) { + if (affixPattern.length() == 0) { + return false; + } + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return false; } + if (tag.type < 0 && getFieldForType(tag.type) == Field(UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD)) { + return true; + } + } + return false; +} + +UnicodeString AffixUtils::replaceType(const UnicodeString &affixPattern, AffixPatternType type, + char16_t replacementChar, UErrorCode &status) { + UnicodeString output(affixPattern); // copy + if (affixPattern.length() == 0) { + return output; + } + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return output; } + if (tag.type == type) { + output.replace(tag.offset - 1, 1, replacementChar); + } + } + return output; +} + +bool AffixUtils::containsOnlySymbolsAndIgnorables(const UnicodeString& affixPattern, + const UnicodeSet& ignorables, UErrorCode& status) { + if (affixPattern.length() == 0) { + return true; + } + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return false; } + if (tag.type == TYPE_CODEPOINT && !ignorables.contains(tag.codePoint)) { + return false; + } + } + return true; +} + +void AffixUtils::iterateWithConsumer(const UnicodeString& affixPattern, TokenConsumer& consumer, + UErrorCode& status) { + if (affixPattern.length() == 0) { + return; + } + AffixTag tag; + while (hasNext(tag, affixPattern)) { + tag = nextToken(tag, affixPattern, status); + if (U_FAILURE(status)) { return; } + consumer.consumeToken(tag.type, tag.codePoint, status); + if (U_FAILURE(status)) { return; } + } +} + +AffixTag AffixUtils::nextToken(AffixTag tag, const UnicodeString &patternString, UErrorCode &status) { + int32_t offset = tag.offset; + int32_t state = tag.state; + for (; offset < patternString.length();) { + UChar32 cp = patternString.char32At(offset); + int32_t count = U16_LENGTH(cp); + + switch (state) { + case STATE_BASE: + switch (cp) { + case u'\'': + state = STATE_FIRST_QUOTE; + offset += count; + // continue to the next code point + break; + case u'-': + return makeTag(offset + count, TYPE_MINUS_SIGN, STATE_BASE, 0); + case u'+': + return makeTag(offset + count, TYPE_PLUS_SIGN, STATE_BASE, 0); + case u'~': + return makeTag(offset + count, TYPE_APPROXIMATELY_SIGN, STATE_BASE, 0); + case u'%': + return makeTag(offset + count, TYPE_PERCENT, STATE_BASE, 0); + case u'‰': + return makeTag(offset + count, TYPE_PERMILLE, STATE_BASE, 0); + case u'¤': + state = STATE_FIRST_CURR; + offset += count; + // continue to the next code point + break; + default: + return makeTag(offset + count, TYPE_CODEPOINT, STATE_BASE, cp); + } + break; + case STATE_FIRST_QUOTE: + if (cp == u'\'') { + return makeTag(offset + count, TYPE_CODEPOINT, STATE_BASE, cp); + } else { + return makeTag(offset + count, TYPE_CODEPOINT, STATE_INSIDE_QUOTE, cp); + } + case STATE_INSIDE_QUOTE: + if (cp == u'\'') { + state = STATE_AFTER_QUOTE; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset + count, TYPE_CODEPOINT, STATE_INSIDE_QUOTE, cp); + } + case STATE_AFTER_QUOTE: + if (cp == u'\'') { + return makeTag(offset + count, TYPE_CODEPOINT, STATE_INSIDE_QUOTE, cp); + } else { + state = STATE_BASE; + // re-evaluate this code point + break; + } + case STATE_FIRST_CURR: + if (cp == u'¤') { + state = STATE_SECOND_CURR; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset, TYPE_CURRENCY_SINGLE, STATE_BASE, 0); + } + case STATE_SECOND_CURR: + if (cp == u'¤') { + state = STATE_THIRD_CURR; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset, TYPE_CURRENCY_DOUBLE, STATE_BASE, 0); + } + case STATE_THIRD_CURR: + if (cp == u'¤') { + state = STATE_FOURTH_CURR; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset, TYPE_CURRENCY_TRIPLE, STATE_BASE, 0); + } + case STATE_FOURTH_CURR: + if (cp == u'¤') { + state = STATE_FIFTH_CURR; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset, TYPE_CURRENCY_QUAD, STATE_BASE, 0); + } + case STATE_FIFTH_CURR: + if (cp == u'¤') { + state = STATE_OVERFLOW_CURR; + offset += count; + // continue to the next code point + break; + } else { + return makeTag(offset, TYPE_CURRENCY_QUINT, STATE_BASE, 0); + } + case STATE_OVERFLOW_CURR: + if (cp == u'¤') { + offset += count; + // continue to the next code point and loop back to this state + break; + } else { + return makeTag(offset, TYPE_CURRENCY_OVERFLOW, STATE_BASE, 0); + } + default: + UPRV_UNREACHABLE_EXIT; + } + } + // End of string + switch (state) { + case STATE_BASE: + // No more tokens in string. + return {-1}; + case STATE_FIRST_QUOTE: + case STATE_INSIDE_QUOTE: + // For consistent behavior with the JDK and ICU 58, set an error here. + status = U_ILLEGAL_ARGUMENT_ERROR; + return {-1}; + case STATE_AFTER_QUOTE: + // No more tokens in string. + return {-1}; + case STATE_FIRST_CURR: + return makeTag(offset, TYPE_CURRENCY_SINGLE, STATE_BASE, 0); + case STATE_SECOND_CURR: + return makeTag(offset, TYPE_CURRENCY_DOUBLE, STATE_BASE, 0); + case STATE_THIRD_CURR: + return makeTag(offset, TYPE_CURRENCY_TRIPLE, STATE_BASE, 0); + case STATE_FOURTH_CURR: + return makeTag(offset, TYPE_CURRENCY_QUAD, STATE_BASE, 0); + case STATE_FIFTH_CURR: + return makeTag(offset, TYPE_CURRENCY_QUINT, STATE_BASE, 0); + case STATE_OVERFLOW_CURR: + return makeTag(offset, TYPE_CURRENCY_OVERFLOW, STATE_BASE, 0); + default: + UPRV_UNREACHABLE_EXIT; + } +} + +bool AffixUtils::hasNext(const AffixTag &tag, const UnicodeString &string) { + // First check for the {-1} and default initializer syntax. + if (tag.offset < 0) { + return false; + } else if (tag.offset == 0) { + return string.length() > 0; + } + // The rest of the fields are safe to use now. + // Special case: the last character in string is an end quote. + if (tag.state == STATE_INSIDE_QUOTE && tag.offset == string.length() - 1 && + string.charAt(tag.offset) == u'\'') { + return false; + } else if (tag.state != STATE_BASE) { + return true; + } else { + return tag.offset < string.length(); + } +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_affixutils.h b/intl/icu/source/i18n/number_affixutils.h new file mode 100644 index 0000000000..5cfde61ffd --- /dev/null +++ b/intl/icu/source/i18n/number_affixutils.h @@ -0,0 +1,244 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_AFFIXUTILS_H__ +#define __NUMBER_AFFIXUTILS_H__ + +#include +#include "number_types.h" +#include "unicode/stringpiece.h" +#include "unicode/unistr.h" +#include "formatted_string_builder.h" +#include "unicode/uniset.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +enum AffixPatternState { + STATE_BASE = 0, + STATE_FIRST_QUOTE = 1, + STATE_INSIDE_QUOTE = 2, + STATE_AFTER_QUOTE = 3, + STATE_FIRST_CURR = 4, + STATE_SECOND_CURR = 5, + STATE_THIRD_CURR = 6, + STATE_FOURTH_CURR = 7, + STATE_FIFTH_CURR = 8, + STATE_OVERFLOW_CURR = 9 +}; + +// enum AffixPatternType defined in internals.h + +struct AffixTag { + int32_t offset; + UChar32 codePoint; + AffixPatternState state; + AffixPatternType type; + + AffixTag() + : offset(0), state(STATE_BASE) {} + + AffixTag(int32_t offset) + : offset(offset) {} + + AffixTag(int32_t offset, UChar32 codePoint, AffixPatternState state, AffixPatternType type) + : offset(offset), codePoint(codePoint), state(state), type(type) {} +}; + +class TokenConsumer { + public: + virtual ~TokenConsumer(); + + virtual void consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) = 0; +}; + +// Exported as U_I18N_API because it is a base class for other exported types +class U_I18N_API SymbolProvider { + public: + virtual ~SymbolProvider(); + + // TODO: Could this be more efficient if it returned by reference? + virtual UnicodeString getSymbol(AffixPatternType type) const = 0; +}; + +/** + * Performs manipulations on affix patterns: the prefix and suffix strings associated with a decimal + * format pattern. For example: + * + * + * + * + * + * + * + *
    Affix PatternExample Unescaped (Formatted) String
    abcabc
    ab-ab−
    ab'-'ab-
    ab''ab'
    + * + * To manually iterate over tokens in a literal string, use the following pattern, which is designed + * to be efficient. + * + * 
    + * long tag = 0L;
+ * while (AffixPatternUtils.hasNext(tag, patternString)) {
+ *   tag = AffixPatternUtils.nextToken(tag, patternString);
+ *   int typeOrCp = AffixPatternUtils.getTypeOrCp(tag);
+ *   switch (typeOrCp) {
+ *     case AffixPatternUtils.TYPE_MINUS_SIGN:
+ *       // Current token is a minus sign.
+ *       break;
+ *     case AffixPatternUtils.TYPE_PLUS_SIGN:
+ *       // Current token is a plus sign.
+ *       break;
+ *     case AffixPatternUtils.TYPE_PERCENT:
+ *       // Current token is a percent sign.
+ *       break;
+ *     // ... other types ...
+ *     default:
+ *       // Current token is an arbitrary code point.
+ *       // The variable typeOrCp is the code point.
+ *       break;
+ *   }
+ * }
+ *
    + */ +class U_I18N_API AffixUtils { + + public: + + /** + * Estimates the number of code points present in an unescaped version of the affix pattern string + * (one that would be returned by {@link #unescape}), assuming that all interpolated symbols + * consume one code point and that currencies consume as many code points as their symbol width. + * Used for computing padding width. + * + * @param patternString The original string whose width will be estimated. + * @return The length of the unescaped string. + */ + static int32_t estimateLength(const UnicodeString& patternString, UErrorCode& status); + + /** + * Takes a string and escapes (quotes) characters that have special meaning in the affix pattern + * syntax. This function does not reverse-lookup symbols. + * + *

    Example input: "-$x"; example output: "'-'$x" + * + * @param input The string to be escaped. + * @return The resulting UnicodeString. + */ + static UnicodeString escape(const UnicodeString& input); + + static Field getFieldForType(AffixPatternType type); + + /** + * Executes the unescape state machine. Replaces the unquoted characters "-", "+", "%", "‰", and + * "¤" with the corresponding symbols provided by the {@link SymbolProvider}, and inserts the + * result into the FormattedStringBuilder at the requested location. + * + *

    Example input: "'-'¤x"; example output: "-$x" + * + * @param affixPattern The original string to be unescaped. + * @param output The FormattedStringBuilder to mutate with the result. + * @param position The index into the FormattedStringBuilder to insert the string. + * @param provider An object to generate locale symbols. + */ + static int32_t unescape(const UnicodeString& affixPattern, FormattedStringBuilder& output, + int32_t position, const SymbolProvider& provider, Field field, + UErrorCode& status); + + /** + * Sames as {@link #unescape}, but only calculates the code point count. More efficient than {@link #unescape} + * if you only need the length but not the string itself. + * + * @param affixPattern The original string to be unescaped. + * @param provider An object to generate locale symbols. + * @return The same return value as if you called {@link #unescape}. + */ + static int32_t unescapedCodePointCount(const UnicodeString& affixPattern, + const SymbolProvider& provider, UErrorCode& status); + + /** + * Checks whether the given affix pattern contains at least one token of the given type, which is + * one of the constants "TYPE_" in {@link AffixPatternUtils}. + * + * @param affixPattern The affix pattern to check. + * @param type The token type. + * @return true if the affix pattern contains the given token type; false otherwise. + */ + static bool containsType(const UnicodeString& affixPattern, AffixPatternType type, UErrorCode& status); + + /** + * Checks whether the specified affix pattern has any unquoted currency symbols ("¤"). + * + * @param affixPattern The string to check for currency symbols. + * @return true if the literal has at least one unquoted currency symbol; false otherwise. + */ + static bool hasCurrencySymbols(const UnicodeString& affixPattern, UErrorCode& status); + + /** + * Replaces all occurrences of tokens with the given type with the given replacement char. + * + * @param affixPattern The source affix pattern (does not get modified). + * @param type The token type. + * @param replacementChar The char to substitute in place of chars of the given token type. + * @return A string containing the new affix pattern. + */ + static UnicodeString replaceType(const UnicodeString& affixPattern, AffixPatternType type, + char16_t replacementChar, UErrorCode& status); + + /** + * Returns whether the given affix pattern contains only symbols and ignorables as defined by the + * given ignorables set. + */ + static bool containsOnlySymbolsAndIgnorables(const UnicodeString& affixPattern, + const UnicodeSet& ignorables, UErrorCode& status); + + /** + * Iterates over the affix pattern, calling the TokenConsumer for each token. + */ + static void iterateWithConsumer(const UnicodeString& affixPattern, TokenConsumer& consumer, + UErrorCode& status); + + /** + * Returns the next token from the affix pattern. + * + * @param tag A bitmask used for keeping track of state from token to token. The initial value + * should be 0L. + * @param patternString The affix pattern. + * @return The bitmask tag to pass to the next call of this method to retrieve the following token + * (never negative), or -1 if there were no more tokens in the affix pattern. + * @see #hasNext + */ + static AffixTag nextToken(AffixTag tag, const UnicodeString& patternString, UErrorCode& status); + + /** + * Returns whether the affix pattern string has any more tokens to be retrieved from a call to + * {@link #nextToken}. + * + * @param tag The bitmask tag of the previous token, as returned by {@link #nextToken}. + * @param string The affix pattern. + * @return true if there are more tokens to consume; false otherwise. + */ + static bool hasNext(const AffixTag& tag, const UnicodeString& string); + + private: + /** + * Encodes the given values into a tag struct. + * The order of the arguments is consistent with Java, but the order of the stored + * fields is not necessarily the same. + */ + static inline AffixTag makeTag(int32_t offset, AffixPatternType type, AffixPatternState state, + UChar32 cp) { + return {offset, cp, state, type}; + } +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_AFFIXUTILS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_asformat.cpp b/intl/icu/source/i18n/number_asformat.cpp new file mode 100644 index 0000000000..8f2314d689 --- /dev/null +++ b/intl/icu/source/i18n/number_asformat.cpp @@ -0,0 +1,117 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include +#include +#include "number_asformat.h" +#include "number_types.h" +#include "number_utils.h" +#include "fphdlimp.h" +#include "number_utypes.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(LocalizedNumberFormatterAsFormat) + +LocalizedNumberFormatterAsFormat::LocalizedNumberFormatterAsFormat( + const LocalizedNumberFormatter& formatter, const Locale& locale) + : fFormatter(formatter), fLocale(locale) { + const char* localeName = locale.getName(); + setLocaleIDs(localeName, localeName); +} + +LocalizedNumberFormatterAsFormat::~LocalizedNumberFormatterAsFormat() = default; + +bool LocalizedNumberFormatterAsFormat::operator==(const Format& other) const { + auto* _other = dynamic_cast(&other); + if (_other == nullptr) { + return false; + } + // TODO: Change this to use LocalizedNumberFormatter::operator== if it is ever proposed. + // This implementation is fine, but not particularly efficient. + UErrorCode localStatus = U_ZERO_ERROR; + return fFormatter.toSkeleton(localStatus) == _other->fFormatter.toSkeleton(localStatus); +} + +LocalizedNumberFormatterAsFormat* LocalizedNumberFormatterAsFormat::clone() const { + return new LocalizedNumberFormatterAsFormat(*this); +} + +UnicodeString& LocalizedNumberFormatterAsFormat::format(const Formattable& obj, UnicodeString& appendTo, + FieldPosition& pos, UErrorCode& status) const { + if (U_FAILURE(status)) { return appendTo; } + UFormattedNumberData data; + obj.populateDecimalQuantity(data.quantity, status); + if (U_FAILURE(status)) { + return appendTo; + } + fFormatter.formatImpl(&data, status); + if (U_FAILURE(status)) { + return appendTo; + } + // always return first occurrence: + pos.setBeginIndex(0); + pos.setEndIndex(0); + bool found = data.nextFieldPosition(pos, status); + if (found && appendTo.length() != 0) { + pos.setBeginIndex(pos.getBeginIndex() + appendTo.length()); + pos.setEndIndex(pos.getEndIndex() + appendTo.length()); + } + appendTo.append(data.toTempString(status)); + return appendTo; +} + +UnicodeString& LocalizedNumberFormatterAsFormat::format(const Formattable& obj, UnicodeString& appendTo, + FieldPositionIterator* posIter, + UErrorCode& status) const { + if (U_FAILURE(status)) { return appendTo; } + UFormattedNumberData data; + obj.populateDecimalQuantity(data.quantity, status); + if (U_FAILURE(status)) { + return appendTo; + } + fFormatter.formatImpl(&data, status); + if (U_FAILURE(status)) { + return appendTo; + } + appendTo.append(data.toTempString(status)); + if (posIter != nullptr) { + FieldPositionIteratorHandler fpih(posIter, status); + data.getAllFieldPositions(fpih, status); + } + return appendTo; +} + +void LocalizedNumberFormatterAsFormat::parseObject(const UnicodeString&, Formattable&, + ParsePosition& parse_pos) const { + // Not supported. + parse_pos.setErrorIndex(0); +} + +const LocalizedNumberFormatter& LocalizedNumberFormatterAsFormat::getNumberFormatter() const { + return fFormatter; +} + + +// Definitions of public API methods (put here for dependency disentanglement) + +Format* LocalizedNumberFormatter::toFormat(UErrorCode& status) const { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer retval( + new LocalizedNumberFormatterAsFormat(*this, fMacros.locale), status); + return retval.orphan(); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_asformat.h b/intl/icu/source/i18n/number_asformat.h new file mode 100644 index 0000000000..f921b42942 --- /dev/null +++ b/intl/icu/source/i18n/number_asformat.h @@ -0,0 +1,106 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_ASFORMAT_H__ +#define __NUMBER_ASFORMAT_H__ + +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "number_scientific.h" +#include "number_patternstring.h" +#include "number_modifiers.h" +#include "number_multiplier.h" +#include "number_roundingutils.h" +#include "decNumber.h" +#include "charstr.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +/** + * A wrapper around LocalizedNumberFormatter implementing the Format interface, enabling improved + * compatibility with other APIs. + * + * @see NumberFormatter + */ +class U_I18N_API LocalizedNumberFormatterAsFormat : public Format { + public: + LocalizedNumberFormatterAsFormat(const LocalizedNumberFormatter& formatter, const Locale& locale); + + /** + * Destructor. + */ + ~LocalizedNumberFormatterAsFormat() override; + + /** + * Equals operator. + */ + bool operator==(const Format& other) const override; + + /** + * Creates a copy of this object. + */ + LocalizedNumberFormatterAsFormat* clone() const override; + + /** + * Formats a Number using the wrapped LocalizedNumberFormatter. The provided formattable must be a + * number type. + */ + UnicodeString& format(const Formattable& obj, UnicodeString& appendTo, FieldPosition& pos, + UErrorCode& status) const override; + + /** + * Formats a Number using the wrapped LocalizedNumberFormatter. The provided formattable must be a + * number type. + */ + UnicodeString& format(const Formattable& obj, UnicodeString& appendTo, FieldPositionIterator* posIter, + UErrorCode& status) const override; + + /** + * Not supported: sets an error index and returns. + */ + void parseObject(const UnicodeString& source, Formattable& result, + ParsePosition& parse_pos) const override; + + /** + * Gets the LocalizedNumberFormatter that this wrapper class uses to format numbers. + * + * For maximum efficiency, this function returns by const reference. You must copy the return value + * into a local variable if you want to use it beyond the lifetime of the current object: + * + * 

    +     * LocalizedNumberFormatter localFormatter = fmt->getNumberFormatter();
+     *
    + * + * You can however use the return value directly when chaining: + * + * 
    +     * FormattedNumber result = fmt->getNumberFormatter().formatDouble(514.23, status);
+     *
    + * + * @return The unwrapped LocalizedNumberFormatter. + */ + const LocalizedNumberFormatter& getNumberFormatter() const; + + UClassID getDynamicClassID() const override; + static UClassID U_EXPORT2 getStaticClassID(); + + private: + LocalizedNumberFormatter fFormatter; + + // Even though the locale is inside the LocalizedNumberFormatter, we have to keep it here, too, because + // LocalizedNumberFormatter doesn't have a getLocale() method, and ICU-TC didn't want to add one. + Locale fLocale; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif // __NUMBER_ASFORMAT_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_capi.cpp b/intl/icu/source/i18n/number_capi.cpp new file mode 100644 index 0000000000..abada9ad86 --- /dev/null +++ b/intl/icu/source/i18n/number_capi.cpp @@ -0,0 +1,430 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "fphdlimp.h" +#include "number_utypes.h" +#include "numparse_types.h" +#include "formattedval_impl.h" +#include "number_decnum.h" +#include "unicode/numberformatter.h" +#include "unicode/unumberformatter.h" +#include "unicode/simplenumberformatter.h" +#include "unicode/usimplenumberformatter.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { + +/** + * Implementation class for UNumberFormatter. Wraps a LocalizedNumberFormatter. + */ +struct UNumberFormatterData : public UMemory, + // Magic number as ASCII == "NFR" (NumberFormatteR) + public IcuCApiHelper { + LocalizedNumberFormatter fFormatter; +}; + +/** + * Implementation class for USimpleNumber. Wraps a SimpleNumberFormatter. + */ +struct USimpleNumberData : public UMemory, + // Magic number as ASCII == "SNM" (SimpleNuMber) + public IcuCApiHelper { + SimpleNumber fNumber; +}; + +/** + * Implementation class for USimpleNumberFormatter. Wraps a SimpleNumberFormatter. + */ +struct USimpleNumberFormatterData : public UMemory, + // Magic number as ASCII == "SNF" (SimpleNumberFormatter) + public IcuCApiHelper { + SimpleNumberFormatter fFormatter; +}; + +struct UFormattedNumberImpl; + +// Magic number as ASCII == "FDN" (FormatteDNumber) +typedef IcuCApiHelper UFormattedNumberApiHelper; + +struct UFormattedNumberImpl : public UFormattedValueImpl, public UFormattedNumberApiHelper { + UFormattedNumberImpl(); + ~UFormattedNumberImpl(); + + FormattedNumber fImpl; + UFormattedNumberData fData; + + void setTo(FormattedNumber value); +}; + +UFormattedNumberImpl::UFormattedNumberImpl() + : fImpl(&fData) { + fFormattedValue = &fImpl; +} + +UFormattedNumberImpl::~UFormattedNumberImpl() { + // Disown the data from fImpl so it doesn't get deleted twice + fImpl.fData = nullptr; +} + +void UFormattedNumberImpl::setTo(FormattedNumber value) { + fData = std::move(*value.fData); +} + +} +} +U_NAMESPACE_END + + +UPRV_FORMATTED_VALUE_CAPI_NO_IMPLTYPE_AUTO_IMPL( + UFormattedNumber, + UFormattedNumberImpl, + UFormattedNumberApiHelper, + unumf) + + +const DecimalQuantity* icu::number::impl::validateUFormattedNumberToDecimalQuantity( + const UFormattedNumber* uresult, UErrorCode& status) { + auto* result = UFormattedNumberApiHelper::validate(uresult, status); + if (U_FAILURE(status)) { + return nullptr; + } + return &result->fData.quantity; +} + + + +U_CAPI UNumberFormatter* U_EXPORT2 +unumf_openForSkeletonAndLocale(const char16_t* skeleton, int32_t skeletonLen, const char* locale, + UErrorCode* ec) { + auto* impl = new UNumberFormatterData(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + // Readonly-alias constructor (first argument is whether we are NUL-terminated) + UnicodeString skeletonString(skeletonLen == -1, skeleton, skeletonLen); + impl->fFormatter = NumberFormatter::forSkeleton(skeletonString, *ec).locale(locale); + return impl->exportForC(); +} + +U_CAPI UNumberFormatter* U_EXPORT2 +unumf_openForSkeletonAndLocaleWithError(const char16_t* skeleton, int32_t skeletonLen, const char* locale, + UParseError* perror, UErrorCode* ec) { + auto* impl = new UNumberFormatterData(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + // Readonly-alias constructor (first argument is whether we are NUL-terminated) + UnicodeString skeletonString(skeletonLen == -1, skeleton, skeletonLen); + UParseError tempParseError; + impl->fFormatter = NumberFormatter::forSkeleton(skeletonString, (perror == nullptr) ? tempParseError : *perror, *ec).locale(locale); + return impl->exportForC(); +} + +U_CAPI void U_EXPORT2 +unumf_formatInt(const UNumberFormatter* uformatter, int64_t value, UFormattedNumber* uresult, + UErrorCode* ec) { + const UNumberFormatterData* formatter = UNumberFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + result->fData.resetString(); + result->fData.quantity.clear(); + result->fData.quantity.setToLong(value); + formatter->fFormatter.formatImpl(&result->fData, *ec); +} + +U_CAPI void U_EXPORT2 +unumf_formatDouble(const UNumberFormatter* uformatter, double value, UFormattedNumber* uresult, + UErrorCode* ec) { + const UNumberFormatterData* formatter = UNumberFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + result->fData.resetString(); + result->fData.quantity.clear(); + result->fData.quantity.setToDouble(value); + formatter->fFormatter.formatImpl(&result->fData, *ec); +} + +U_CAPI void U_EXPORT2 +unumf_formatDecimal(const UNumberFormatter* uformatter, const char* value, int32_t valueLen, + UFormattedNumber* uresult, UErrorCode* ec) { + const UNumberFormatterData* formatter = UNumberFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + result->fData.resetString(); + result->fData.quantity.clear(); + result->fData.quantity.setToDecNumber({value, valueLen}, *ec); + if (U_FAILURE(*ec)) { return; } + formatter->fFormatter.formatImpl(&result->fData, *ec); +} + +U_CAPI int32_t U_EXPORT2 +unumf_resultToString(const UFormattedNumber* uresult, char16_t* buffer, int32_t bufferCapacity, + UErrorCode* ec) { + const auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return 0; } + + if (buffer == nullptr ? bufferCapacity != 0 : bufferCapacity < 0) { + *ec = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + return result->fData.toTempString(*ec).extract(buffer, bufferCapacity, *ec); +} + +U_CAPI UBool U_EXPORT2 +unumf_resultNextFieldPosition(const UFormattedNumber* uresult, UFieldPosition* ufpos, UErrorCode* ec) { + const auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return false; } + + if (ufpos == nullptr) { + *ec = U_ILLEGAL_ARGUMENT_ERROR; + return false; + } + + FieldPosition fp; + fp.setField(ufpos->field); + fp.setBeginIndex(ufpos->beginIndex); + fp.setEndIndex(ufpos->endIndex); + bool retval = result->fData.nextFieldPosition(fp, *ec); + ufpos->beginIndex = fp.getBeginIndex(); + ufpos->endIndex = fp.getEndIndex(); + // NOTE: MSVC sometimes complains when implicitly converting between bool and UBool + return retval ? true : false; +} + +U_CAPI void U_EXPORT2 +unumf_resultGetAllFieldPositions(const UFormattedNumber* uresult, UFieldPositionIterator* ufpositer, + UErrorCode* ec) { + const auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + if (ufpositer == nullptr) { + *ec = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + auto* fpi = reinterpret_cast(ufpositer); + FieldPositionIteratorHandler fpih(fpi, *ec); + result->fData.getAllFieldPositions(fpih, *ec); +} + +U_CAPI int32_t U_EXPORT2 +unumf_resultToDecimalNumber( + const UFormattedNumber* uresult, + char* dest, + int32_t destCapacity, + UErrorCode* ec) { + const auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + DecNum decnum; + return result->fData.quantity + .toDecNum(decnum, *ec) + .toCharString(*ec) + .extract(dest, destCapacity, *ec); +} + +U_CAPI void U_EXPORT2 +unumf_close(UNumberFormatter* f) { + UErrorCode localStatus = U_ZERO_ERROR; + const UNumberFormatterData* impl = UNumberFormatterData::validate(f, localStatus); + delete impl; +} + + +///// SIMPLE NUMBER FORMATTER ///// + +U_CAPI USimpleNumber* U_EXPORT2 +usnum_openForInt64(int64_t value, UErrorCode* ec) { + auto* number = new USimpleNumberData(); + if (number == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + number->fNumber = SimpleNumber::forInt64(value, *ec); + return number->exportForC(); +} + +U_CAPI void U_EXPORT2 +usnum_setToInt64(USimpleNumber* unumber, int64_t value, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber = SimpleNumber::forInt64(value, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_multiplyByPowerOfTen(USimpleNumber* unumber, int32_t power, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.multiplyByPowerOfTen(power, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_roundTo(USimpleNumber* unumber, int32_t position, UNumberFormatRoundingMode roundingMode, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.roundTo(position, roundingMode, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_setMinimumIntegerDigits(USimpleNumber* unumber, int32_t minimumIntegerDigits, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.setMinimumIntegerDigits(minimumIntegerDigits, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_setMinimumFractionDigits(USimpleNumber* unumber, int32_t minimumFractionDigits, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.setMinimumFractionDigits(minimumFractionDigits, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_truncateStart(USimpleNumber* unumber, int32_t maximumIntegerDigits, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.truncateStart(maximumIntegerDigits, *ec); +} + +U_CAPI void U_EXPORT2 +usnum_setSign(USimpleNumber* unumber, USimpleNumberSign sign, UErrorCode* ec) { + auto* number = USimpleNumberData::validate(unumber, *ec); + if (U_FAILURE(*ec)) { + return; + } + number->fNumber.setSign(sign, *ec); +} + +U_CAPI USimpleNumberFormatter* U_EXPORT2 +usnumf_openForLocale(const char* locale, UErrorCode* ec) { + auto* impl = new USimpleNumberFormatterData(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + impl->fFormatter = SimpleNumberFormatter::forLocale(locale, *ec); + return impl->exportForC(); +} + +U_CAPI USimpleNumberFormatter* U_EXPORT2 +usnumf_openForLocaleAndGroupingStrategy( + const char* locale, UNumberGroupingStrategy groupingStrategy, UErrorCode* ec) { + auto* impl = new USimpleNumberFormatterData(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + impl->fFormatter = SimpleNumberFormatter::forLocaleAndGroupingStrategy(locale, groupingStrategy, *ec); + return impl->exportForC(); +} + +U_CAPI void U_EXPORT2 +usnumf_format( + const USimpleNumberFormatter* uformatter, + USimpleNumber* unumber, + UFormattedNumber* uresult, + UErrorCode* ec) { + auto* formatter = USimpleNumberFormatterData::validate(uformatter, *ec); + auto* number = USimpleNumberData::validate(unumber, *ec); + auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return; + } + auto localResult = formatter->fFormatter.format(std::move(number->fNumber), *ec); + if (U_FAILURE(*ec)) { + return; + } + result->setTo(std::move(localResult)); +} + +U_CAPI void U_EXPORT2 +usnumf_formatInt64( + const USimpleNumberFormatter* uformatter, + int64_t value, + UFormattedNumber* uresult, + UErrorCode* ec) { + auto* formatter = USimpleNumberFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return; + } + auto localResult = formatter->fFormatter.formatInt64(value, *ec); + result->setTo(std::move(localResult)); +} + +U_CAPI void U_EXPORT2 +usnum_close(USimpleNumber* unumber) { + UErrorCode localStatus = U_ZERO_ERROR; + const USimpleNumberData* impl = USimpleNumberData::validate(unumber, localStatus); + delete impl; +} + +U_CAPI void U_EXPORT2 +usnumf_close(USimpleNumberFormatter* uformatter) { + UErrorCode localStatus = U_ZERO_ERROR; + const USimpleNumberFormatterData* impl = USimpleNumberFormatterData::validate(uformatter, localStatus); + delete impl; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/intl/icu/source/i18n/number_compact.cpp b/intl/icu/source/i18n/number_compact.cpp new file mode 100644 index 0000000000..2cfa65a031 --- /dev/null +++ b/intl/icu/source/i18n/number_compact.cpp @@ -0,0 +1,353 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/ustring.h" +#include "unicode/ures.h" +#include "cstring.h" +#include "charstr.h" +#include "resource.h" +#include "number_compact.h" +#include "number_microprops.h" +#include "uresimp.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +// A dummy object used when a "0" compact decimal entry is encountered. This is necessary +// in order to prevent falling back to root. Object equality ("==") is intended. +const char16_t *USE_FALLBACK = u""; + +/** Produces a string like "NumberElements/latn/patternsShort/decimalFormat". */ +void getResourceBundleKey(const char *nsName, CompactStyle compactStyle, CompactType compactType, + CharString &sb, UErrorCode &status) { + sb.clear(); + sb.append("NumberElements/", status); + sb.append(nsName, status); + sb.append(compactStyle == CompactStyle::UNUM_SHORT ? "/patternsShort" : "/patternsLong", status); + sb.append(compactType == CompactType::TYPE_DECIMAL ? "/decimalFormat" : "/currencyFormat", status); +} + +int32_t getIndex(int32_t magnitude, StandardPlural::Form plural) { + return magnitude * StandardPlural::COUNT + plural; +} + +int32_t countZeros(const char16_t *patternString, int32_t patternLength) { + // NOTE: This strategy for computing the number of zeros is a hack for efficiency. + // It could break if there are any 0s that aren't part of the main pattern. + int32_t numZeros = 0; + for (int32_t i = 0; i < patternLength; i++) { + if (patternString[i] == u'0') { + numZeros++; + } else if (numZeros > 0) { + break; // zeros should always be contiguous + } + } + return numZeros; +} + +} // namespace + +// NOTE: patterns and multipliers both get zero-initialized. +CompactData::CompactData() : patterns(), multipliers(), largestMagnitude(0), isEmpty(true) { +} + +void CompactData::populate(const Locale &locale, const char *nsName, CompactStyle compactStyle, + CompactType compactType, UErrorCode &status) { + CompactDataSink sink(*this); + LocalUResourceBundlePointer rb(ures_open(nullptr, locale.getName(), &status)); + if (U_FAILURE(status)) { return; } + + bool nsIsLatn = strcmp(nsName, "latn") == 0; + bool compactIsShort = compactStyle == CompactStyle::UNUM_SHORT; + + // Fall back to latn numbering system and/or short compact style. + CharString resourceKey; + getResourceBundleKey(nsName, compactStyle, compactType, resourceKey, status); + UErrorCode localStatus = U_ZERO_ERROR; + ures_getAllItemsWithFallback(rb.getAlias(), resourceKey.data(), sink, localStatus); + if (isEmpty && !nsIsLatn) { + getResourceBundleKey("latn", compactStyle, compactType, resourceKey, status); + localStatus = U_ZERO_ERROR; + ures_getAllItemsWithFallback(rb.getAlias(), resourceKey.data(), sink, localStatus); + } + if (isEmpty && !compactIsShort) { + getResourceBundleKey(nsName, CompactStyle::UNUM_SHORT, compactType, resourceKey, status); + localStatus = U_ZERO_ERROR; + ures_getAllItemsWithFallback(rb.getAlias(), resourceKey.data(), sink, localStatus); + } + if (isEmpty && !nsIsLatn && !compactIsShort) { + getResourceBundleKey("latn", CompactStyle::UNUM_SHORT, compactType, resourceKey, status); + localStatus = U_ZERO_ERROR; + ures_getAllItemsWithFallback(rb.getAlias(), resourceKey.data(), sink, localStatus); + } + + // The last fallback should be guaranteed to return data. + if (isEmpty) { + status = U_INTERNAL_PROGRAM_ERROR; + } +} + +int32_t CompactData::getMultiplier(int32_t magnitude) const { + if (magnitude < 0) { + return 0; + } + if (magnitude > largestMagnitude) { + magnitude = largestMagnitude; + } + return multipliers[magnitude]; +} + +const char16_t *CompactData::getPattern( + int32_t magnitude, + const PluralRules *rules, + const DecimalQuantity &dq) const { + if (magnitude < 0) { + return nullptr; + } + if (magnitude > largestMagnitude) { + magnitude = largestMagnitude; + } + const char16_t *patternString = nullptr; + if (dq.hasIntegerValue()) { + int64_t i = dq.toLong(true); + if (i == 0) { + patternString = patterns[getIndex(magnitude, StandardPlural::Form::EQ_0)]; + } else if (i == 1) { + patternString = patterns[getIndex(magnitude, StandardPlural::Form::EQ_1)]; + } + if (patternString != nullptr) { + return patternString; + } + } + StandardPlural::Form plural = utils::getStandardPlural(rules, dq); + patternString = patterns[getIndex(magnitude, plural)]; + if (patternString == nullptr && plural != StandardPlural::OTHER) { + // Fall back to "other" plural variant + patternString = patterns[getIndex(magnitude, StandardPlural::OTHER)]; + } + if (patternString == USE_FALLBACK) { // == is intended + // Return null if USE_FALLBACK is present + patternString = nullptr; + } + return patternString; +} + +void CompactData::getUniquePatterns(UVector &output, UErrorCode &status) const { + U_ASSERT(output.isEmpty()); + // NOTE: In C++, this is done more manually with a UVector. + // In Java, we can take advantage of JDK HashSet. + for (auto pattern : patterns) { + if (pattern == nullptr || pattern == USE_FALLBACK) { + continue; + } + + // Insert pattern into the UVector if the UVector does not already contain the pattern. + // Search the UVector from the end since identical patterns are likely to be adjacent. + for (int32_t i = output.size() - 1; i >= 0; i--) { + if (u_strcmp(pattern, static_cast(output[i])) == 0) { + goto continue_outer; + } + } + + // The string was not found; add it to the UVector. + // Note: must cast off const from pattern to store it in a UVector, which expects (void *) + output.addElement(const_cast(pattern), status); + + continue_outer: + continue; + } +} + +void CompactData::CompactDataSink::put(const char *key, ResourceValue &value, UBool /*noFallback*/, + UErrorCode &status) { + // traverse into the table of powers of ten + ResourceTable powersOfTenTable = value.getTable(status); + if (U_FAILURE(status)) { return; } + for (int i3 = 0; powersOfTenTable.getKeyAndValue(i3, key, value); ++i3) { + + // Assumes that the keys are always of the form "10000" where the magnitude is the + // length of the key minus one. We only support magnitudes less than COMPACT_MAX_DIGITS; + // ignore entries that have greater magnitude. + auto magnitude = static_cast (strlen(key) - 1); + U_ASSERT(magnitude < COMPACT_MAX_DIGITS); // debug assert + if (magnitude >= COMPACT_MAX_DIGITS) { // skip in production + continue; + } + int8_t multiplier = data.multipliers[magnitude]; + + // Iterate over the plural variants ("one", "other", etc) + ResourceTable pluralVariantsTable = value.getTable(status); + if (U_FAILURE(status)) { return; } + for (int i4 = 0; pluralVariantsTable.getKeyAndValue(i4, key, value); ++i4) { + // Skip this magnitude/plural if we already have it from a child locale. + // Note: This also skips USE_FALLBACK entries. + StandardPlural::Form plural = StandardPlural::fromString(key, status); + if (U_FAILURE(status)) { return; } + if (data.patterns[getIndex(magnitude, plural)] != nullptr) { + continue; + } + + // The value "0" means that we need to use the default pattern and not fall back + // to parent locales. Example locale where this is relevant: 'it'. + int32_t patternLength; + const char16_t *patternString = value.getString(patternLength, status); + if (U_FAILURE(status)) { return; } + if (u_strcmp(patternString, u"0") == 0) { + patternString = USE_FALLBACK; + patternLength = 0; + } + + // Save the pattern string. We will parse it lazily. + data.patterns[getIndex(magnitude, plural)] = patternString; + + // If necessary, compute the multiplier: the difference between the magnitude + // and the number of zeros in the pattern. + if (multiplier == 0) { + int32_t numZeros = countZeros(patternString, patternLength); + if (numZeros > 0) { // numZeros==0 in certain cases, like Somali "Kun" + multiplier = static_cast (numZeros - magnitude - 1); + } + } + } + + // Save the multiplier. + if (data.multipliers[magnitude] == 0) { + data.multipliers[magnitude] = multiplier; + if (magnitude > data.largestMagnitude) { + data.largestMagnitude = magnitude; + } + data.isEmpty = false; + } else { + U_ASSERT(data.multipliers[magnitude] == multiplier); + } + } +} + +/////////////////////////////////////////////////////////// +/// END OF CompactData.java; BEGIN CompactNotation.java /// +/////////////////////////////////////////////////////////// + +CompactHandler::CompactHandler( + CompactStyle compactStyle, + const Locale &locale, + const char *nsName, + CompactType compactType, + const PluralRules *rules, + MutablePatternModifier *buildReference, + bool safe, + const MicroPropsGenerator *parent, + UErrorCode &status) + : rules(rules), parent(parent), safe(safe) { + data.populate(locale, nsName, compactStyle, compactType, status); + if (safe) { + // Safe code path + precomputeAllModifiers(*buildReference, status); + } else { + // Unsafe code path + // Store the MutablePatternModifier reference. + unsafePatternModifier = buildReference; + } +} + +CompactHandler::~CompactHandler() { + for (int32_t i = 0; i < precomputedModsLength; i++) { + delete precomputedMods[i].mod; + } +} + +void CompactHandler::precomputeAllModifiers(MutablePatternModifier &buildReference, UErrorCode &status) { + if (U_FAILURE(status)) { return; } + + // Initial capacity of 12 for 0K, 00K, 000K, ...M, ...B, and ...T + UVector allPatterns(12, status); + if (U_FAILURE(status)) { return; } + data.getUniquePatterns(allPatterns, status); + if (U_FAILURE(status)) { return; } + + // C++ only: ensure that precomputedMods has room. + precomputedModsLength = allPatterns.size(); + if (precomputedMods.getCapacity() < precomputedModsLength) { + precomputedMods.resize(allPatterns.size(), status); + if (U_FAILURE(status)) { return; } + } + + for (int32_t i = 0; i < precomputedModsLength; i++) { + auto patternString = static_cast(allPatterns[i]); + UnicodeString hello(patternString); + CompactModInfo &info = precomputedMods[i]; + ParsedPatternInfo patternInfo; + PatternParser::parseToPatternInfo(UnicodeString(patternString), patternInfo, status); + if (U_FAILURE(status)) { return; } + buildReference.setPatternInfo(&patternInfo, {UFIELD_CATEGORY_NUMBER, UNUM_COMPACT_FIELD}); + info.mod = buildReference.createImmutable(status); + if (U_FAILURE(status)) { return; } + info.patternString = patternString; + } +} + +void CompactHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + parent->processQuantity(quantity, micros, status); + if (U_FAILURE(status)) { return; } + + // Treat zero, NaN, and infinity as if they had magnitude 0 + int32_t magnitude; + int32_t multiplier = 0; + if (quantity.isZeroish()) { + magnitude = 0; + micros.rounder.apply(quantity, status); + } else { + // TODO: Revisit chooseMultiplierAndApply + multiplier = micros.rounder.chooseMultiplierAndApply(quantity, data, status); + magnitude = quantity.isZeroish() ? 0 : quantity.getMagnitude(); + magnitude -= multiplier; + } + + const char16_t *patternString = data.getPattern(magnitude, rules, quantity); + if (patternString == nullptr) { + // Use the default (non-compact) modifier. + // No need to take any action. + } else if (safe) { + // Safe code path. + // Java uses a hash set here for O(1) lookup. C++ uses a linear search. + // TODO: Benchmark this and maybe change to a binary search or hash table. + int32_t i = 0; + for (; i < precomputedModsLength; i++) { + const CompactModInfo &info = precomputedMods[i]; + if (u_strcmp(patternString, info.patternString) == 0) { + info.mod->applyToMicros(micros, quantity, status); + break; + } + } + // It should be guaranteed that we found the entry. + U_ASSERT(i < precomputedModsLength); + } else { + // Unsafe code path. + // Overwrite the PatternInfo in the existing modMiddle. + // C++ Note: Use unsafePatternInfo for proper lifecycle. + ParsedPatternInfo &patternInfo = const_cast(this)->unsafePatternInfo; + PatternParser::parseToPatternInfo(UnicodeString(patternString), patternInfo, status); + unsafePatternModifier->setPatternInfo( + &unsafePatternInfo, + {UFIELD_CATEGORY_NUMBER, UNUM_COMPACT_FIELD}); + unsafePatternModifier->setNumberProperties(quantity.signum(), StandardPlural::Form::COUNT); + micros.modMiddle = unsafePatternModifier; + } + + // Change the exponent only after we select appropriate plural form + // for formatting purposes so that we preserve expected formatted + // string behavior. + quantity.adjustExponent(-1 * multiplier); + + // We already performed rounding. Do not perform it again. + micros.rounder = RoundingImpl::passThrough(); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_compact.h b/intl/icu/source/i18n/number_compact.h new file mode 100644 index 0000000000..aee1df7452 --- /dev/null +++ b/intl/icu/source/i18n/number_compact.h @@ -0,0 +1,100 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_COMPACT_H__ +#define __NUMBER_COMPACT_H__ + +#include "standardplural.h" +#include "number_types.h" +#include "unicode/unum.h" +#include "uvector.h" +#include "resource.h" +#include "number_patternmodifier.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +static const int32_t COMPACT_MAX_DIGITS = 20; + +class CompactData : public MultiplierProducer { + public: + CompactData(); + + void populate(const Locale &locale, const char *nsName, CompactStyle compactStyle, + CompactType compactType, UErrorCode &status); + + int32_t getMultiplier(int32_t magnitude) const override; + + const char16_t *getPattern( + int32_t magnitude, + const PluralRules *rules, + const DecimalQuantity &dq) const; + + void getUniquePatterns(UVector &output, UErrorCode &status) const; + + private: + const char16_t *patterns[(COMPACT_MAX_DIGITS + 1) * StandardPlural::COUNT]; + int8_t multipliers[COMPACT_MAX_DIGITS + 1]; + int8_t largestMagnitude; + UBool isEmpty; + + class CompactDataSink : public ResourceSink { + public: + explicit CompactDataSink(CompactData &data) : data(data) {} + + void put(const char *key, ResourceValue &value, UBool /*noFallback*/, UErrorCode &status) override; + + private: + CompactData &data; + }; +}; + +struct CompactModInfo { + const ImmutablePatternModifier *mod; + const char16_t* patternString; +}; + +class CompactHandler : public MicroPropsGenerator, public UMemory { + public: + CompactHandler( + CompactStyle compactStyle, + const Locale &locale, + const char *nsName, + CompactType compactType, + const PluralRules *rules, + MutablePatternModifier *buildReference, + bool safe, + const MicroPropsGenerator *parent, + UErrorCode &status); + + ~CompactHandler() override; + + void + processQuantity(DecimalQuantity &quantity, MicroProps µs, UErrorCode &status) const override; + + private: + const PluralRules *rules; + const MicroPropsGenerator *parent; + // Initial capacity of 12 for 0K, 00K, 000K, ...M, ...B, and ...T + MaybeStackArray precomputedMods; + int32_t precomputedModsLength = 0; + CompactData data; + ParsedPatternInfo unsafePatternInfo; + MutablePatternModifier* unsafePatternModifier; + UBool safe; + + /** Used by the safe code path */ + void precomputeAllModifiers(MutablePatternModifier &buildReference, UErrorCode &status); +}; + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_COMPACT_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_currencysymbols.cpp b/intl/icu/source/i18n/number_currencysymbols.cpp new file mode 100644 index 0000000000..8d5127556b --- /dev/null +++ b/intl/icu/source/i18n/number_currencysymbols.cpp @@ -0,0 +1,135 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "number_currencysymbols.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +CurrencySymbols::CurrencySymbols(CurrencyUnit currency, const Locale& locale, UErrorCode& status) + : fCurrency(currency), fLocaleName(locale.getName(), status) { + fCurrencySymbol.setToBogus(); + fIntlCurrencySymbol.setToBogus(); +} + +CurrencySymbols::CurrencySymbols(CurrencyUnit currency, const Locale& locale, + const DecimalFormatSymbols& symbols, UErrorCode& status) + : CurrencySymbols(currency, locale, status) { + // If either of the overrides is present, save it in the local UnicodeString. + if (symbols.isCustomCurrencySymbol()) { + fCurrencySymbol = symbols.getConstSymbol(DecimalFormatSymbols::kCurrencySymbol); + } + if (symbols.isCustomIntlCurrencySymbol()) { + fIntlCurrencySymbol = symbols.getConstSymbol(DecimalFormatSymbols::kIntlCurrencySymbol); + } +} + +const char16_t* CurrencySymbols::getIsoCode() const { + return fCurrency.getISOCurrency(); +} + +UnicodeString CurrencySymbols::getNarrowCurrencySymbol(UErrorCode& status) const { + // Note: currently no override is available for narrow currency symbol + return loadSymbol(UCURR_NARROW_SYMBOL_NAME, status); +} + +UnicodeString CurrencySymbols::getFormalCurrencySymbol(UErrorCode& status) const { + // Note: currently no override is available for formal currency symbol + return loadSymbol(UCURR_FORMAL_SYMBOL_NAME, status); +} + +UnicodeString CurrencySymbols::getVariantCurrencySymbol(UErrorCode& status) const { + // Note: currently no override is available for variant currency symbol + return loadSymbol(UCURR_VARIANT_SYMBOL_NAME, status); +} + +UnicodeString CurrencySymbols::getCurrencySymbol(UErrorCode& status) const { + if (!fCurrencySymbol.isBogus()) { + return fCurrencySymbol; + } + return loadSymbol(UCURR_SYMBOL_NAME, status); +} + +UnicodeString CurrencySymbols::loadSymbol(UCurrNameStyle selector, UErrorCode& status) const { + const char16_t* isoCode = fCurrency.getISOCurrency(); + int32_t symbolLen = 0; + const char16_t* symbol = ucurr_getName( + isoCode, + fLocaleName.data(), + selector, + nullptr /* isChoiceFormat */, + &symbolLen, + &status); + // If given an unknown currency, ucurr_getName returns the input string, which we can't alias safely! + // Otherwise, symbol points to a resource bundle, and we can use readonly-aliasing constructor. + if (symbol == isoCode) { + return UnicodeString(isoCode, 3); + } else { + return UnicodeString(true, symbol, symbolLen); + } +} + +UnicodeString CurrencySymbols::getIntlCurrencySymbol(UErrorCode&) const { + if (!fIntlCurrencySymbol.isBogus()) { + return fIntlCurrencySymbol; + } + // Note: Not safe to use readonly-aliasing constructor here because the buffer belongs to this object, + // which could be destructed or moved during the lifetime of the return value. + return UnicodeString(fCurrency.getISOCurrency(), 3); +} + +UnicodeString CurrencySymbols::getPluralName(StandardPlural::Form plural, UErrorCode& status) const { + const char16_t* isoCode = fCurrency.getISOCurrency(); + int32_t symbolLen = 0; + const char16_t* symbol = ucurr_getPluralName( + isoCode, + fLocaleName.data(), + nullptr /* isChoiceFormat */, + StandardPlural::getKeyword(plural), + &symbolLen, + &status); + // If given an unknown currency, ucurr_getName returns the input string, which we can't alias safely! + // Otherwise, symbol points to a resource bundle, and we can use readonly-aliasing constructor. + if (symbol == isoCode) { + return UnicodeString(isoCode, 3); + } else { + return UnicodeString(true, symbol, symbolLen); + } +} + +bool CurrencySymbols::hasEmptyCurrencySymbol() const { + return !fCurrencySymbol.isBogus() && fCurrencySymbol.isEmpty(); +} + + +CurrencyUnit +icu::number::impl::resolveCurrency(const DecimalFormatProperties& properties, const Locale& locale, + UErrorCode& status) { + if (!properties.currency.isNull()) { + return properties.currency.getNoError(); + } else { + UErrorCode localStatus = U_ZERO_ERROR; + char16_t buf[4] = {}; + ucurr_forLocale(locale.getName(), buf, 4, &localStatus); + if (U_SUCCESS(localStatus)) { + return CurrencyUnit(buf, status); + } else { + // Default currency (XXX) + return CurrencyUnit(); + } + } +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_currencysymbols.h b/intl/icu/source/i18n/number_currencysymbols.h new file mode 100644 index 0000000000..c2223bd0f0 --- /dev/null +++ b/intl/icu/source/i18n/number_currencysymbols.h @@ -0,0 +1,71 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMBER_CURRENCYSYMBOLS_H__ +#define __SOURCE_NUMBER_CURRENCYSYMBOLS_H__ + +#include "numparse_types.h" +#include "charstr.h" +#include "number_decimfmtprops.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + + +// Exported as U_I18N_API for tests +class U_I18N_API CurrencySymbols : public UMemory { + public: + CurrencySymbols() = default; // default constructor: leaves class in valid but undefined state + + /** Creates an instance in which all symbols are loaded from data. */ + CurrencySymbols(CurrencyUnit currency, const Locale& locale, UErrorCode& status); + + /** Creates an instance in which some symbols might be pre-populated. */ + CurrencySymbols(CurrencyUnit currency, const Locale& locale, const DecimalFormatSymbols& symbols, + UErrorCode& status); + + const char16_t* getIsoCode() const; + + UnicodeString getNarrowCurrencySymbol(UErrorCode& status) const; + + UnicodeString getFormalCurrencySymbol(UErrorCode& status) const; + + UnicodeString getVariantCurrencySymbol(UErrorCode& status) const; + + UnicodeString getCurrencySymbol(UErrorCode& status) const; + + UnicodeString getIntlCurrencySymbol(UErrorCode& status) const; + + UnicodeString getPluralName(StandardPlural::Form plural, UErrorCode& status) const; + + bool hasEmptyCurrencySymbol() const; + + protected: + // Required fields: + CurrencyUnit fCurrency; + CharString fLocaleName; + + // Optional fields: + UnicodeString fCurrencySymbol; + UnicodeString fIntlCurrencySymbol; + + UnicodeString loadSymbol(UCurrNameStyle selector, UErrorCode& status) const; +}; + + +/** + * Resolves the effective currency from the property bag. + */ +CurrencyUnit +resolveCurrency(const DecimalFormatProperties& properties, const Locale& locale, UErrorCode& status); + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__SOURCE_NUMBER_CURRENCYSYMBOLS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_decimalquantity.cpp b/intl/icu/source/i18n/number_decimalquantity.cpp new file mode 100644 index 0000000000..659465d087 --- /dev/null +++ b/intl/icu/source/i18n/number_decimalquantity.cpp @@ -0,0 +1,1474 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include +#include +#include +#include + +#include "unicode/plurrule.h" +#include "cmemory.h" +#include "number_decnum.h" +#include "putilimp.h" +#include "number_decimalquantity.h" +#include "number_roundingutils.h" +#include "double-conversion.h" +#include "charstr.h" +#include "number_utils.h" +#include "uassert.h" +#include "util.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +using icu::double_conversion::DoubleToStringConverter; +using icu::double_conversion::StringToDoubleConverter; + +namespace { + +int8_t NEGATIVE_FLAG = 1; +int8_t INFINITY_FLAG = 2; +int8_t NAN_FLAG = 4; + +/** Helper function for safe subtraction (no overflow). */ +inline int32_t safeSubtract(int32_t a, int32_t b) { + // Note: In C++, signed integer subtraction is undefined behavior. + int32_t diff = static_cast(static_cast(a) - static_cast(b)); + if (b < 0 && diff < a) { return INT32_MAX; } + if (b > 0 && diff > a) { return INT32_MIN; } + return diff; +} + +static double DOUBLE_MULTIPLIERS[] = { + 1e0, + 1e1, + 1e2, + 1e3, + 1e4, + 1e5, + 1e6, + 1e7, + 1e8, + 1e9, + 1e10, + 1e11, + 1e12, + 1e13, + 1e14, + 1e15, + 1e16, + 1e17, + 1e18, + 1e19, + 1e20, + 1e21}; + +} // namespace + +icu::IFixedDecimal::~IFixedDecimal() = default; + +DecimalQuantity::DecimalQuantity() { + setBcdToZero(); + flags = 0; +} + +DecimalQuantity::~DecimalQuantity() { + if (usingBytes) { + uprv_free(fBCD.bcdBytes.ptr); + fBCD.bcdBytes.ptr = nullptr; + usingBytes = false; + } +} + +DecimalQuantity::DecimalQuantity(const DecimalQuantity &other) { + *this = other; +} + +DecimalQuantity::DecimalQuantity(DecimalQuantity&& src) noexcept { + *this = std::move(src); +} + +DecimalQuantity &DecimalQuantity::operator=(const DecimalQuantity &other) { + if (this == &other) { + return *this; + } + copyBcdFrom(other); + copyFieldsFrom(other); + return *this; +} + +DecimalQuantity& DecimalQuantity::operator=(DecimalQuantity&& src) noexcept { + if (this == &src) { + return *this; + } + moveBcdFrom(src); + copyFieldsFrom(src); + return *this; +} + +void DecimalQuantity::copyFieldsFrom(const DecimalQuantity& other) { + bogus = other.bogus; + lReqPos = other.lReqPos; + rReqPos = other.rReqPos; + scale = other.scale; + precision = other.precision; + flags = other.flags; + origDouble = other.origDouble; + origDelta = other.origDelta; + isApproximate = other.isApproximate; + exponent = other.exponent; +} + +void DecimalQuantity::clear() { + lReqPos = 0; + rReqPos = 0; + flags = 0; + setBcdToZero(); // sets scale, precision, hasDouble, origDouble, origDelta, and BCD data +} + +void DecimalQuantity::setMinInteger(int32_t minInt) { + // Validation should happen outside of DecimalQuantity, e.g., in the Precision class. + U_ASSERT(minInt >= 0); + + // Special behavior: do not set minInt to be less than what is already set. + // This is so significant digits rounding can set the integer length. + if (minInt < lReqPos) { + minInt = lReqPos; + } + + // Save values into internal state + lReqPos = minInt; +} + +void DecimalQuantity::setMinFraction(int32_t minFrac) { + // Validation should happen outside of DecimalQuantity, e.g., in the Precision class. + U_ASSERT(minFrac >= 0); + + // Save values into internal state + // Negation is safe for minFrac/maxFrac because -Integer.MAX_VALUE > Integer.MIN_VALUE + rReqPos = -minFrac; +} + +void DecimalQuantity::applyMaxInteger(int32_t maxInt) { + // Validation should happen outside of DecimalQuantity, e.g., in the Precision class. + U_ASSERT(maxInt >= 0); + + if (precision == 0) { + return; + } + + if (maxInt <= scale) { + setBcdToZero(); + return; + } + + int32_t magnitude = getMagnitude(); + if (maxInt <= magnitude) { + popFromLeft(magnitude - maxInt + 1); + compact(); + } +} + +uint64_t DecimalQuantity::getPositionFingerprint() const { + uint64_t fingerprint = 0; + fingerprint ^= (lReqPos << 16); + fingerprint ^= (static_cast(rReqPos) << 32); + return fingerprint; +} + +void DecimalQuantity::roundToIncrement( + uint64_t increment, + digits_t magnitude, + RoundingMode roundingMode, + UErrorCode& status) { + // Do not call this method with an increment having only a 1 or a 5 digit! + // Use a more efficient call to either roundToMagnitude() or roundToNickel(). + // Check a few popular rounding increments; a more thorough check is in Java. + U_ASSERT(increment != 1); + U_ASSERT(increment != 5); + + DecimalQuantity incrementDQ; + incrementDQ.setToLong(increment); + incrementDQ.adjustMagnitude(magnitude); + DecNum incrementDN; + incrementDQ.toDecNum(incrementDN, status); + if (U_FAILURE(status)) { return; } + + // Divide this DecimalQuantity by the increment, round, then multiply back. + divideBy(incrementDN, status); + if (U_FAILURE(status)) { return; } + roundToMagnitude(0, roundingMode, status); + if (U_FAILURE(status)) { return; } + multiplyBy(incrementDN, status); + if (U_FAILURE(status)) { return; } +} + +void DecimalQuantity::multiplyBy(const DecNum& multiplicand, UErrorCode& status) { + if (isZeroish()) { + return; + } + // Convert to DecNum, multiply, and convert back. + DecNum decnum; + toDecNum(decnum, status); + if (U_FAILURE(status)) { return; } + decnum.multiplyBy(multiplicand, status); + if (U_FAILURE(status)) { return; } + setToDecNum(decnum, status); +} + +void DecimalQuantity::divideBy(const DecNum& divisor, UErrorCode& status) { + if (isZeroish()) { + return; + } + // Convert to DecNum, multiply, and convert back. + DecNum decnum; + toDecNum(decnum, status); + if (U_FAILURE(status)) { return; } + decnum.divideBy(divisor, status); + if (U_FAILURE(status)) { return; } + setToDecNum(decnum, status); +} + +void DecimalQuantity::negate() { + flags ^= NEGATIVE_FLAG; +} + +int32_t DecimalQuantity::getMagnitude() const { + U_ASSERT(precision != 0); + return scale + precision - 1; +} + +bool DecimalQuantity::adjustMagnitude(int32_t delta) { + if (precision != 0) { + // i.e., scale += delta; origDelta += delta + bool overflow = uprv_add32_overflow(scale, delta, &scale); + overflow = uprv_add32_overflow(origDelta, delta, &origDelta) || overflow; + // Make sure that precision + scale won't overflow, either + int32_t dummy; + overflow = overflow || uprv_add32_overflow(scale, precision, &dummy); + return overflow; + } + return false; +} + +int32_t DecimalQuantity::adjustToZeroScale() { + int32_t retval = scale; + scale = 0; + return retval; +} + +double DecimalQuantity::getPluralOperand(PluralOperand operand) const { + // If this assertion fails, you need to call roundToInfinity() or some other rounding method. + // See the comment at the top of this file explaining the "isApproximate" field. + U_ASSERT(!isApproximate); + + switch (operand) { + case PLURAL_OPERAND_I: + // Invert the negative sign if necessary + return static_cast(isNegative() ? -toLong(true) : toLong(true)); + case PLURAL_OPERAND_F: + return static_cast(toFractionLong(true)); + case PLURAL_OPERAND_T: + return static_cast(toFractionLong(false)); + case PLURAL_OPERAND_V: + return fractionCount(); + case PLURAL_OPERAND_W: + return fractionCountWithoutTrailingZeros(); + case PLURAL_OPERAND_E: + return static_cast(getExponent()); + case PLURAL_OPERAND_C: + // Plural operand `c` is currently an alias for `e`. + return static_cast(getExponent()); + default: + return std::abs(toDouble()); + } +} + +int32_t DecimalQuantity::getExponent() const { + return exponent; +} + +void DecimalQuantity::adjustExponent(int delta) { + exponent = exponent + delta; +} + +void DecimalQuantity::resetExponent() { + adjustMagnitude(exponent); + exponent = 0; +} + +bool DecimalQuantity::hasIntegerValue() const { + return scale >= 0; +} + +int32_t DecimalQuantity::getUpperDisplayMagnitude() const { + // If this assertion fails, you need to call roundToInfinity() or some other rounding method. + // See the comment in the header file explaining the "isApproximate" field. + U_ASSERT(!isApproximate); + + int32_t magnitude = scale + precision; + int32_t result = (lReqPos > magnitude) ? lReqPos : magnitude; + return result - 1; +} + +int32_t DecimalQuantity::getLowerDisplayMagnitude() const { + // If this assertion fails, you need to call roundToInfinity() or some other rounding method. + // See the comment in the header file explaining the "isApproximate" field. + U_ASSERT(!isApproximate); + + int32_t magnitude = scale; + int32_t result = (rReqPos < magnitude) ? rReqPos : magnitude; + return result; +} + +int8_t DecimalQuantity::getDigit(int32_t magnitude) const { + // If this assertion fails, you need to call roundToInfinity() or some other rounding method. + // See the comment at the top of this file explaining the "isApproximate" field. + U_ASSERT(!isApproximate); + + return getDigitPos(magnitude - scale); +} + +int32_t DecimalQuantity::fractionCount() const { + int32_t fractionCountWithExponent = -getLowerDisplayMagnitude() - exponent; + return fractionCountWithExponent > 0 ? fractionCountWithExponent : 0; +} + +int32_t DecimalQuantity::fractionCountWithoutTrailingZeros() const { + int32_t fractionCountWithExponent = -scale - exponent; + return fractionCountWithExponent > 0 ? fractionCountWithExponent : 0; // max(-fractionCountWithExponent, 0) +} + +bool DecimalQuantity::isNegative() const { + return (flags & NEGATIVE_FLAG) != 0; +} + +Signum DecimalQuantity::signum() const { + bool isZero = (isZeroish() && !isInfinite()); + bool isNeg = isNegative(); + if (isZero && isNeg) { + return SIGNUM_NEG_ZERO; + } else if (isZero) { + return SIGNUM_POS_ZERO; + } else if (isNeg) { + return SIGNUM_NEG; + } else { + return SIGNUM_POS; + } +} + +bool DecimalQuantity::isInfinite() const { + return (flags & INFINITY_FLAG) != 0; +} + +bool DecimalQuantity::isNaN() const { + return (flags & NAN_FLAG) != 0; +} + +bool DecimalQuantity::isZeroish() const { + return precision == 0; +} + +DecimalQuantity &DecimalQuantity::setToInt(int32_t n) { + setBcdToZero(); + flags = 0; + if (n == INT32_MIN) { + flags |= NEGATIVE_FLAG; + // leave as INT32_MIN; handled below in _setToInt() + } else if (n < 0) { + flags |= NEGATIVE_FLAG; + n = -n; + } + if (n != 0) { + _setToInt(n); + compact(); + } + return *this; +} + +void DecimalQuantity::_setToInt(int32_t n) { + if (n == INT32_MIN) { + readLongToBcd(-static_cast(n)); + } else { + readIntToBcd(n); + } +} + +DecimalQuantity &DecimalQuantity::setToLong(int64_t n) { + setBcdToZero(); + flags = 0; + if (n < 0 && n > INT64_MIN) { + flags |= NEGATIVE_FLAG; + n = -n; + } + if (n != 0) { + _setToLong(n); + compact(); + } + return *this; +} + +void DecimalQuantity::_setToLong(int64_t n) { + if (n == INT64_MIN) { + DecNum decnum; + UErrorCode localStatus = U_ZERO_ERROR; + decnum.setTo("9.223372036854775808E+18", localStatus); + if (U_FAILURE(localStatus)) { return; } // unexpected + flags |= NEGATIVE_FLAG; + readDecNumberToBcd(decnum); + } else if (n <= INT32_MAX) { + readIntToBcd(static_cast(n)); + } else { + readLongToBcd(n); + } +} + +DecimalQuantity &DecimalQuantity::setToDouble(double n) { + setBcdToZero(); + flags = 0; + // signbit() from handles +0.0 vs -0.0 + if (std::signbit(n)) { + flags |= NEGATIVE_FLAG; + n = -n; + } + if (std::isnan(n) != 0) { + flags |= NAN_FLAG; + } else if (std::isfinite(n) == 0) { + flags |= INFINITY_FLAG; + } else if (n != 0) { + _setToDoubleFast(n); + compact(); + } + return *this; +} + +void DecimalQuantity::_setToDoubleFast(double n) { + isApproximate = true; + origDouble = n; + origDelta = 0; + + // Make sure the double is an IEEE 754 double. If not, fall back to the slow path right now. + // TODO: Make a fast path for other types of doubles. + if (!std::numeric_limits::is_iec559) { + convertToAccurateDouble(); + return; + } + + // To get the bits from the double, use memcpy, which takes care of endianness. + uint64_t ieeeBits; + uprv_memcpy(&ieeeBits, &n, sizeof(n)); + int32_t exponent = static_cast((ieeeBits & 0x7ff0000000000000L) >> 52) - 0x3ff; + + // Not all integers can be represented exactly for exponent > 52 + if (exponent <= 52 && static_cast(n) == n) { + _setToLong(static_cast(n)); + return; + } + + if (exponent == -1023 || exponent == 1024) { + // The extreme values of exponent are special; use slow path. + convertToAccurateDouble(); + return; + } + + // 3.3219... is log2(10) + auto fracLength = static_cast ((52 - exponent) / 3.32192809488736234787031942948939017586); + if (fracLength >= 0) { + int32_t i = fracLength; + // 1e22 is the largest exact double. + for (; i >= 22; i -= 22) n *= 1e22; + n *= DOUBLE_MULTIPLIERS[i]; + } else { + int32_t i = fracLength; + // 1e22 is the largest exact double. + for (; i <= -22; i += 22) n /= 1e22; + n /= DOUBLE_MULTIPLIERS[-i]; + } + auto result = static_cast(uprv_round(n)); + if (result != 0) { + _setToLong(result); + scale -= fracLength; + } +} + +void DecimalQuantity::convertToAccurateDouble() { + U_ASSERT(origDouble != 0); + int32_t delta = origDelta; + + // Call the slow oracle function (Double.toString in Java, DoubleToAscii in C++). + char buffer[DoubleToStringConverter::kBase10MaximalLength + 1]; + bool sign; // unused; always positive + int32_t length; + int32_t point; + DoubleToStringConverter::DoubleToAscii( + origDouble, + DoubleToStringConverter::DtoaMode::SHORTEST, + 0, + buffer, + sizeof(buffer), + &sign, + &length, + &point + ); + + setBcdToZero(); + readDoubleConversionToBcd(buffer, length, point); + scale += delta; + explicitExactDouble = true; +} + +DecimalQuantity &DecimalQuantity::setToDecNumber(StringPiece n, UErrorCode& status) { + setBcdToZero(); + flags = 0; + + // Compute the decNumber representation + DecNum decnum; + decnum.setTo(n, status); + + _setToDecNum(decnum, status); + return *this; +} + +DecimalQuantity& DecimalQuantity::setToDecNum(const DecNum& decnum, UErrorCode& status) { + setBcdToZero(); + flags = 0; + + _setToDecNum(decnum, status); + return *this; +} + +void DecimalQuantity::_setToDecNum(const DecNum& decnum, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + if (decnum.isNegative()) { + flags |= NEGATIVE_FLAG; + } + if (decnum.isNaN()) { + flags |= NAN_FLAG; + } else if (decnum.isInfinity()) { + flags |= INFINITY_FLAG; + } else if (!decnum.isZero()) { + readDecNumberToBcd(decnum); + compact(); + } +} + +DecimalQuantity DecimalQuantity::fromExponentString(UnicodeString num, UErrorCode& status) { + if (num.indexOf(u'e') >= 0 || num.indexOf(u'c') >= 0 + || num.indexOf(u'E') >= 0 || num.indexOf(u'C') >= 0) { + int32_t ePos = num.lastIndexOf('e'); + if (ePos < 0) { + ePos = num.lastIndexOf('c'); + } + if (ePos < 0) { + ePos = num.lastIndexOf('E'); + } + if (ePos < 0) { + ePos = num.lastIndexOf('C'); + } + int32_t expNumPos = ePos + 1; + UnicodeString exponentStr = num.tempSubString(expNumPos, num.length() - expNumPos); + + // parse exponentStr into exponent, but note that parseAsciiInteger doesn't handle the minus sign + bool isExpStrNeg = num[expNumPos] == u'-'; + int32_t exponentParsePos = isExpStrNeg ? 1 : 0; + int32_t exponent = ICU_Utility::parseAsciiInteger(exponentStr, exponentParsePos); + exponent = isExpStrNeg ? -exponent : exponent; + + // Compute the decNumber representation + UnicodeString fractionStr = num.tempSubString(0, ePos); + CharString fracCharStr = CharString(); + fracCharStr.appendInvariantChars(fractionStr, status); + DecNum decnum; + decnum.setTo(fracCharStr.toStringPiece(), status); + + // Clear and set this DecimalQuantity instance + DecimalQuantity dq; + dq.setToDecNum(decnum, status); + int32_t numFracDigit = getVisibleFractionCount(fractionStr); + dq.setMinFraction(numFracDigit); + dq.adjustExponent(exponent); + + return dq; + } else { + DecimalQuantity dq; + int numFracDigit = getVisibleFractionCount(num); + + CharString numCharStr = CharString(); + numCharStr.appendInvariantChars(num, status); + dq.setToDecNumber(numCharStr.toStringPiece(), status); + + dq.setMinFraction(numFracDigit); + return dq; + } +} + +int32_t DecimalQuantity::getVisibleFractionCount(UnicodeString value) { + int decimalPos = value.indexOf('.') + 1; + if (decimalPos == 0) { + return 0; + } else { + return value.length() - decimalPos; + } +} + +int64_t DecimalQuantity::toLong(bool truncateIfOverflow) const { + // NOTE: Call sites should be guarded by fitsInLong(), like this: + // if (dq.fitsInLong()) { /* use dq.toLong() */ } else { /* use some fallback */ } + // Fallback behavior upon truncateIfOverflow is to truncate at 17 digits. + uint64_t result = 0L; + int32_t upperMagnitude = exponent + scale + precision - 1; + if (truncateIfOverflow) { + upperMagnitude = std::min(upperMagnitude, 17); + } + for (int32_t magnitude = upperMagnitude; magnitude >= 0; magnitude--) { + result = result * 10 + getDigitPos(magnitude - scale - exponent); + } + if (isNegative()) { + return static_cast(0LL - result); // i.e., -result + } + return static_cast(result); +} + +uint64_t DecimalQuantity::toFractionLong(bool includeTrailingZeros) const { + uint64_t result = 0L; + int32_t magnitude = -1 - exponent; + int32_t lowerMagnitude = scale; + if (includeTrailingZeros) { + lowerMagnitude = std::min(lowerMagnitude, rReqPos); + } + for (; magnitude >= lowerMagnitude && result <= 1e18L; magnitude--) { + result = result * 10 + getDigitPos(magnitude - scale); + } + // Remove trailing zeros; this can happen during integer overflow cases. + if (!includeTrailingZeros) { + while (result > 0 && (result % 10) == 0) { + result /= 10; + } + } + return result; +} + +bool DecimalQuantity::fitsInLong(bool ignoreFraction) const { + if (isInfinite() || isNaN()) { + return false; + } + if (isZeroish()) { + return true; + } + if (exponent + scale < 0 && !ignoreFraction) { + return false; + } + int magnitude = getMagnitude(); + if (magnitude < 18) { + return true; + } + if (magnitude > 18) { + return false; + } + // Hard case: the magnitude is 10^18. + // The largest int64 is: 9,223,372,036,854,775,807 + for (int p = 0; p < precision; p++) { + int8_t digit = getDigit(18 - p); + static int8_t INT64_BCD[] = { 9, 2, 2, 3, 3, 7, 2, 0, 3, 6, 8, 5, 4, 7, 7, 5, 8, 0, 8 }; + if (digit < INT64_BCD[p]) { + return true; + } else if (digit > INT64_BCD[p]) { + return false; + } + } + // Exactly equal to max long plus one. + return isNegative(); +} + +double DecimalQuantity::toDouble() const { + // If this assertion fails, you need to call roundToInfinity() or some other rounding method. + // See the comment in the header file explaining the "isApproximate" field. + U_ASSERT(!isApproximate); + + if (isNaN()) { + return NAN; + } else if (isInfinite()) { + return isNegative() ? -INFINITY : INFINITY; + } + + // We are processing well-formed input, so we don't need any special options to StringToDoubleConverter. + StringToDoubleConverter converter(0, 0, 0, "", ""); + UnicodeString numberString = this->toScientificString(); + int32_t count; + return converter.StringToDouble( + reinterpret_cast(numberString.getBuffer()), + numberString.length(), + &count); +} + +DecNum& DecimalQuantity::toDecNum(DecNum& output, UErrorCode& status) const { + // Special handling for zero + if (precision == 0) { + output.setTo("0", status); + return output; + } + + // Use the BCD constructor. We need to do a little bit of work to convert, though. + // The decNumber constructor expects most-significant first, but we store least-significant first. + MaybeStackArray ubcd(precision, status); + if (U_FAILURE(status)) { + return output; + } + for (int32_t m = 0; m < precision; m++) { + ubcd[precision - m - 1] = static_cast(getDigitPos(m)); + } + output.setTo(ubcd.getAlias(), precision, scale, isNegative(), status); + return output; +} + +void DecimalQuantity::truncate() { + if (scale < 0) { + shiftRight(-scale); + scale = 0; + compact(); + } +} + +void DecimalQuantity::roundToNickel(int32_t magnitude, RoundingMode roundingMode, UErrorCode& status) { + roundToMagnitude(magnitude, roundingMode, true, status); +} + +void DecimalQuantity::roundToMagnitude(int32_t magnitude, RoundingMode roundingMode, UErrorCode& status) { + roundToMagnitude(magnitude, roundingMode, false, status); +} + +void DecimalQuantity::roundToMagnitude(int32_t magnitude, RoundingMode roundingMode, bool nickel, UErrorCode& status) { + // The position in the BCD at which rounding will be performed; digits to the right of position + // will be rounded away. + int position = safeSubtract(magnitude, scale); + + // "trailing" = least significant digit to the left of rounding + int8_t trailingDigit = getDigitPos(position); + + if (position <= 0 && !isApproximate && (!nickel || trailingDigit == 0 || trailingDigit == 5)) { + // All digits are to the left of the rounding magnitude. + } else if (precision == 0) { + // No rounding for zero. + } else { + // Perform rounding logic. + // "leading" = most significant digit to the right of rounding + int8_t leadingDigit = getDigitPos(safeSubtract(position, 1)); + + // Compute which section of the number we are in. + // EDGE means we are at the bottom or top edge, like 1.000 or 1.999 (used by doubles) + // LOWER means we are between the bottom edge and the midpoint, like 1.391 + // MIDPOINT means we are exactly in the middle, like 1.500 + // UPPER means we are between the midpoint and the top edge, like 1.916 + roundingutils::Section section; + if (!isApproximate) { + if (nickel && trailingDigit != 2 && trailingDigit != 7) { + // Nickel rounding, and not at .02x or .07x + if (trailingDigit < 2) { + // .00, .01 => down to .00 + section = roundingutils::SECTION_LOWER; + } else if (trailingDigit < 5) { + // .03, .04 => up to .05 + section = roundingutils::SECTION_UPPER; + } else if (trailingDigit < 7) { + // .05, .06 => down to .05 + section = roundingutils::SECTION_LOWER; + } else { + // .08, .09 => up to .10 + section = roundingutils::SECTION_UPPER; + } + } else if (leadingDigit < 5) { + // Includes nickel rounding .020-.024 and .070-.074 + section = roundingutils::SECTION_LOWER; + } else if (leadingDigit > 5) { + // Includes nickel rounding .026-.029 and .076-.079 + section = roundingutils::SECTION_UPPER; + } else { + // Includes nickel rounding .025 and .075 + section = roundingutils::SECTION_MIDPOINT; + for (int p = safeSubtract(position, 2); p >= 0; p--) { + if (getDigitPos(p) != 0) { + section = roundingutils::SECTION_UPPER; + break; + } + } + } + } else { + int32_t p = safeSubtract(position, 2); + int32_t minP = uprv_max(0, precision - 14); + if (leadingDigit == 0 && (!nickel || trailingDigit == 0 || trailingDigit == 5)) { + section = roundingutils::SECTION_LOWER_EDGE; + for (; p >= minP; p--) { + if (getDigitPos(p) != 0) { + section = roundingutils::SECTION_LOWER; + break; + } + } + } else if (leadingDigit == 4 && (!nickel || trailingDigit == 2 || trailingDigit == 7)) { + section = roundingutils::SECTION_MIDPOINT; + for (; p >= minP; p--) { + if (getDigitPos(p) != 9) { + section = roundingutils::SECTION_LOWER; + break; + } + } + } else if (leadingDigit == 5 && (!nickel || trailingDigit == 2 || trailingDigit == 7)) { + section = roundingutils::SECTION_MIDPOINT; + for (; p >= minP; p--) { + if (getDigitPos(p) != 0) { + section = roundingutils::SECTION_UPPER; + break; + } + } + } else if (leadingDigit == 9 && (!nickel || trailingDigit == 4 || trailingDigit == 9)) { + section = roundingutils::SECTION_UPPER_EDGE; + for (; p >= minP; p--) { + if (getDigitPos(p) != 9) { + section = roundingutils::SECTION_UPPER; + break; + } + } + } else if (nickel && trailingDigit != 2 && trailingDigit != 7) { + // Nickel rounding, and not at .02x or .07x + if (trailingDigit < 2) { + // .00, .01 => down to .00 + section = roundingutils::SECTION_LOWER; + } else if (trailingDigit < 5) { + // .03, .04 => up to .05 + section = roundingutils::SECTION_UPPER; + } else if (trailingDigit < 7) { + // .05, .06 => down to .05 + section = roundingutils::SECTION_LOWER; + } else { + // .08, .09 => up to .10 + section = roundingutils::SECTION_UPPER; + } + } else if (leadingDigit < 5) { + // Includes nickel rounding .020-.024 and .070-.074 + section = roundingutils::SECTION_LOWER; + } else { + // Includes nickel rounding .026-.029 and .076-.079 + section = roundingutils::SECTION_UPPER; + } + + bool roundsAtMidpoint = roundingutils::roundsAtMidpoint(roundingMode); + if (safeSubtract(position, 1) < precision - 14 || + (roundsAtMidpoint && section == roundingutils::SECTION_MIDPOINT) || + (!roundsAtMidpoint && section < 0 /* i.e. at upper or lower edge */)) { + // Oops! This means that we have to get the exact representation of the double, + // because the zone of uncertainty is along the rounding boundary. + convertToAccurateDouble(); + roundToMagnitude(magnitude, roundingMode, nickel, status); // start over + return; + } + + // Turn off the approximate double flag, since the value is now confirmed to be exact. + isApproximate = false; + origDouble = 0.0; + origDelta = 0; + + if (position <= 0 && (!nickel || trailingDigit == 0 || trailingDigit == 5)) { + // All digits are to the left of the rounding magnitude. + return; + } + + // Good to continue rounding. + if (section == -1) { section = roundingutils::SECTION_LOWER; } + if (section == -2) { section = roundingutils::SECTION_UPPER; } + } + + // Nickel rounding "half even" goes to the nearest whole (away from the 5). + bool isEven = nickel + ? (trailingDigit < 2 || trailingDigit > 7 + || (trailingDigit == 2 && section != roundingutils::SECTION_UPPER) + || (trailingDigit == 7 && section == roundingutils::SECTION_UPPER)) + : (trailingDigit % 2) == 0; + + bool roundDown = roundingutils::getRoundingDirection(isEven, + isNegative(), + section, + roundingMode, + status); + if (U_FAILURE(status)) { + return; + } + + // Perform truncation + if (position >= precision) { + U_ASSERT(trailingDigit == 0); + setBcdToZero(); + scale = magnitude; + } else { + shiftRight(position); + } + + if (nickel) { + if (trailingDigit < 5 && roundDown) { + setDigitPos(0, 0); + compact(); + return; + } else if (trailingDigit >= 5 && !roundDown) { + setDigitPos(0, 9); + trailingDigit = 9; + // do not return: use the bubbling logic below + } else { + setDigitPos(0, 5); + // If the quantity was set to 0, we may need to restore a digit. + if (precision == 0) { + precision = 1; + } + // compact not necessary: digit at position 0 is nonzero + return; + } + } + + // Bubble the result to the higher digits + if (!roundDown) { + if (trailingDigit == 9) { + int bubblePos = 0; + // Note: in the long implementation, the most digits BCD can have at this point is + // 15, so bubblePos <= 15 and getDigitPos(bubblePos) is safe. + for (; getDigitPos(bubblePos) == 9; bubblePos++) {} + shiftRight(bubblePos); // shift off the trailing 9s + } + int8_t digit0 = getDigitPos(0); + U_ASSERT(digit0 != 9); + setDigitPos(0, static_cast(digit0 + 1)); + precision += 1; // in case an extra digit got added + } + + compact(); + } +} + +void DecimalQuantity::roundToInfinity() { + if (isApproximate) { + convertToAccurateDouble(); + } +} + +void DecimalQuantity::appendDigit(int8_t value, int32_t leadingZeros, bool appendAsInteger) { + U_ASSERT(leadingZeros >= 0); + + // Zero requires special handling to maintain the invariant that the least-significant digit + // in the BCD is nonzero. + if (value == 0) { + if (appendAsInteger && precision != 0) { + scale += leadingZeros + 1; + } + return; + } + + // Deal with trailing zeros + if (scale > 0) { + leadingZeros += scale; + if (appendAsInteger) { + scale = 0; + } + } + + // Append digit + shiftLeft(leadingZeros + 1); + setDigitPos(0, value); + + // Fix scale if in integer mode + if (appendAsInteger) { + scale += leadingZeros + 1; + } +} + +UnicodeString DecimalQuantity::toPlainString() const { + U_ASSERT(!isApproximate); + UnicodeString sb; + if (isNegative()) { + sb.append(u'-'); + } + if (precision == 0) { + sb.append(u'0'); + return sb; + } + int32_t upper = scale + precision + exponent - 1; + int32_t lower = scale + exponent; + if (upper < lReqPos - 1) { + upper = lReqPos - 1; + } + if (lower > rReqPos) { + lower = rReqPos; + } + int32_t p = upper; + if (p < 0) { + sb.append(u'0'); + } + for (; p >= 0; p--) { + sb.append(u'0' + getDigitPos(p - scale - exponent)); + } + if (lower < 0) { + sb.append(u'.'); + } + for(; p >= lower; p--) { + sb.append(u'0' + getDigitPos(p - scale - exponent)); + } + return sb; +} + + +UnicodeString DecimalQuantity::toExponentString() const { + U_ASSERT(!isApproximate); + UnicodeString sb; + if (isNegative()) { + sb.append(u'-'); + } + + int32_t upper = scale + precision - 1; + int32_t lower = scale; + if (upper < lReqPos - 1) { + upper = lReqPos - 1; + } + if (lower > rReqPos) { + lower = rReqPos; + } + int32_t p = upper; + if (p < 0) { + sb.append(u'0'); + } + for (; p >= 0; p--) { + sb.append(u'0' + getDigitPos(p - scale)); + } + if (lower < 0) { + sb.append(u'.'); + } + for(; p >= lower; p--) { + sb.append(u'0' + getDigitPos(p - scale)); + } + + if (exponent != 0) { + sb.append(u'c'); + ICU_Utility::appendNumber(sb, exponent); + } + + return sb; +} + +UnicodeString DecimalQuantity::toScientificString() const { + U_ASSERT(!isApproximate); + UnicodeString result; + if (isNegative()) { + result.append(u'-'); + } + if (precision == 0) { + result.append(u"0E+0", -1); + return result; + } + int32_t upperPos = precision - 1; + int32_t lowerPos = 0; + int32_t p = upperPos; + result.append(u'0' + getDigitPos(p)); + if ((--p) >= lowerPos) { + result.append(u'.'); + for (; p >= lowerPos; p--) { + result.append(u'0' + getDigitPos(p)); + } + } + result.append(u'E'); + int32_t _scale = upperPos + scale + exponent; + if (_scale == INT32_MIN) { + result.append({u"-2147483648", -1}); + return result; + } else if (_scale < 0) { + _scale *= -1; + result.append(u'-'); + } else { + result.append(u'+'); + } + if (_scale == 0) { + result.append(u'0'); + } + int32_t insertIndex = result.length(); + while (_scale > 0) { + std::div_t res = std::div(_scale, 10); + result.insert(insertIndex, u'0' + res.rem); + _scale = res.quot; + } + return result; +} + +//////////////////////////////////////////////////// +/// End of DecimalQuantity_AbstractBCD.java /// +/// Start of DecimalQuantity_DualStorageBCD.java /// +//////////////////////////////////////////////////// + +int8_t DecimalQuantity::getDigitPos(int32_t position) const { + if (usingBytes) { + if (position < 0 || position >= precision) { return 0; } + return fBCD.bcdBytes.ptr[position]; + } else { + if (position < 0 || position >= 16) { return 0; } + return (int8_t) ((fBCD.bcdLong >> (position * 4)) & 0xf); + } +} + +void DecimalQuantity::setDigitPos(int32_t position, int8_t value) { + U_ASSERT(position >= 0); + if (usingBytes) { + ensureCapacity(position + 1); + fBCD.bcdBytes.ptr[position] = value; + } else if (position >= 16) { + switchStorage(); + ensureCapacity(position + 1); + fBCD.bcdBytes.ptr[position] = value; + } else { + int shift = position * 4; + fBCD.bcdLong = (fBCD.bcdLong & ~(0xfL << shift)) | ((long) value << shift); + } +} + +void DecimalQuantity::shiftLeft(int32_t numDigits) { + if (!usingBytes && precision + numDigits > 16) { + switchStorage(); + } + if (usingBytes) { + ensureCapacity(precision + numDigits); + uprv_memmove(fBCD.bcdBytes.ptr + numDigits, fBCD.bcdBytes.ptr, precision); + uprv_memset(fBCD.bcdBytes.ptr, 0, numDigits); + } else { + fBCD.bcdLong <<= (numDigits * 4); + } + scale -= numDigits; + precision += numDigits; +} + +void DecimalQuantity::shiftRight(int32_t numDigits) { + if (usingBytes) { + int i = 0; + for (; i < precision - numDigits; i++) { + fBCD.bcdBytes.ptr[i] = fBCD.bcdBytes.ptr[i + numDigits]; + } + for (; i < precision; i++) { + fBCD.bcdBytes.ptr[i] = 0; + } + } else { + fBCD.bcdLong >>= (numDigits * 4); + } + scale += numDigits; + precision -= numDigits; +} + +void DecimalQuantity::popFromLeft(int32_t numDigits) { + U_ASSERT(numDigits <= precision); + if (usingBytes) { + int i = precision - 1; + for (; i >= precision - numDigits; i--) { + fBCD.bcdBytes.ptr[i] = 0; + } + } else { + fBCD.bcdLong &= (static_cast(1) << ((precision - numDigits) * 4)) - 1; + } + precision -= numDigits; +} + +void DecimalQuantity::setBcdToZero() { + if (usingBytes) { + uprv_free(fBCD.bcdBytes.ptr); + fBCD.bcdBytes.ptr = nullptr; + usingBytes = false; + } + fBCD.bcdLong = 0L; + scale = 0; + precision = 0; + isApproximate = false; + origDouble = 0; + origDelta = 0; + exponent = 0; +} + +void DecimalQuantity::readIntToBcd(int32_t n) { + U_ASSERT(n != 0); + // ints always fit inside the long implementation. + uint64_t result = 0L; + int i = 16; + for (; n != 0; n /= 10, i--) { + result = (result >> 4) + ((static_cast(n) % 10) << 60); + } + U_ASSERT(!usingBytes); + fBCD.bcdLong = result >> (i * 4); + scale = 0; + precision = 16 - i; +} + +void DecimalQuantity::readLongToBcd(int64_t n) { + U_ASSERT(n != 0); + if (n >= 10000000000000000L) { + ensureCapacity(); + int i = 0; + for (; n != 0L; n /= 10L, i++) { + fBCD.bcdBytes.ptr[i] = static_cast(n % 10); + } + U_ASSERT(usingBytes); + scale = 0; + precision = i; + } else { + uint64_t result = 0L; + int i = 16; + for (; n != 0L; n /= 10L, i--) { + result = (result >> 4) + ((n % 10) << 60); + } + U_ASSERT(i >= 0); + U_ASSERT(!usingBytes); + fBCD.bcdLong = result >> (i * 4); + scale = 0; + precision = 16 - i; + } +} + +void DecimalQuantity::readDecNumberToBcd(const DecNum& decnum) { + const decNumber* dn = decnum.getRawDecNumber(); + if (dn->digits > 16) { + ensureCapacity(dn->digits); + for (int32_t i = 0; i < dn->digits; i++) { + fBCD.bcdBytes.ptr[i] = dn->lsu[i]; + } + } else { + uint64_t result = 0L; + for (int32_t i = 0; i < dn->digits; i++) { + result |= static_cast(dn->lsu[i]) << (4 * i); + } + fBCD.bcdLong = result; + } + scale = dn->exponent; + precision = dn->digits; +} + +void DecimalQuantity::readDoubleConversionToBcd( + const char* buffer, int32_t length, int32_t point) { + // NOTE: Despite the fact that double-conversion's API is called + // "DoubleToAscii", they actually use '0' (as opposed to u8'0'). + if (length > 16) { + ensureCapacity(length); + for (int32_t i = 0; i < length; i++) { + fBCD.bcdBytes.ptr[i] = buffer[length-i-1] - '0'; + } + } else { + uint64_t result = 0L; + for (int32_t i = 0; i < length; i++) { + result |= static_cast(buffer[length-i-1] - '0') << (4 * i); + } + fBCD.bcdLong = result; + } + scale = point - length; + precision = length; +} + +void DecimalQuantity::compact() { + if (usingBytes) { + int32_t delta = 0; + for (; delta < precision && fBCD.bcdBytes.ptr[delta] == 0; delta++); + if (delta == precision) { + // Number is zero + setBcdToZero(); + return; + } else { + // Remove trailing zeros + shiftRight(delta); + } + + // Compute precision + int32_t leading = precision - 1; + for (; leading >= 0 && fBCD.bcdBytes.ptr[leading] == 0; leading--); + precision = leading + 1; + + // Switch storage mechanism if possible + if (precision <= 16) { + switchStorage(); + } + + } else { + if (fBCD.bcdLong == 0L) { + // Number is zero + setBcdToZero(); + return; + } + + // Compact the number (remove trailing zeros) + // TODO: Use a more efficient algorithm here and below. There is a logarithmic one. + int32_t delta = 0; + for (; delta < precision && getDigitPos(delta) == 0; delta++); + fBCD.bcdLong >>= delta * 4; + scale += delta; + + // Compute precision + int32_t leading = precision - 1; + for (; leading >= 0 && getDigitPos(leading) == 0; leading--); + precision = leading + 1; + } +} + +void DecimalQuantity::ensureCapacity() { + ensureCapacity(40); +} + +void DecimalQuantity::ensureCapacity(int32_t capacity) { + if (capacity == 0) { return; } + int32_t oldCapacity = usingBytes ? fBCD.bcdBytes.len : 0; + if (!usingBytes) { + // TODO: There is nothing being done to check for memory allocation failures. + // TODO: Consider indexing by nybbles instead of bytes in C++, so that we can + // make these arrays half the size. + fBCD.bcdBytes.ptr = static_cast(uprv_malloc(capacity * sizeof(int8_t))); + fBCD.bcdBytes.len = capacity; + // Initialize the byte array to zeros (this is done automatically in Java) + uprv_memset(fBCD.bcdBytes.ptr, 0, capacity * sizeof(int8_t)); + } else if (oldCapacity < capacity) { + auto bcd1 = static_cast(uprv_malloc(capacity * 2 * sizeof(int8_t))); + uprv_memcpy(bcd1, fBCD.bcdBytes.ptr, oldCapacity * sizeof(int8_t)); + // Initialize the rest of the byte array to zeros (this is done automatically in Java) + uprv_memset(bcd1 + oldCapacity, 0, (capacity - oldCapacity) * sizeof(int8_t)); + uprv_free(fBCD.bcdBytes.ptr); + fBCD.bcdBytes.ptr = bcd1; + fBCD.bcdBytes.len = capacity * 2; + } + usingBytes = true; +} + +void DecimalQuantity::switchStorage() { + if (usingBytes) { + // Change from bytes to long + uint64_t bcdLong = 0L; + for (int i = precision - 1; i >= 0; i--) { + bcdLong <<= 4; + bcdLong |= fBCD.bcdBytes.ptr[i]; + } + uprv_free(fBCD.bcdBytes.ptr); + fBCD.bcdBytes.ptr = nullptr; + fBCD.bcdLong = bcdLong; + usingBytes = false; + } else { + // Change from long to bytes + // Copy the long into a local variable since it will get munged when we allocate the bytes + uint64_t bcdLong = fBCD.bcdLong; + ensureCapacity(); + for (int i = 0; i < precision; i++) { + fBCD.bcdBytes.ptr[i] = static_cast(bcdLong & 0xf); + bcdLong >>= 4; + } + U_ASSERT(usingBytes); + } +} + +void DecimalQuantity::copyBcdFrom(const DecimalQuantity &other) { + setBcdToZero(); + if (other.usingBytes) { + ensureCapacity(other.precision); + uprv_memcpy(fBCD.bcdBytes.ptr, other.fBCD.bcdBytes.ptr, other.precision * sizeof(int8_t)); + } else { + fBCD.bcdLong = other.fBCD.bcdLong; + } +} + +void DecimalQuantity::moveBcdFrom(DecimalQuantity &other) { + setBcdToZero(); + if (other.usingBytes) { + usingBytes = true; + fBCD.bcdBytes.ptr = other.fBCD.bcdBytes.ptr; + fBCD.bcdBytes.len = other.fBCD.bcdBytes.len; + // Take ownership away from the old instance: + other.fBCD.bcdBytes.ptr = nullptr; + other.usingBytes = false; + } else { + fBCD.bcdLong = other.fBCD.bcdLong; + } +} + +const char16_t* DecimalQuantity::checkHealth() const { + if (usingBytes) { + if (precision == 0) { return u"Zero precision but we are in byte mode"; } + int32_t capacity = fBCD.bcdBytes.len; + if (precision > capacity) { return u"Precision exceeds length of byte array"; } + if (getDigitPos(precision - 1) == 0) { return u"Most significant digit is zero in byte mode"; } + if (getDigitPos(0) == 0) { return u"Least significant digit is zero in long mode"; } + for (int i = 0; i < precision; i++) { + if (getDigitPos(i) >= 10) { return u"Digit exceeding 10 in byte array"; } + if (getDigitPos(i) < 0) { return u"Digit below 0 in byte array"; } + } + for (int i = precision; i < capacity; i++) { + if (getDigitPos(i) != 0) { return u"Nonzero digits outside of range in byte array"; } + } + } else { + if (precision == 0 && fBCD.bcdLong != 0) { + return u"Value in bcdLong even though precision is zero"; + } + if (precision > 16) { return u"Precision exceeds length of long"; } + if (precision != 0 && getDigitPos(precision - 1) == 0) { + return u"Most significant digit is zero in long mode"; + } + if (precision != 0 && getDigitPos(0) == 0) { + return u"Least significant digit is zero in long mode"; + } + for (int i = 0; i < precision; i++) { + if (getDigitPos(i) >= 10) { return u"Digit exceeding 10 in long"; } + if (getDigitPos(i) < 0) { return u"Digit below 0 in long (?!)"; } + } + for (int i = precision; i < 16; i++) { + if (getDigitPos(i) != 0) { return u"Nonzero digits outside of range in long"; } + } + } + + // No error + return nullptr; +} + +bool DecimalQuantity::operator==(const DecimalQuantity& other) const { + bool basicEquals = + scale == other.scale + && precision == other.precision + && flags == other.flags + && lReqPos == other.lReqPos + && rReqPos == other.rReqPos + && isApproximate == other.isApproximate; + if (!basicEquals) { + return false; + } + + if (precision == 0) { + return true; + } else if (isApproximate) { + return origDouble == other.origDouble && origDelta == other.origDelta; + } else { + for (int m = getUpperDisplayMagnitude(); m >= getLowerDisplayMagnitude(); m--) { + if (getDigit(m) != other.getDigit(m)) { + return false; + } + } + return true; + } +} + +UnicodeString DecimalQuantity::toString() const { + UErrorCode localStatus = U_ZERO_ERROR; + MaybeStackArray digits(precision + 1, localStatus); + if (U_FAILURE(localStatus)) { + return ICU_Utility::makeBogusString(); + } + for (int32_t i = 0; i < precision; i++) { + digits[i] = getDigitPos(precision - i - 1) + '0'; + } + digits[precision] = 0; // terminate buffer + char buffer8[100]; + snprintf( + buffer8, + sizeof(buffer8), + "", + lReqPos, + rReqPos, + (usingBytes ? "bytes" : "long"), + (isNegative() ? "-" : ""), + (precision == 0 ? "0" : digits.getAlias()), + "E", + scale); + return UnicodeString(buffer8, -1, US_INV); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_decimalquantity.h b/intl/icu/source/i18n/number_decimalquantity.h new file mode 100644 index 0000000000..2211fe9c19 --- /dev/null +++ b/intl/icu/source/i18n/number_decimalquantity.h @@ -0,0 +1,559 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_DECIMALQUANTITY_H__ +#define __NUMBER_DECIMALQUANTITY_H__ + +#include +#include "unicode/umachine.h" +#include "standardplural.h" +#include "plurrule_impl.h" +#include "number_types.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +// Forward-declare (maybe don't want number_utils.h included here): +class DecNum; + +/** + * A class for representing a number to be processed by the decimal formatting pipeline. Includes + * methods for rounding, plural rules, and decimal digit extraction. + * + *

    By design, this is NOT IMMUTABLE and NOT THREAD SAFE. It is intended to be an intermediate + * object holding state during a pass through the decimal formatting pipeline. + * + *

    Represents numbers and digit display properties using Binary Coded Decimal (BCD). + * + *

    Java has multiple implementations for testing, but C++ has only one implementation. + */ +class U_I18N_API DecimalQuantity : public IFixedDecimal, public UMemory { + public: + /** Copy constructor. */ + DecimalQuantity(const DecimalQuantity &other); + + /** Move constructor. */ + DecimalQuantity(DecimalQuantity &&src) noexcept; + + DecimalQuantity(); + + ~DecimalQuantity() override; + + /** + * Sets this instance to be equal to another instance. + * + * @param other The instance to copy from. + */ + DecimalQuantity &operator=(const DecimalQuantity &other); + + /** Move assignment */ + DecimalQuantity &operator=(DecimalQuantity&& src) noexcept; + + /** + * Sets the minimum integer digits that this {@link DecimalQuantity} should generate. + * This method does not perform rounding. + * + * @param minInt The minimum number of integer digits. + */ + void setMinInteger(int32_t minInt); + + /** + * Sets the minimum fraction digits that this {@link DecimalQuantity} should generate. + * This method does not perform rounding. + * + * @param minFrac The minimum number of fraction digits. + */ + void setMinFraction(int32_t minFrac); + + /** + * Truncates digits from the upper magnitude of the number in order to satisfy the + * specified maximum number of integer digits. + * + * @param maxInt The maximum number of integer digits. + */ + void applyMaxInteger(int32_t maxInt); + + /** + * Rounds the number to a specified interval, such as 0.05. + * + *

    If rounding to a power of ten, use the more efficient {@link #roundToMagnitude} instead. + * + * @param increment The increment to which to round. + * @param magnitude The power of 10 to which to round. + * @param roundingMode The {@link RoundingMode} to use if rounding is necessary. + */ + void roundToIncrement( + uint64_t increment, + digits_t magnitude, + RoundingMode roundingMode, + UErrorCode& status); + + /** Removes all fraction digits. */ + void truncate(); + + /** + * Rounds the number to the nearest multiple of 5 at the specified magnitude. + * For example, when magnitude == -2, this performs rounding to the nearest 0.05. + * + * @param magnitude The magnitude at which the digit should become either 0 or 5. + * @param roundingMode Rounding strategy. + */ + void roundToNickel(int32_t magnitude, RoundingMode roundingMode, UErrorCode& status); + + /** + * Rounds the number to a specified magnitude (power of ten). + * + * @param roundingMagnitude The power of ten to which to round. For example, a value of -2 will + * round to 2 decimal places. + * @param roundingMode The {@link RoundingMode} to use if rounding is necessary. + */ + void roundToMagnitude(int32_t magnitude, RoundingMode roundingMode, UErrorCode& status); + + /** + * Rounds the number to an infinite number of decimal points. This has no effect except for + * forcing the double in {@link DecimalQuantity_AbstractBCD} to adopt its exact representation. + */ + void roundToInfinity(); + + /** + * Multiply the internal value. Uses decNumber. + * + * @param multiplicand The value by which to multiply. + */ + void multiplyBy(const DecNum& multiplicand, UErrorCode& status); + + /** + * Divide the internal value. Uses decNumber. + * + * @param multiplicand The value by which to multiply. + */ + void divideBy(const DecNum& divisor, UErrorCode& status); + + /** Flips the sign from positive to negative and back. */ + void negate(); + + /** + * Scales the number by a power of ten. For example, if the value is currently "1234.56", calling + * this method with delta=-3 will change the value to "1.23456". + * + * @param delta The number of magnitudes of ten to change by. + * @return true if integer overflow occurred; false otherwise. + */ + bool adjustMagnitude(int32_t delta); + + /** + * Scales the number such that the least significant nonzero digit is at magnitude 0. + * + * @return The previous magnitude of the least significant digit. + */ + int32_t adjustToZeroScale(); + + /** + * @return The power of ten corresponding to the most significant nonzero digit. + * The number must not be zero. + */ + int32_t getMagnitude() const; + + /** + * @return The value of the (suppressed) exponent after the number has been + * put into a notation with exponents (ex: compact, scientific). Ex: given + * the number 1000 as "1K" / "1E3", the return value will be 3 (positive). + */ + int32_t getExponent() const; + + /** + * Adjusts the value for the (suppressed) exponent stored when using + * notation with exponents (ex: compact, scientific). + * + *

    Adjusting the exponent is decoupled from {@link #adjustMagnitude} in + * order to allow flexibility for {@link StandardPlural} to be selected in + * formatting (ex: for compact notation) either with or without the exponent + * applied in the value of the number. + * @param delta + * The value to adjust the exponent by. + */ + void adjustExponent(int32_t delta); + + /** + * Resets the DecimalQuantity to the value before adjustMagnitude and adjustExponent. + */ + void resetExponent(); + + /** + * @return Whether the value represented by this {@link DecimalQuantity} is + * zero, infinity, or NaN. + */ + bool isZeroish() const; + + /** @return Whether the value represented by this {@link DecimalQuantity} is less than zero. */ + bool isNegative() const; + + /** @return The appropriate value from the Signum enum. */ + Signum signum() const; + + /** @return Whether the value represented by this {@link DecimalQuantity} is infinite. */ + bool isInfinite() const override; + + /** @return Whether the value represented by this {@link DecimalQuantity} is not a number. */ + bool isNaN() const override; + + /** + * Note: this method incorporates the value of {@code exponent} + * (for cases such as compact notation) to return the proper long value + * represented by the result. + * @param truncateIfOverflow if false and the number does NOT fit, fails with an assertion error. + */ + int64_t toLong(bool truncateIfOverflow = false) const; + + /** + * Note: this method incorporates the value of {@code exponent} + * (for cases such as compact notation) to return the proper long value + * represented by the result. + */ + uint64_t toFractionLong(bool includeTrailingZeros) const; + + /** + * Returns whether or not a Long can fully represent the value stored in this DecimalQuantity. + * @param ignoreFraction if true, silently ignore digits after the decimal place. + */ + bool fitsInLong(bool ignoreFraction = false) const; + + /** @return The value contained in this {@link DecimalQuantity} approximated as a double. */ + double toDouble() const; + + /** Computes a DecNum representation of this DecimalQuantity, saving it to the output parameter. */ + DecNum& toDecNum(DecNum& output, UErrorCode& status) const; + + DecimalQuantity &setToInt(int32_t n); + + DecimalQuantity &setToLong(int64_t n); + + DecimalQuantity &setToDouble(double n); + + /** + * Produces a DecimalQuantity that was parsed from a string by the decNumber + * C Library. + * + * decNumber is similar to BigDecimal in Java, and supports parsing strings + * such as "123.456621E+40". + */ + DecimalQuantity &setToDecNumber(StringPiece n, UErrorCode& status); + + /** Internal method if the caller already has a DecNum. */ + DecimalQuantity &setToDecNum(const DecNum& n, UErrorCode& status); + + /** Returns a DecimalQuantity after parsing the input string. */ + static DecimalQuantity fromExponentString(UnicodeString n, UErrorCode& status); + + /** + * Appends a digit, optionally with one or more leading zeros, to the end of the value represented + * by this DecimalQuantity. + * + *

    The primary use of this method is to construct numbers during a parsing loop. It allows + * parsing to take advantage of the digit list infrastructure primarily designed for formatting. + * + * @param value The digit to append. + * @param leadingZeros The number of zeros to append before the digit. For example, if the value + * in this instance starts as 12.3, and you append a 4 with 1 leading zero, the value becomes + * 12.304. + * @param appendAsInteger If true, increase the magnitude of existing digits to make room for the + * new digit. If false, append to the end like a fraction digit. If true, there must not be + * any fraction digits already in the number. + * @internal + * @deprecated This API is ICU internal only. + */ + void appendDigit(int8_t value, int32_t leadingZeros, bool appendAsInteger); + + double getPluralOperand(PluralOperand operand) const override; + + bool hasIntegerValue() const override; + + /** + * Gets the digit at the specified magnitude. For example, if the represented number is 12.3, + * getDigit(-1) returns 3, since 3 is the digit corresponding to 10^-1. + * + * @param magnitude The magnitude of the digit. + * @return The digit at the specified magnitude. + */ + int8_t getDigit(int32_t magnitude) const; + + /** + * Gets the largest power of ten that needs to be displayed. The value returned by this function + * will be bounded between minInt and maxInt. + * + * @return The highest-magnitude digit to be displayed. + */ + int32_t getUpperDisplayMagnitude() const; + + /** + * Gets the smallest power of ten that needs to be displayed. The value returned by this function + * will be bounded between -minFrac and -maxFrac. + * + * @return The lowest-magnitude digit to be displayed. + */ + int32_t getLowerDisplayMagnitude() const; + + int32_t fractionCount() const; + + int32_t fractionCountWithoutTrailingZeros() const; + + void clear(); + + /** This method is for internal testing only. */ + uint64_t getPositionFingerprint() const; + +// /** +// * If the given {@link FieldPosition} is a {@link UFieldPosition}, populates it with the fraction +// * length and fraction long value. If the argument is not a {@link UFieldPosition}, nothing +// * happens. +// * +// * @param fp The {@link UFieldPosition} to populate. +// */ +// void populateUFieldPosition(FieldPosition fp); + + /** + * Checks whether the bytes stored in this instance are all valid. For internal unit testing only. + * + * @return An error message if this instance is invalid, or null if this instance is healthy. + */ + const char16_t* checkHealth() const; + + UnicodeString toString() const; + + /** Returns the string in standard exponential notation. */ + UnicodeString toScientificString() const; + + /** Returns the string without exponential notation. Slightly slower than toScientificString(). */ + UnicodeString toPlainString() const; + + /** Returns the string using ASCII digits and using exponential notation for non-zero + exponents, following the UTS 35 specification for plural rule samples. */ + UnicodeString toExponentString() const; + + /** Visible for testing */ + inline bool isUsingBytes() { return usingBytes; } + + /** Visible for testing */ + inline bool isExplicitExactDouble() { return explicitExactDouble; } + + bool operator==(const DecimalQuantity& other) const; + + inline bool operator!=(const DecimalQuantity& other) const { + return !(*this == other); + } + + /** + * Bogus flag for when a DecimalQuantity is stored on the stack. + */ + bool bogus = false; + + private: + /** + * The power of ten corresponding to the least significant digit in the BCD. For example, if this + * object represents the number "3.14", the BCD will be "0x314" and the scale will be -2. + * + *

    Note that in {@link java.math.BigDecimal}, the scale is defined differently: the number of + * digits after the decimal place, which is the negative of our definition of scale. + */ + int32_t scale; + + /** + * The number of digits in the BCD. For example, "1007" has BCD "0x1007" and precision 4. The + * maximum precision is 16 since a long can hold only 16 digits. + * + *

    This value must be re-calculated whenever the value in bcd changes by using {@link + * #computePrecisionAndCompact()}. + */ + int32_t precision; + + /** + * A bitmask of properties relating to the number represented by this object. + * + * @see #NEGATIVE_FLAG + * @see #INFINITY_FLAG + * @see #NAN_FLAG + */ + int8_t flags; + + // The following three fields relate to the double-to-ascii fast path algorithm. + // When a double is given to DecimalQuantityBCD, it is converted to using a fast algorithm. The + // fast algorithm guarantees correctness to only the first ~12 digits of the double. The process + // of rounding the number ensures that the converted digits are correct, falling back to a slow- + // path algorithm if required. Therefore, if a DecimalQuantity is constructed from a double, it + // is *required* that roundToMagnitude(), roundToIncrement(), or roundToInfinity() is called. If + // you don't round, assertions will fail in certain other methods if you try calling them. + + /** + * Whether the value in the BCD comes from the double fast path without having been rounded to + * ensure correctness + */ + UBool isApproximate; + + /** + * The original number provided by the user and which is represented in BCD. Used when we need to + * re-compute the BCD for an exact double representation. + */ + double origDouble; + + /** + * The change in magnitude relative to the original double. Used when we need to re-compute the + * BCD for an exact double representation. + */ + int32_t origDelta; + + // Positions to keep track of leading and trailing zeros. + // lReqPos is the magnitude of the first required leading zero. + // rReqPos is the magnitude of the last required trailing zero. + int32_t lReqPos = 0; + int32_t rReqPos = 0; + + // The value of the (suppressed) exponent after the number has been put into + // a notation with exponents (ex: compact, scientific). + int32_t exponent = 0; + + /** + * The BCD of the 16 digits of the number represented by this object. Every 4 bits of the long map + * to one digit. For example, the number "12345" in BCD is "0x12345". + * + *

    Whenever bcd changes internally, {@link #compact()} must be called, except in special cases + * like setting the digit to zero. + */ + union { + struct { + int8_t *ptr; + int32_t len; + } bcdBytes; + uint64_t bcdLong; + } fBCD; + + bool usingBytes = false; + + /** + * Whether this {@link DecimalQuantity} has been explicitly converted to an exact double. true if + * backed by a double that was explicitly converted via convertToAccurateDouble; false otherwise. + * Used for testing. + */ + bool explicitExactDouble = false; + + void roundToMagnitude(int32_t magnitude, RoundingMode roundingMode, bool nickel, UErrorCode& status); + + /** + * Returns a single digit from the BCD list. No internal state is changed by calling this method. + * + * @param position The position of the digit to pop, counted in BCD units from the least + * significant digit. If outside the range supported by the implementation, zero is returned. + * @return The digit at the specified location. + */ + int8_t getDigitPos(int32_t position) const; + + /** + * Sets the digit in the BCD list. This method only sets the digit; it is the caller's + * responsibility to call {@link #compact} after setting the digit, and to ensure + * that the precision field is updated to reflect the correct number of digits if a + * nonzero digit is added to the decimal. + * + * @param position The position of the digit to pop, counted in BCD units from the least + * significant digit. If outside the range supported by the implementation, an AssertionError + * is thrown. + * @param value The digit to set at the specified location. + */ + void setDigitPos(int32_t position, int8_t value); + + /** + * Adds zeros to the end of the BCD list. This will result in an invalid BCD representation; it is + * the caller's responsibility to do further manipulation and then call {@link #compact}. + * + * @param numDigits The number of zeros to add. + */ + void shiftLeft(int32_t numDigits); + + /** + * Directly removes digits from the end of the BCD list. + * Updates the scale and precision. + * + * CAUTION: it is the caller's responsibility to call {@link #compact} after this method. + */ + void shiftRight(int32_t numDigits); + + /** + * Directly removes digits from the front of the BCD list. + * Updates precision. + * + * CAUTION: it is the caller's responsibility to call {@link #compact} after this method. + */ + void popFromLeft(int32_t numDigits); + + /** + * Sets the internal representation to zero. Clears any values stored in scale, precision, + * hasDouble, origDouble, origDelta, exponent, and BCD data. + */ + void setBcdToZero(); + + /** + * Sets the internal BCD state to represent the value in the given int. The int is guaranteed to + * be either positive. The internal state is guaranteed to be empty when this method is called. + * + * @param n The value to consume. + */ + void readIntToBcd(int32_t n); + + /** + * Sets the internal BCD state to represent the value in the given long. The long is guaranteed to + * be either positive. The internal state is guaranteed to be empty when this method is called. + * + * @param n The value to consume. + */ + void readLongToBcd(int64_t n); + + void readDecNumberToBcd(const DecNum& dn); + + void readDoubleConversionToBcd(const char* buffer, int32_t length, int32_t point); + + void copyFieldsFrom(const DecimalQuantity& other); + + void copyBcdFrom(const DecimalQuantity &other); + + void moveBcdFrom(DecimalQuantity& src); + + /** + * Removes trailing zeros from the BCD (adjusting the scale as required) and then computes the + * precision. The precision is the number of digits in the number up through the greatest nonzero + * digit. + * + *

    This method must always be called when bcd changes in order for assumptions to be correct in + * methods like {@link #fractionCount()}. + */ + void compact(); + + void _setToInt(int32_t n); + + void _setToLong(int64_t n); + + void _setToDoubleFast(double n); + + void _setToDecNum(const DecNum& dn, UErrorCode& status); + + static int32_t getVisibleFractionCount(UnicodeString value); + + void convertToAccurateDouble(); + + /** Ensure that a byte array of at least 40 digits is allocated. */ + void ensureCapacity(); + + void ensureCapacity(int32_t capacity); + + /** Switches the internal storage mechanism between the 64-bit long and the byte array. */ + void switchStorage(); +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_DECIMALQUANTITY_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_decimfmtprops.cpp b/intl/icu/source/i18n/number_decimfmtprops.cpp new file mode 100644 index 0000000000..6dbfc69ec8 --- /dev/null +++ b/intl/icu/source/i18n/number_decimfmtprops.cpp @@ -0,0 +1,154 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "number_decimfmtprops.h" +#include "umutex.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +namespace { + +alignas(DecimalFormatProperties) +char kRawDefaultProperties[sizeof(DecimalFormatProperties)]; + +icu::UInitOnce gDefaultPropertiesInitOnce {}; + +void U_CALLCONV initDefaultProperties(UErrorCode&) { + // can't fail, uses placement new into statically allocated space. + new(kRawDefaultProperties) DecimalFormatProperties(); // set to the default instance +} + +} + + +DecimalFormatProperties::DecimalFormatProperties() { + clear(); +} + +void DecimalFormatProperties::clear() { + compactStyle.nullify(); + currency.nullify(); + currencyPluralInfo.fPtr.adoptInstead(nullptr); + currencyUsage.nullify(); + decimalPatternMatchRequired = false; + decimalSeparatorAlwaysShown = false; + exponentSignAlwaysShown = false; + currencyAsDecimal = false; + formatFailIfMoreThanMaxDigits = false; + formatWidth = -1; + groupingSize = -1; + groupingUsed = true; + magnitudeMultiplier = 0; + maximumFractionDigits = -1; + maximumIntegerDigits = -1; + maximumSignificantDigits = -1; + minimumExponentDigits = -1; + minimumFractionDigits = -1; + minimumGroupingDigits = -1; + minimumIntegerDigits = -1; + minimumSignificantDigits = -1; + multiplier = 1; + multiplierScale = 0; + negativePrefix.setToBogus(); + negativePrefixPattern.setToBogus(); + negativeSuffix.setToBogus(); + negativeSuffixPattern.setToBogus(); + padPosition.nullify(); + padString.setToBogus(); + parseCaseSensitive = false; + parseIntegerOnly = false; + parseMode.nullify(); + parseNoExponent = false; + parseToBigDecimal = false; + parseAllInput = UNUM_MAYBE; + positivePrefix.setToBogus(); + positivePrefixPattern.setToBogus(); + positiveSuffix.setToBogus(); + positiveSuffixPattern.setToBogus(); + roundingIncrement = 0.0; + roundingMode.nullify(); + secondaryGroupingSize = -1; + signAlwaysShown = false; +} + +bool +DecimalFormatProperties::_equals(const DecimalFormatProperties& other, bool ignoreForFastFormat) const { + bool eq = true; + + // Properties that must be equal both normally and for fast-path formatting + eq = eq && compactStyle == other.compactStyle; + eq = eq && currency == other.currency; + eq = eq && currencyPluralInfo.fPtr.getAlias() == other.currencyPluralInfo.fPtr.getAlias(); + eq = eq && currencyUsage == other.currencyUsage; + eq = eq && decimalSeparatorAlwaysShown == other.decimalSeparatorAlwaysShown; + eq = eq && exponentSignAlwaysShown == other.exponentSignAlwaysShown; + eq = eq && currencyAsDecimal == other.currencyAsDecimal; + eq = eq && formatFailIfMoreThanMaxDigits == other.formatFailIfMoreThanMaxDigits; + eq = eq && formatWidth == other.formatWidth; + eq = eq && magnitudeMultiplier == other.magnitudeMultiplier; + eq = eq && maximumSignificantDigits == other.maximumSignificantDigits; + eq = eq && minimumExponentDigits == other.minimumExponentDigits; + eq = eq && minimumGroupingDigits == other.minimumGroupingDigits; + eq = eq && minimumSignificantDigits == other.minimumSignificantDigits; + eq = eq && multiplier == other.multiplier; + eq = eq && multiplierScale == other.multiplierScale; + eq = eq && negativePrefix == other.negativePrefix; + eq = eq && negativeSuffix == other.negativeSuffix; + eq = eq && padPosition == other.padPosition; + eq = eq && padString == other.padString; + eq = eq && positivePrefix == other.positivePrefix; + eq = eq && positiveSuffix == other.positiveSuffix; + eq = eq && roundingIncrement == other.roundingIncrement; + eq = eq && roundingMode == other.roundingMode; + eq = eq && secondaryGroupingSize == other.secondaryGroupingSize; + eq = eq && signAlwaysShown == other.signAlwaysShown; + + if (ignoreForFastFormat) { + return eq; + } + + // Properties ignored by fast-path formatting + // Formatting (special handling required): + eq = eq && groupingSize == other.groupingSize; + eq = eq && groupingUsed == other.groupingUsed; + eq = eq && minimumFractionDigits == other.minimumFractionDigits; + eq = eq && maximumFractionDigits == other.maximumFractionDigits; + eq = eq && maximumIntegerDigits == other.maximumIntegerDigits; + eq = eq && minimumIntegerDigits == other.minimumIntegerDigits; + eq = eq && negativePrefixPattern == other.negativePrefixPattern; + eq = eq && negativeSuffixPattern == other.negativeSuffixPattern; + eq = eq && positivePrefixPattern == other.positivePrefixPattern; + eq = eq && positiveSuffixPattern == other.positiveSuffixPattern; + + // Parsing (always safe to ignore): + eq = eq && decimalPatternMatchRequired == other.decimalPatternMatchRequired; + eq = eq && parseCaseSensitive == other.parseCaseSensitive; + eq = eq && parseIntegerOnly == other.parseIntegerOnly; + eq = eq && parseMode == other.parseMode; + eq = eq && parseNoExponent == other.parseNoExponent; + eq = eq && parseToBigDecimal == other.parseToBigDecimal; + eq = eq && parseAllInput == other.parseAllInput; + + return eq; +} + +bool DecimalFormatProperties::equalsDefaultExceptFastFormat() const { + UErrorCode localStatus = U_ZERO_ERROR; + umtx_initOnce(gDefaultPropertiesInitOnce, &initDefaultProperties, localStatus); + return _equals(*reinterpret_cast(kRawDefaultProperties), true); +} + +const DecimalFormatProperties& DecimalFormatProperties::getDefault() { + UErrorCode localStatus = U_ZERO_ERROR; + umtx_initOnce(gDefaultPropertiesInitOnce, &initDefaultProperties, localStatus); + return *reinterpret_cast(kRawDefaultProperties); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_decimfmtprops.h b/intl/icu/source/i18n/number_decimfmtprops.h new file mode 100644 index 0000000000..5f72f64984 --- /dev/null +++ b/intl/icu/source/i18n/number_decimfmtprops.h @@ -0,0 +1,176 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_DECIMFMTPROPS_H__ +#define __NUMBER_DECIMFMTPROPS_H__ + +#include "unicode/unistr.h" +#include +#include "unicode/plurrule.h" +#include "unicode/currpinf.h" +#include "unicode/unum.h" +#include "unicode/localpointer.h" +#include "number_types.h" + +U_NAMESPACE_BEGIN + +// Export an explicit template instantiation of the LocalPointer that is used as a +// data member of CurrencyPluralInfoWrapper. +// (When building DLLs for Windows this is required.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#if defined(_MSC_VER) +// Ignore warning 4661 as LocalPointerBase does not use operator== or operator!= +#pragma warning(push) +#pragma warning(disable: 4661) +#endif +template class U_I18N_API LocalPointerBase; +template class U_I18N_API LocalPointer; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + +namespace number { +namespace impl { + +// Exported as U_I18N_API because it is a public member field of exported DecimalFormatProperties +// Using this wrapper is rather unfortunate, but is needed on Windows platforms in order to allow +// for DLL-exporting a fully specified template instantiation. +class U_I18N_API CurrencyPluralInfoWrapper { +public: + LocalPointer fPtr; + + CurrencyPluralInfoWrapper() = default; + + CurrencyPluralInfoWrapper(const CurrencyPluralInfoWrapper& other) { + if (!other.fPtr.isNull()) { + fPtr.adoptInstead(new CurrencyPluralInfo(*other.fPtr)); + } + } + + CurrencyPluralInfoWrapper& operator=(const CurrencyPluralInfoWrapper& other) { + if (this != &other && // self-assignment: no-op + !other.fPtr.isNull()) { + fPtr.adoptInstead(new CurrencyPluralInfo(*other.fPtr)); + } + return *this; + } +}; + +/** Controls the set of rules for parsing a string from the old DecimalFormat API. */ +enum ParseMode { + /** + * Lenient mode should be used if you want to accept malformed user input. It will use heuristics + * to attempt to parse through typographical errors in the string. + */ + PARSE_MODE_LENIENT, + + /** + * Strict mode should be used if you want to require that the input is well-formed. More + * specifically, it differs from lenient mode in the following ways: + * + *

      + *
    • Grouping widths must match the grouping settings. For example, "12,3,45" will fail if the + * grouping width is 3, as in the pattern "#,##0". + *
    • The string must contain a complete prefix and suffix. For example, if the pattern is + * "{#};(#)", then "{123}" or "(123)" would match, but "{123", "123}", and "123" would all fail. + * (The latter strings would be accepted in lenient mode.) + *
    • Whitespace may not appear at arbitrary places in the string. In lenient mode, whitespace + * is allowed to occur arbitrarily before and after prefixes and exponent separators. + *
    • Leading grouping separators are not allowed, as in ",123". + *
    • Minus and plus signs can only appear if specified in the pattern. In lenient mode, a plus + * or minus sign can always precede a number. + *
    • The set of characters that can be interpreted as a decimal or grouping separator is + * smaller. + *
    • If currency parsing is enabled, currencies must only appear where + * specified in either the current pattern string or in a valid pattern string for the current + * locale. For example, if the pattern is "¤0.00", then "$1.23" would match, but "1.23$" would + * fail to match. + *
    + */ + PARSE_MODE_STRICT, +}; + +// Exported as U_I18N_API because it is needed for the unit test PatternStringTest +struct U_I18N_API DecimalFormatProperties : public UMemory { + + public: + NullableValue compactStyle; + NullableValue currency; + CurrencyPluralInfoWrapper currencyPluralInfo; + NullableValue currencyUsage; + bool decimalPatternMatchRequired; + bool decimalSeparatorAlwaysShown; + bool exponentSignAlwaysShown; + bool currencyAsDecimal; + bool formatFailIfMoreThanMaxDigits; // ICU4C-only + int32_t formatWidth; + int32_t groupingSize; + bool groupingUsed; + int32_t magnitudeMultiplier; // internal field like multiplierScale but separate to avoid conflict + int32_t maximumFractionDigits; + int32_t maximumIntegerDigits; + int32_t maximumSignificantDigits; + int32_t minimumExponentDigits; + int32_t minimumFractionDigits; + int32_t minimumGroupingDigits; + int32_t minimumIntegerDigits; + int32_t minimumSignificantDigits; + int32_t multiplier; + int32_t multiplierScale; // ICU4C-only + UnicodeString negativePrefix; + UnicodeString negativePrefixPattern; + UnicodeString negativeSuffix; + UnicodeString negativeSuffixPattern; + NullableValue padPosition; + UnicodeString padString; + bool parseCaseSensitive; + bool parseIntegerOnly; + NullableValue parseMode; + bool parseNoExponent; + bool parseToBigDecimal; // TODO: Not needed in ICU4C? + UNumberFormatAttributeValue parseAllInput; // ICU4C-only + //PluralRules pluralRules; + UnicodeString positivePrefix; + UnicodeString positivePrefixPattern; + UnicodeString positiveSuffix; + UnicodeString positiveSuffixPattern; + double roundingIncrement; + NullableValue roundingMode; + int32_t secondaryGroupingSize; + bool signAlwaysShown; + + DecimalFormatProperties(); + + inline bool operator==(const DecimalFormatProperties& other) const { + return _equals(other, false); + } + + void clear(); + + /** + * Checks for equality to the default DecimalFormatProperties, but ignores the prescribed set of + * options for fast-path formatting. + */ + bool equalsDefaultExceptFastFormat() const; + + /** + * Returns the default DecimalFormatProperties instance. + */ + static const DecimalFormatProperties& getDefault(); + + private: + bool _equals(const DecimalFormatProperties& other, bool ignoreForFastFormat) const; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_DECIMFMTPROPS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_decnum.h b/intl/icu/source/i18n/number_decnum.h new file mode 100644 index 0000000000..94a0b31bcb --- /dev/null +++ b/intl/icu/source/i18n/number_decnum.h @@ -0,0 +1,94 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_DECNUM_H__ +#define __NUMBER_DECNUM_H__ + +#include "decNumber.h" +#include "charstr.h" +#include "bytesinkutil.h" + +U_NAMESPACE_BEGIN + +#define DECNUM_INITIAL_CAPACITY 34 + +// Export an explicit template instantiation of the MaybeStackHeaderAndArray that is used as a data member of DecNum. +// When building DLLs for Windows this is required even though no direct access to the MaybeStackHeaderAndArray leaks out of the i18n library. +// (See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackHeaderAndArray; +#endif + +namespace number { +namespace impl { + +/** A very thin C++ wrapper around decNumber.h */ +// Exported as U_I18N_API for tests +class U_I18N_API DecNum : public UMemory { + public: + DecNum(); // leaves object in valid but undefined state + + // Copy-like constructor; use the default move operators. + DecNum(const DecNum& other, UErrorCode& status); + + /** Sets the decNumber to the StringPiece. */ + void setTo(StringPiece str, UErrorCode& status); + + /** Sets the decNumber to the NUL-terminated char string. */ + void setTo(const char* str, UErrorCode& status); + + /** Uses double_conversion to set this decNumber to the given double. */ + void setTo(double d, UErrorCode& status); + + /** Sets the decNumber to the BCD representation. */ + void setTo(const uint8_t* bcd, int32_t length, int32_t scale, bool isNegative, UErrorCode& status); + + void normalize(); + + void multiplyBy(const DecNum& rhs, UErrorCode& status); + + void divideBy(const DecNum& rhs, UErrorCode& status); + + bool isNegative() const; + + bool isZero() const; + + /** Is infinity or NaN */ + bool isSpecial() const; + + bool isInfinity() const; + + bool isNaN() const; + + void toString(ByteSink& output, UErrorCode& status) const; + + inline CharString toCharString(UErrorCode& status) const { + CharString cstr; + CharStringByteSink sink(&cstr); + toString(sink, status); + return cstr; + } + + inline const decNumber* getRawDecNumber() const { + return fData.getAlias(); + } + + private: + static constexpr int32_t kDefaultDigits = DECNUM_INITIAL_CAPACITY; + MaybeStackHeaderAndArray fData; + decContext fContext; + + void _setTo(const char* str, int32_t maxDigits, UErrorCode& status); +}; + +} // namespace impl +} // namespace number + +U_NAMESPACE_END + +#endif // __NUMBER_DECNUM_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_fluent.cpp b/intl/icu/source/i18n/number_fluent.cpp new file mode 100644 index 0000000000..45d6b06c6d --- /dev/null +++ b/intl/icu/source/i18n/number_fluent.cpp @@ -0,0 +1,738 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "uassert.h" +#include "unicode/numberformatter.h" +#include "number_decimalquantity.h" +#include "number_formatimpl.h" +#include "umutex.h" +#include "number_asformat.h" +#include "number_utils.h" +#include "number_utypes.h" +#include "number_mapper.h" +#include "util.h" +#include "fphdlimp.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +#if (U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN) && defined(_MSC_VER) +// Ignore MSVC warning 4661. This is generated for NumberFormatterSettings<>::toSkeleton() as this method +// is defined elsewhere (in number_skeletons.cpp). The compiler is warning that the explicit template instantiation +// inside this single translation unit (CPP file) is incomplete, and thus it isn't sure if the template class is +// fully defined. However, since each translation unit explicitly instantiates all the necessary template classes, +// they will all be passed to the linker, and the linker will still find and export all the class members. +#pragma warning(push) +#pragma warning(disable: 4661) +#endif + +template +Derived NumberFormatterSettings::notation(const Notation& notation) const& { + Derived copy(*this); + // NOTE: Slicing is OK. + copy.fMacros.notation = notation; + return copy; +} + +template +Derived NumberFormatterSettings::notation(const Notation& notation)&& { + Derived move(std::move(*this)); + // NOTE: Slicing is OK. + move.fMacros.notation = notation; + return move; +} + +template +Derived NumberFormatterSettings::unit(const icu::MeasureUnit& unit) const& { + Derived copy(*this); + // NOTE: Slicing occurs here. However, CurrencyUnit can be restored from MeasureUnit. + // TimeUnit may be affected, but TimeUnit is not as relevant to number formatting. + copy.fMacros.unit = unit; + return copy; +} + +template +Derived NumberFormatterSettings::unit(const icu::MeasureUnit& unit)&& { + Derived move(std::move(*this)); + // See comments above about slicing. + move.fMacros.unit = unit; + return move; +} + +template +Derived NumberFormatterSettings::adoptUnit(icu::MeasureUnit* unit) const& { + Derived copy(*this); + // Just move the unit into the MacroProps by value, and delete it since we have ownership. + // NOTE: Slicing occurs here. However, CurrencyUnit can be restored from MeasureUnit. + // TimeUnit may be affected, but TimeUnit is not as relevant to number formatting. + if (unit != nullptr) { + // TODO: On nullptr, reset to default value? + copy.fMacros.unit = std::move(*unit); + delete unit; + } + return copy; +} + +template +Derived NumberFormatterSettings::adoptUnit(icu::MeasureUnit* unit)&& { + Derived move(std::move(*this)); + // See comments above about slicing and ownership. + if (unit != nullptr) { + // TODO: On nullptr, reset to default value? + move.fMacros.unit = std::move(*unit); + delete unit; + } + return move; +} + +template +Derived NumberFormatterSettings::perUnit(const icu::MeasureUnit& perUnit) const& { + Derived copy(*this); + // See comments above about slicing. + copy.fMacros.perUnit = perUnit; + return copy; +} + +template +Derived NumberFormatterSettings::perUnit(const icu::MeasureUnit& perUnit)&& { + Derived move(std::move(*this)); + // See comments above about slicing. + move.fMacros.perUnit = perUnit; + return move; +} + +template +Derived NumberFormatterSettings::adoptPerUnit(icu::MeasureUnit* perUnit) const& { + Derived copy(*this); + // See comments above about slicing and ownership. + if (perUnit != nullptr) { + // TODO: On nullptr, reset to default value? + copy.fMacros.perUnit = std::move(*perUnit); + delete perUnit; + } + return copy; +} + +template +Derived NumberFormatterSettings::adoptPerUnit(icu::MeasureUnit* perUnit)&& { + Derived move(std::move(*this)); + // See comments above about slicing and ownership. + if (perUnit != nullptr) { + // TODO: On nullptr, reset to default value? + move.fMacros.perUnit = std::move(*perUnit); + delete perUnit; + } + return move; +} + +template +Derived NumberFormatterSettings::precision(const Precision& precision) const& { + Derived copy(*this); + // NOTE: Slicing is OK. + copy.fMacros.precision = precision; + return copy; +} + +template +Derived NumberFormatterSettings::precision(const Precision& precision)&& { + Derived move(std::move(*this)); + // NOTE: Slicing is OK. + move.fMacros.precision = precision; + return move; +} + +template +Derived NumberFormatterSettings::roundingMode(UNumberFormatRoundingMode roundingMode) const& { + Derived copy(*this); + copy.fMacros.roundingMode = roundingMode; + return copy; +} + +template +Derived NumberFormatterSettings::roundingMode(UNumberFormatRoundingMode roundingMode)&& { + Derived move(std::move(*this)); + move.fMacros.roundingMode = roundingMode; + return move; +} + +template +Derived NumberFormatterSettings::grouping(UNumberGroupingStrategy strategy) const& { + Derived copy(*this); + // NOTE: This is slightly different than how the setting is stored in Java + // because we want to put it on the stack. + copy.fMacros.grouper = Grouper::forStrategy(strategy); + return copy; +} + +template +Derived NumberFormatterSettings::grouping(UNumberGroupingStrategy strategy)&& { + Derived move(std::move(*this)); + move.fMacros.grouper = Grouper::forStrategy(strategy); + return move; +} + +template +Derived NumberFormatterSettings::integerWidth(const IntegerWidth& style) const& { + Derived copy(*this); + copy.fMacros.integerWidth = style; + return copy; +} + +template +Derived NumberFormatterSettings::integerWidth(const IntegerWidth& style)&& { + Derived move(std::move(*this)); + move.fMacros.integerWidth = style; + return move; +} + +template +Derived NumberFormatterSettings::symbols(const DecimalFormatSymbols& symbols) const& { + Derived copy(*this); + copy.fMacros.symbols.setTo(symbols); + return copy; +} + +template +Derived NumberFormatterSettings::symbols(const DecimalFormatSymbols& symbols)&& { + Derived move(std::move(*this)); + move.fMacros.symbols.setTo(symbols); + return move; +} + +template +Derived NumberFormatterSettings::adoptSymbols(NumberingSystem* ns) const& { + Derived copy(*this); + copy.fMacros.symbols.setTo(ns); + return copy; +} + +template +Derived NumberFormatterSettings::adoptSymbols(NumberingSystem* ns)&& { + Derived move(std::move(*this)); + move.fMacros.symbols.setTo(ns); + return move; +} + +template +Derived NumberFormatterSettings::unitWidth(UNumberUnitWidth width) const& { + Derived copy(*this); + copy.fMacros.unitWidth = width; + return copy; +} + +template +Derived NumberFormatterSettings::unitWidth(UNumberUnitWidth width)&& { + Derived move(std::move(*this)); + move.fMacros.unitWidth = width; + return move; +} + +template +Derived NumberFormatterSettings::sign(UNumberSignDisplay style) const& { + Derived copy(*this); + copy.fMacros.sign = style; + return copy; +} + +template +Derived NumberFormatterSettings::sign(UNumberSignDisplay style)&& { + Derived move(std::move(*this)); + move.fMacros.sign = style; + return move; +} + +template +Derived NumberFormatterSettings::decimal(UNumberDecimalSeparatorDisplay style) const& { + Derived copy(*this); + copy.fMacros.decimal = style; + return copy; +} + +template +Derived NumberFormatterSettings::decimal(UNumberDecimalSeparatorDisplay style)&& { + Derived move(std::move(*this)); + move.fMacros.decimal = style; + return move; +} + +template +Derived NumberFormatterSettings::scale(const Scale& scale) const& { + Derived copy(*this); + copy.fMacros.scale = scale; + return copy; +} + +template +Derived NumberFormatterSettings::scale(const Scale& scale)&& { + Derived move(std::move(*this)); + move.fMacros.scale = scale; + return move; +} + +template +Derived NumberFormatterSettings::usage(const StringPiece usage) const& { + Derived copy(*this); + copy.fMacros.usage.set(usage); + return copy; +} + +template +Derived NumberFormatterSettings::usage(const StringPiece usage)&& { + Derived move(std::move(*this)); + move.fMacros.usage.set(usage); + return move; +} + +template +Derived NumberFormatterSettings::displayOptions(const DisplayOptions &displayOptions) const & { + Derived copy(*this); + // `displayCase` does not recognise the `undefined` + if (displayOptions.getGrammaticalCase() == UDISPOPT_GRAMMATICAL_CASE_UNDEFINED) { + copy.fMacros.unitDisplayCase.set(nullptr); + return copy; + } + + copy.fMacros.unitDisplayCase.set( + udispopt_getGrammaticalCaseIdentifier(displayOptions.getGrammaticalCase())); + return copy; +} + +template +Derived NumberFormatterSettings::displayOptions(const DisplayOptions &displayOptions) && { + Derived move(std::move(*this)); + // `displayCase` does not recognise the `undefined` + if (displayOptions.getGrammaticalCase() == UDISPOPT_GRAMMATICAL_CASE_UNDEFINED) { + move.fMacros.unitDisplayCase.set(nullptr); + return move; + } + + move.fMacros.unitDisplayCase.set( + udispopt_getGrammaticalCaseIdentifier(displayOptions.getGrammaticalCase())); + return move; +} + +template +Derived NumberFormatterSettings::unitDisplayCase(const StringPiece unitDisplayCase) const& { + Derived copy(*this); + copy.fMacros.unitDisplayCase.set(unitDisplayCase); + return copy; +} + +template +Derived NumberFormatterSettings::unitDisplayCase(const StringPiece unitDisplayCase)&& { + Derived move(std::move(*this)); + move.fMacros.unitDisplayCase.set(unitDisplayCase); + return move; +} + +template +Derived NumberFormatterSettings::padding(const Padder& padder) const& { + Derived copy(*this); + copy.fMacros.padder = padder; + return copy; +} + +template +Derived NumberFormatterSettings::padding(const Padder& padder)&& { + Derived move(std::move(*this)); + move.fMacros.padder = padder; + return move; +} + +template +Derived NumberFormatterSettings::threshold(int32_t threshold) const& { + Derived copy(*this); + copy.fMacros.threshold = threshold; + return copy; +} + +template +Derived NumberFormatterSettings::threshold(int32_t threshold)&& { + Derived move(std::move(*this)); + move.fMacros.threshold = threshold; + return move; +} + +template +Derived NumberFormatterSettings::macros(const impl::MacroProps& macros) const& { + Derived copy(*this); + copy.fMacros = macros; + return copy; +} + +template +Derived NumberFormatterSettings::macros(const impl::MacroProps& macros)&& { + Derived move(std::move(*this)); + move.fMacros = macros; + return move; +} + +template +Derived NumberFormatterSettings::macros(impl::MacroProps&& macros) const& { + Derived copy(*this); + copy.fMacros = std::move(macros); + return copy; +} + +template +Derived NumberFormatterSettings::macros(impl::MacroProps&& macros)&& { + Derived move(std::move(*this)); + move.fMacros = std::move(macros); + return move; +} + +// Note: toSkeleton defined in number_skeletons.cpp + +template +LocalPointer NumberFormatterSettings::clone() const & { + return LocalPointer(new Derived(*this)); +} + +template +LocalPointer NumberFormatterSettings::clone() && { + return LocalPointer(new Derived(std::move(*this))); +} + +// Declare all classes that implement NumberFormatterSettings +// See https://stackoverflow.com/a/495056/1407170 +template +class icu::number::NumberFormatterSettings; +template +class icu::number::NumberFormatterSettings; + + +UnlocalizedNumberFormatter NumberFormatter::with() { + UnlocalizedNumberFormatter result; + return result; +} + +LocalizedNumberFormatter NumberFormatter::withLocale(const Locale& locale) { + return with().locale(locale); +} + +// Note: forSkeleton defined in number_skeletons.cpp + + +template using NFS = NumberFormatterSettings; +using LNF = LocalizedNumberFormatter; +using UNF = UnlocalizedNumberFormatter; + +UnlocalizedNumberFormatter::UnlocalizedNumberFormatter(const UNF& other) + : UNF(static_cast&>(other)) {} + +UnlocalizedNumberFormatter::UnlocalizedNumberFormatter(const NFS& other) + : NFS(other) { + // No additional fields to assign +} + +// Make default copy constructor call the NumberFormatterSettings copy constructor. +UnlocalizedNumberFormatter::UnlocalizedNumberFormatter(UNF&& src) noexcept + : UNF(static_cast&&>(src)) {} + +UnlocalizedNumberFormatter::UnlocalizedNumberFormatter(NFS&& src) noexcept + : NFS(std::move(src)) { + // No additional fields to assign +} + +UnlocalizedNumberFormatter& UnlocalizedNumberFormatter::operator=(const UNF& other) { + NFS::operator=(static_cast&>(other)); + // No additional fields to assign + return *this; +} + +UnlocalizedNumberFormatter& UnlocalizedNumberFormatter::operator=(UNF&& src) noexcept { + NFS::operator=(static_cast&&>(src)); + // No additional fields to assign + return *this; +} + +// Make default copy constructor call the NumberFormatterSettings copy constructor. +LocalizedNumberFormatter::LocalizedNumberFormatter(const LNF& other) + : LNF(static_cast&>(other)) {} + +LocalizedNumberFormatter::LocalizedNumberFormatter(const NFS& other) + : NFS(other) { + UErrorCode localStatus = U_ZERO_ERROR; // Can't bubble up the error + lnfCopyHelper(static_cast(other), localStatus); +} + +LocalizedNumberFormatter::LocalizedNumberFormatter(LocalizedNumberFormatter&& src) noexcept + : LNF(static_cast&&>(src)) {} + +LocalizedNumberFormatter::LocalizedNumberFormatter(NFS&& src) noexcept + : NFS(std::move(src)) { + lnfMoveHelper(std::move(static_cast(src))); +} + +LocalizedNumberFormatter& LocalizedNumberFormatter::operator=(const LNF& other) { + if (this == &other) { return *this; } // self-assignment: no-op + NFS::operator=(static_cast&>(other)); + UErrorCode localStatus = U_ZERO_ERROR; // Can't bubble up the error + lnfCopyHelper(other, localStatus); + return *this; +} + +LocalizedNumberFormatter& LocalizedNumberFormatter::operator=(LNF&& src) noexcept { + NFS::operator=(static_cast&&>(src)); + lnfMoveHelper(std::move(src)); + return *this; +} + +void LocalizedNumberFormatter::resetCompiled() { + auto* callCount = reinterpret_cast(fUnsafeCallCount); + umtx_storeRelease(*callCount, 0); + fCompiled = nullptr; +} + +void LocalizedNumberFormatter::lnfMoveHelper(LNF&& src) { + // Copy over the compiled formatter and set call count to INT32_MIN as in computeCompiled(). + // Don't copy the call count directly because doing so requires a loadAcquire/storeRelease. + // The bits themselves appear to be platform-dependent, so copying them might not be safe. + delete fCompiled; + if (src.fCompiled != nullptr) { + auto* callCount = reinterpret_cast(fUnsafeCallCount); + umtx_storeRelease(*callCount, INT32_MIN); + fCompiled = src.fCompiled; + // Reset the source object to leave it in a safe state. + src.resetCompiled(); + } else { + resetCompiled(); + } + + // Unconditionally move the warehouse + delete fWarehouse; + fWarehouse = src.fWarehouse; + src.fWarehouse = nullptr; +} + +void LocalizedNumberFormatter::lnfCopyHelper(const LNF&, UErrorCode& status) { + // When copying, always reset the compiled formatter. + delete fCompiled; + resetCompiled(); + + // If MacroProps has a reference to AffixPatternProvider, we need to copy it. + // If MacroProps has a reference to PluralRules, copy that one, too. + delete fWarehouse; + if (fMacros.affixProvider || fMacros.rules) { + LocalPointer warehouse(new DecimalFormatWarehouse(), status); + if (U_FAILURE(status)) { + fWarehouse = nullptr; + return; + } + if (fMacros.affixProvider) { + warehouse->affixProvider.setTo(fMacros.affixProvider, status); + fMacros.affixProvider = &warehouse->affixProvider.get(); + } + if (fMacros.rules) { + warehouse->rules.adoptInsteadAndCheckErrorCode( + new PluralRules(*fMacros.rules), status); + fMacros.rules = warehouse->rules.getAlias(); + } + fWarehouse = warehouse.orphan(); + } else { + fWarehouse = nullptr; + } +} + + +LocalizedNumberFormatter::~LocalizedNumberFormatter() { + delete fCompiled; + delete fWarehouse; +} + +LocalizedNumberFormatter::LocalizedNumberFormatter(const MacroProps& macros, const Locale& locale) { + fMacros = macros; + fMacros.locale = locale; +} + +LocalizedNumberFormatter::LocalizedNumberFormatter(MacroProps&& macros, const Locale& locale) { + fMacros = std::move(macros); + fMacros.locale = locale; +} + +LocalizedNumberFormatter UnlocalizedNumberFormatter::locale(const Locale& locale) const& { + return LocalizedNumberFormatter(fMacros, locale); +} + +LocalizedNumberFormatter UnlocalizedNumberFormatter::locale(const Locale& locale)&& { + return LocalizedNumberFormatter(std::move(fMacros), locale); +} + +FormattedNumber LocalizedNumberFormatter::formatInt(int64_t value, UErrorCode& status) const { + if (U_FAILURE(status)) { return FormattedNumber(U_ILLEGAL_ARGUMENT_ERROR); } + auto results = new UFormattedNumberData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return FormattedNumber(status); + } + results->quantity.setToLong(value); + formatImpl(results, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + return FormattedNumber(results); + } else { + delete results; + return FormattedNumber(status); + } +} + +FormattedNumber LocalizedNumberFormatter::formatDouble(double value, UErrorCode& status) const { + if (U_FAILURE(status)) { return FormattedNumber(U_ILLEGAL_ARGUMENT_ERROR); } + auto results = new UFormattedNumberData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return FormattedNumber(status); + } + results->quantity.setToDouble(value); + formatImpl(results, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + return FormattedNumber(results); + } else { + delete results; + return FormattedNumber(status); + } +} + +FormattedNumber LocalizedNumberFormatter::formatDecimal(StringPiece value, UErrorCode& status) const { + if (U_FAILURE(status)) { return FormattedNumber(U_ILLEGAL_ARGUMENT_ERROR); } + auto results = new UFormattedNumberData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return FormattedNumber(status); + } + results->quantity.setToDecNumber(value, status); + formatImpl(results, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + return FormattedNumber(results); + } else { + delete results; + return FormattedNumber(status); + } +} + +FormattedNumber +LocalizedNumberFormatter::formatDecimalQuantity(const DecimalQuantity& dq, UErrorCode& status) const { + if (U_FAILURE(status)) { return FormattedNumber(U_ILLEGAL_ARGUMENT_ERROR); } + auto results = new UFormattedNumberData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return FormattedNumber(status); + } + results->quantity = dq; + formatImpl(results, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + return FormattedNumber(results); + } else { + delete results; + return FormattedNumber(status); + } +} + +void LocalizedNumberFormatter::formatImpl(impl::UFormattedNumberData* results, UErrorCode& status) const { + if (computeCompiled(status)) { + fCompiled->format(results, status); + } else { + NumberFormatterImpl::formatStatic(fMacros, results, status); + } + if (U_FAILURE(status)) { + return; + } + results->getStringRef().writeTerminator(status); +} + +void LocalizedNumberFormatter::getAffixImpl(bool isPrefix, bool isNegative, UnicodeString& result, + UErrorCode& status) const { + FormattedStringBuilder string; + auto signum = static_cast(isNegative ? SIGNUM_NEG : SIGNUM_POS); + // Always return affixes for plural form OTHER. + static const StandardPlural::Form plural = StandardPlural::OTHER; + int32_t prefixLength; + if (computeCompiled(status)) { + prefixLength = fCompiled->getPrefixSuffix(signum, plural, string, status); + } else { + prefixLength = NumberFormatterImpl::getPrefixSuffixStatic(fMacros, signum, plural, string, status); + } + result.remove(); + if (isPrefix) { + result.append(string.toTempUnicodeString().tempSubStringBetween(0, prefixLength)); + } else { + result.append(string.toTempUnicodeString().tempSubStringBetween(prefixLength, string.length())); + } +} + +bool LocalizedNumberFormatter::computeCompiled(UErrorCode& status) const { + // fUnsafeCallCount contains memory to be interpreted as an atomic int, most commonly + // std::atomic. Since the type of atomic int is platform-dependent, we cast the + // bytes in fUnsafeCallCount to u_atomic_int32_t, a typedef for the platform-dependent + // atomic int type defined in umutex.h. + static_assert( + sizeof(u_atomic_int32_t) <= sizeof(fUnsafeCallCount), + "Atomic integer size on this platform exceeds the size allocated by fUnsafeCallCount"); + auto* callCount = reinterpret_cast( + const_cast(this)->fUnsafeCallCount); + + // A positive value in the atomic int indicates that the data structure is not yet ready; + // a negative value indicates that it is ready. If, after the increment, the atomic int + // is exactly threshold, then it is the current thread's job to build the data structure. + // Note: We set the callCount to INT32_MIN so that if another thread proceeds to increment + // the atomic int, the value remains below zero. + int32_t currentCount = umtx_loadAcquire(*callCount); + if (0 <= currentCount && currentCount <= fMacros.threshold && fMacros.threshold > 0) { + currentCount = umtx_atomic_inc(callCount); + } + + if (currentCount == fMacros.threshold && fMacros.threshold > 0) { + // Build the data structure and then use it (slow to fast path). + const NumberFormatterImpl* compiled = new NumberFormatterImpl(fMacros, status); + if (compiled == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + U_ASSERT(fCompiled == nullptr); + const_cast(this)->fCompiled = compiled; + umtx_storeRelease(*callCount, INT32_MIN); + return true; + } else if (currentCount < 0) { + // The data structure is already built; use it (fast path). + U_ASSERT(fCompiled != nullptr); + return true; + } else { + // Format the number without building the data structure (slow path). + return false; + } +} + +const impl::NumberFormatterImpl* LocalizedNumberFormatter::getCompiled() const { + return fCompiled; +} + +int32_t LocalizedNumberFormatter::getCallCount() const { + auto* callCount = reinterpret_cast( + const_cast(this)->fUnsafeCallCount); + return umtx_loadAcquire(*callCount); +} + +// Note: toFormat defined in number_asformat.cpp + +const DecimalFormatSymbols* LocalizedNumberFormatter::getDecimalFormatSymbols() const { + return fMacros.symbols.getDecimalFormatSymbols(); +} + +#if (U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN) && defined(_MSC_VER) +// Warning 4661. +#pragma warning(pop) +#endif + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_formatimpl.cpp b/intl/icu/source/i18n/number_formatimpl.cpp new file mode 100644 index 0000000000..53bac49a55 --- /dev/null +++ b/intl/icu/source/i18n/number_formatimpl.cpp @@ -0,0 +1,647 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "cstring.h" +#include "unicode/ures.h" +#include "uresimp.h" +#include "charstr.h" +#include "number_formatimpl.h" +#include "unicode/numfmt.h" +#include "number_patternstring.h" +#include "number_utils.h" +#include "unicode/numberformatter.h" +#include "unicode/dcfmtsym.h" +#include "number_scientific.h" +#include "number_compact.h" +#include "uresimp.h" +#include "ureslocs.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +NumberFormatterImpl::NumberFormatterImpl(const MacroProps& macros, UErrorCode& status) + : NumberFormatterImpl(macros, true, status) { +} + +int32_t NumberFormatterImpl::formatStatic(const MacroProps ¯os, UFormattedNumberData *results, + UErrorCode &status) { + DecimalQuantity &inValue = results->quantity; + FormattedStringBuilder &outString = results->getStringRef(); + NumberFormatterImpl impl(macros, false, status); + MicroProps& micros = impl.preProcessUnsafe(inValue, status); + if (U_FAILURE(status)) { return 0; } + int32_t length = writeNumber(micros.simple, inValue, outString, 0, status); + length += writeAffixes(micros, outString, 0, length, status); + results->outputUnit = std::move(micros.outputUnit); + results->gender = micros.gender; + return length; +} + +int32_t NumberFormatterImpl::getPrefixSuffixStatic(const MacroProps& macros, Signum signum, + StandardPlural::Form plural, + FormattedStringBuilder& outString, UErrorCode& status) { + NumberFormatterImpl impl(macros, false, status); + return impl.getPrefixSuffixUnsafe(signum, plural, outString, status); +} + +// NOTE: C++ SPECIFIC DIFFERENCE FROM JAVA: +// The "safe" apply method uses a new MicroProps. In the MicroPropsGenerator, fMicros is copied into the new instance. +// The "unsafe" method simply re-uses fMicros, eliminating the extra copy operation. +// See MicroProps::processQuantity() for details. + +int32_t NumberFormatterImpl::format(UFormattedNumberData *results, UErrorCode &status) const { + DecimalQuantity &inValue = results->quantity; + FormattedStringBuilder &outString = results->getStringRef(); + MicroProps micros; + preProcess(inValue, micros, status); + if (U_FAILURE(status)) { return 0; } + int32_t length = writeNumber(micros.simple, inValue, outString, 0, status); + length += writeAffixes(micros, outString, 0, length, status); + results->outputUnit = std::move(micros.outputUnit); + results->gender = micros.gender; + return length; +} + +void NumberFormatterImpl::preProcess(DecimalQuantity& inValue, MicroProps& microsOut, + UErrorCode& status) const { + if (U_FAILURE(status)) { return; } + if (fMicroPropsGenerator == nullptr) { + status = U_INTERNAL_PROGRAM_ERROR; + return; + } + fMicroPropsGenerator->processQuantity(inValue, microsOut, status); + microsOut.integerWidth.apply(inValue, status); +} + +MicroProps& NumberFormatterImpl::preProcessUnsafe(DecimalQuantity& inValue, UErrorCode& status) { + if (U_FAILURE(status)) { + return fMicros; // must always return a value + } + if (fMicroPropsGenerator == nullptr) { + status = U_INTERNAL_PROGRAM_ERROR; + return fMicros; // must always return a value + } + fMicroPropsGenerator->processQuantity(inValue, fMicros, status); + fMicros.integerWidth.apply(inValue, status); + return fMicros; +} + +int32_t NumberFormatterImpl::getPrefixSuffix(Signum signum, StandardPlural::Form plural, + FormattedStringBuilder& outString, UErrorCode& status) const { + if (U_FAILURE(status)) { return 0; } + // #13453: DecimalFormat wants the affixes from the pattern only (modMiddle, aka pattern modifier). + // Safe path: use fImmutablePatternModifier. + const Modifier* modifier = fImmutablePatternModifier->getModifier(signum, plural); + modifier->apply(outString, 0, 0, status); + if (U_FAILURE(status)) { return 0; } + return modifier->getPrefixLength(); +} + +int32_t NumberFormatterImpl::getPrefixSuffixUnsafe(Signum signum, StandardPlural::Form plural, + FormattedStringBuilder& outString, UErrorCode& status) { + if (U_FAILURE(status)) { return 0; } + // #13453: DecimalFormat wants the affixes from the pattern only (modMiddle, aka pattern modifier). + // Unsafe path: use fPatternModifier. + fPatternModifier->setNumberProperties(signum, plural); + fPatternModifier->apply(outString, 0, 0, status); + if (U_FAILURE(status)) { return 0; } + return fPatternModifier->getPrefixLength(); +} + +NumberFormatterImpl::NumberFormatterImpl(const MacroProps& macros, bool safe, UErrorCode& status) { + fMicroPropsGenerator = macrosToMicroGenerator(macros, safe, status); +} + +////////// + +const MicroPropsGenerator* +NumberFormatterImpl::macrosToMicroGenerator(const MacroProps& macros, bool safe, UErrorCode& status) { + if (U_FAILURE(status)) { return nullptr; } + const MicroPropsGenerator* chain = &fMicros; + + // Check that macros is error-free before continuing. + if (macros.copyErrorTo(status)) { + return nullptr; + } + + // TODO: Accept currency symbols from DecimalFormatSymbols? + + // Pre-compute a few values for efficiency. + bool isCurrency = utils::unitIsCurrency(macros.unit); + bool isBaseUnit = utils::unitIsBaseUnit(macros.unit); + bool isPercent = utils::unitIsPercent(macros.unit); + bool isPermille = utils::unitIsPermille(macros.unit); + bool isCompactNotation = macros.notation.fType == Notation::NTN_COMPACT; + bool isAccounting = + macros.sign == UNUM_SIGN_ACCOUNTING || + macros.sign == UNUM_SIGN_ACCOUNTING_ALWAYS || + macros.sign == UNUM_SIGN_ACCOUNTING_EXCEPT_ZERO || + macros.sign == UNUM_SIGN_ACCOUNTING_NEGATIVE; + CurrencyUnit currency(u"", status); + if (isCurrency) { + currency = CurrencyUnit(macros.unit, status); // Restore CurrencyUnit from MeasureUnit + } + UNumberUnitWidth unitWidth = UNUM_UNIT_WIDTH_SHORT; + if (macros.unitWidth != UNUM_UNIT_WIDTH_COUNT) { + unitWidth = macros.unitWidth; + } + // Use CLDR unit data for all MeasureUnits (not currency and not + // no-unit), except use the dedicated percent pattern for percent and + // permille. However, use the CLDR unit data for percent/permille if a + // long name was requested OR if compact notation is being used, since + // compact notation overrides the middle modifier (micros.modMiddle) + // normally used for the percent pattern. + bool isCldrUnit = !isCurrency + && !isBaseUnit + && (unitWidth == UNUM_UNIT_WIDTH_FULL_NAME + || !(isPercent || isPermille) + || isCompactNotation + ); + bool isMixedUnit = isCldrUnit && (uprv_strcmp(macros.unit.getType(), "") == 0) && + macros.unit.getComplexity(status) == UMEASURE_UNIT_MIXED; + + // Select the numbering system. + LocalPointer nsLocal; + const NumberingSystem* ns; + if (macros.symbols.isNumberingSystem()) { + ns = macros.symbols.getNumberingSystem(); + } else { + // TODO: Is there a way to avoid creating the NumberingSystem object? + ns = NumberingSystem::createInstance(macros.locale, status); + // Give ownership to the function scope. + nsLocal.adoptInstead(ns); + } + const char* nsName = U_SUCCESS(status) ? ns->getName() : "latn"; + uprv_strncpy(fMicros.nsName, nsName, 8); + fMicros.nsName[8] = 0; // guarantee NUL-terminated + + // Default gender: none. + fMicros.gender = ""; + + // Resolve the symbols. Do this here because currency may need to customize them. + if (macros.symbols.isDecimalFormatSymbols()) { + fMicros.simple.symbols = macros.symbols.getDecimalFormatSymbols(); + } else { + LocalPointer newSymbols( + new DecimalFormatSymbols(macros.locale, *ns, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + if (isCurrency) { + newSymbols->setCurrency(currency.getISOCurrency(), status); + if (U_FAILURE(status)) { + return nullptr; + } + } + fMicros.simple.symbols = newSymbols.getAlias(); + fSymbols.adoptInstead(newSymbols.orphan()); + } + + // Load and parse the pattern string. It is used for grouping sizes and affixes only. + // If we are formatting currency, check for a currency-specific pattern. + const char16_t* pattern = nullptr; + if (isCurrency && fMicros.simple.symbols->getCurrencyPattern() != nullptr) { + pattern = fMicros.simple.symbols->getCurrencyPattern(); + } + if (pattern == nullptr) { + CldrPatternStyle patternStyle; + if (isCldrUnit) { + patternStyle = CLDR_PATTERN_STYLE_DECIMAL; + } else if (isPercent || isPermille) { + patternStyle = CLDR_PATTERN_STYLE_PERCENT; + } else if (!isCurrency || unitWidth == UNUM_UNIT_WIDTH_FULL_NAME) { + patternStyle = CLDR_PATTERN_STYLE_DECIMAL; + } else if (isAccounting) { + // NOTE: Although ACCOUNTING and ACCOUNTING_ALWAYS are only supported in currencies right now, + // the API contract allows us to add support to other units in the future. + patternStyle = CLDR_PATTERN_STYLE_ACCOUNTING; + } else { + patternStyle = CLDR_PATTERN_STYLE_CURRENCY; + } + pattern = utils::getPatternForStyle(macros.locale, nsName, patternStyle, status); + if (U_FAILURE(status)) { + return nullptr; + } + } + auto patternInfo = new ParsedPatternInfo(); + if (patternInfo == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + fPatternInfo.adoptInstead(patternInfo); + PatternParser::parseToPatternInfo(UnicodeString(pattern), *patternInfo, status); + if (U_FAILURE(status)) { + return nullptr; + } + + ///////////////////////////////////////////////////////////////////////////////////// + /// START POPULATING THE DEFAULT MICROPROPS AND BUILDING THE MICROPROPS GENERATOR /// + ///////////////////////////////////////////////////////////////////////////////////// + + // Unit Preferences and Conversions as our first step + if (macros.usage.isSet()) { + if (!isCldrUnit) { + // We only support "usage" when the input unit is specified, and is + // a CLDR Unit. + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + auto usagePrefsHandler = + new UsagePrefsHandler(macros.locale, macros.unit, macros.usage.fValue, chain, status); + fUsagePrefsHandler.adoptInsteadAndCheckErrorCode(usagePrefsHandler, status); + chain = fUsagePrefsHandler.getAlias(); + } else if (isMixedUnit) { + auto unitConversionHandler = new UnitConversionHandler(macros.unit, chain, status); + fUnitConversionHandler.adoptInsteadAndCheckErrorCode(unitConversionHandler, status); + chain = fUnitConversionHandler.getAlias(); + } + + // Multiplier + if (macros.scale.isValid()) { + fMicros.helpers.multiplier.setAndChain(macros.scale, chain); + chain = &fMicros.helpers.multiplier; + } + + // Rounding strategy + Precision precision; + if (!macros.precision.isBogus()) { + precision = macros.precision; + } else if (isCompactNotation) { + precision = Precision::integer().withMinDigits(2); + } else if (isCurrency) { + precision = Precision::currency(UCURR_USAGE_STANDARD); + } else if (macros.usage.isSet()) { + // Bogus Precision - it will get set in the UsagePrefsHandler instead + precision = Precision(); + } else { + precision = Precision::maxFraction(6); + } + UNumberFormatRoundingMode roundingMode; + roundingMode = macros.roundingMode; + fMicros.rounder = {precision, roundingMode, currency, status}; + if (U_FAILURE(status)) { + return nullptr; + } + + // Grouping strategy + if (!macros.grouper.isBogus()) { + fMicros.simple.grouping = macros.grouper; + } else if (isCompactNotation) { + // Compact notation uses minGrouping by default since ICU 59 + fMicros.simple.grouping = Grouper::forStrategy(UNUM_GROUPING_MIN2); + } else { + fMicros.simple.grouping = Grouper::forStrategy(UNUM_GROUPING_AUTO); + } + fMicros.simple.grouping.setLocaleData(*fPatternInfo, macros.locale); + + // Padding strategy + if (!macros.padder.isBogus()) { + fMicros.padding = macros.padder; + } else { + fMicros.padding = Padder::none(); + } + + // Integer width + if (!macros.integerWidth.isBogus()) { + fMicros.integerWidth = macros.integerWidth; + } else { + fMicros.integerWidth = IntegerWidth::standard(); + } + + // Sign display + if (macros.sign != UNUM_SIGN_COUNT) { + fMicros.sign = macros.sign; + } else { + fMicros.sign = UNUM_SIGN_AUTO; + } + + // Decimal mark display + if (macros.decimal != UNUM_DECIMAL_SEPARATOR_COUNT) { + fMicros.simple.decimal = macros.decimal; + } else { + fMicros.simple.decimal = UNUM_DECIMAL_SEPARATOR_AUTO; + } + + // Use monetary separator symbols + fMicros.simple.useCurrency = isCurrency; + + // Inner modifier (scientific notation) + if (macros.notation.fType == Notation::NTN_SCIENTIFIC) { + auto newScientificHandler = new ScientificHandler(¯os.notation, fMicros.simple.symbols, chain); + if (newScientificHandler == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + fScientificHandler.adoptInstead(newScientificHandler); + chain = fScientificHandler.getAlias(); + } else { + // No inner modifier required + fMicros.modInner = &fMicros.helpers.emptyStrongModifier; + } + + // Middle modifier (patterns, positive/negative, currency symbols, percent) + auto patternModifier = new MutablePatternModifier(false); + if (patternModifier == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + fPatternModifier.adoptInstead(patternModifier); + const AffixPatternProvider* affixProvider = + macros.affixProvider != nullptr && ( + // For more information on this condition, see ICU-22073 + !isCompactNotation || isCurrency == macros.affixProvider->hasCurrencySign()) + ? macros.affixProvider + : static_cast(fPatternInfo.getAlias()); + patternModifier->setPatternInfo(affixProvider, kUndefinedField); + patternModifier->setPatternAttributes(fMicros.sign, isPermille, macros.approximately); + if (patternModifier->needsPlurals()) { + patternModifier->setSymbols( + fMicros.simple.symbols, + currency, + unitWidth, + resolvePluralRules(macros.rules, macros.locale, status), + status); + } else { + patternModifier->setSymbols(fMicros.simple.symbols, currency, unitWidth, nullptr, status); + } + if (safe) { + fImmutablePatternModifier.adoptInsteadAndCheckErrorCode(patternModifier->createImmutable(status), + status); + } + if (U_FAILURE(status)) { + return nullptr; + } + + // currencyAsDecimal + if (affixProvider->currencyAsDecimal()) { + fMicros.simple.currencyAsDecimal = patternModifier->getCurrencySymbolForUnitWidth(status); + } + + // Outer modifier (CLDR units and currency long names) + if (isCldrUnit) { + const char *unitDisplayCase = ""; + if (macros.unitDisplayCase.isSet()) { + unitDisplayCase = macros.unitDisplayCase.fValue; + } + if (macros.usage.isSet()) { + fLongNameMultiplexer.adoptInsteadAndCheckErrorCode( + LongNameMultiplexer::forMeasureUnits( + macros.locale, *fUsagePrefsHandler->getOutputUnits(), unitWidth, unitDisplayCase, + resolvePluralRules(macros.rules, macros.locale, status), chain, status), + status); + chain = fLongNameMultiplexer.getAlias(); + } else if (isMixedUnit) { + fMixedUnitLongNameHandler.adoptInsteadAndCheckErrorCode(new MixedUnitLongNameHandler(), + status); + MixedUnitLongNameHandler::forMeasureUnit( + macros.locale, macros.unit, unitWidth, unitDisplayCase, + resolvePluralRules(macros.rules, macros.locale, status), chain, + fMixedUnitLongNameHandler.getAlias(), status); + chain = fMixedUnitLongNameHandler.getAlias(); + } else { + MeasureUnit unit = macros.unit; + if (!utils::unitIsBaseUnit(macros.perUnit)) { + unit = unit.product(macros.perUnit.reciprocal(status), status); + // This isn't strictly necessary, but was what we specced out + // when perUnit became a backward-compatibility thing: + // unit/perUnit use case is only valid if both units are + // built-ins, or the product is a built-in. + if (uprv_strcmp(unit.getType(), "") == 0 && + (uprv_strcmp(macros.unit.getType(), "") == 0 || + uprv_strcmp(macros.perUnit.getType(), "") == 0)) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + } + fLongNameHandler.adoptInsteadAndCheckErrorCode(new LongNameHandler(), status); + LongNameHandler::forMeasureUnit(macros.locale, unit, unitWidth, unitDisplayCase, + resolvePluralRules(macros.rules, macros.locale, status), + chain, fLongNameHandler.getAlias(), status); + chain = fLongNameHandler.getAlias(); + } + } else if (isCurrency && unitWidth == UNUM_UNIT_WIDTH_FULL_NAME) { + fLongNameHandler.adoptInsteadAndCheckErrorCode( + LongNameHandler::forCurrencyLongNames( + macros.locale, currency, resolvePluralRules(macros.rules, macros.locale, status), chain, + status), + status); + chain = fLongNameHandler.getAlias(); + } else { + // No outer modifier required + fMicros.modOuter = &fMicros.helpers.emptyWeakModifier; + } + if (U_FAILURE(status)) { + return nullptr; + } + + // Compact notation + if (isCompactNotation) { + CompactType compactType = (isCurrency && unitWidth != UNUM_UNIT_WIDTH_FULL_NAME) + ? CompactType::TYPE_CURRENCY : CompactType::TYPE_DECIMAL; + auto newCompactHandler = new CompactHandler( + macros.notation.fUnion.compactStyle, + macros.locale, + nsName, + compactType, + resolvePluralRules(macros.rules, macros.locale, status), + patternModifier, + safe, + chain, + status); + if (U_FAILURE(status)) { + return nullptr; + } + if (newCompactHandler == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + fCompactHandler.adoptInstead(newCompactHandler); + chain = fCompactHandler.getAlias(); + } + if (U_FAILURE(status)) { + return nullptr; + } + + // Always add the pattern modifier as the last element of the chain. + if (safe) { + fImmutablePatternModifier->addToChain(chain); + chain = fImmutablePatternModifier.getAlias(); + } else { + patternModifier->addToChain(chain); + chain = patternModifier; + } + + return chain; +} + +const PluralRules* +NumberFormatterImpl::resolvePluralRules( + const PluralRules* rulesPtr, + const Locale& locale, + UErrorCode& status) { + if (rulesPtr != nullptr) { + return rulesPtr; + } + // Lazily create PluralRules + if (fRules.isNull()) { + fRules.adoptInstead(PluralRules::forLocale(locale, status)); + } + return fRules.getAlias(); +} + +int32_t NumberFormatterImpl::writeAffixes( + const MicroProps& micros, + FormattedStringBuilder& string, + int32_t start, + int32_t end, + UErrorCode& status) { + U_ASSERT(micros.modOuter != nullptr); + // Always apply the inner modifier (which is "strong"). + int32_t length = micros.modInner->apply(string, start, end, status); + if (micros.padding.isValid()) { + length += micros.padding + .padAndApply(*micros.modMiddle, *micros.modOuter, string, start, length + end, status); + } else { + length += micros.modMiddle->apply(string, start, length + end, status); + length += micros.modOuter->apply(string, start, length + end, status); + } + return length; +} + +int32_t NumberFormatterImpl::writeNumber( + const SimpleMicroProps& micros, + DecimalQuantity& quantity, + FormattedStringBuilder& string, + int32_t index, + UErrorCode& status) { + int32_t length = 0; + if (quantity.isInfinite()) { + length += string.insert( + length + index, + micros.symbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kInfinitySymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD}, + status); + + } else if (quantity.isNaN()) { + length += string.insert( + length + index, + micros.symbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kNaNSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD}, + status); + + } else { + // Add the integer digits + length += writeIntegerDigits( + micros, + quantity, + string, + length + index, + status); + + // Add the decimal point + if (quantity.getLowerDisplayMagnitude() < 0 || micros.decimal == UNUM_DECIMAL_SEPARATOR_ALWAYS) { + if (!micros.currencyAsDecimal.isBogus()) { + length += string.insert( + length + index, + micros.currencyAsDecimal, + {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}, + status); + } else if (micros.useCurrency) { + length += string.insert( + length + index, + micros.symbols->getSymbol( + DecimalFormatSymbols::ENumberFormatSymbol::kMonetarySeparatorSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_DECIMAL_SEPARATOR_FIELD}, + status); + } else { + length += string.insert( + length + index, + micros.symbols->getSymbol( + DecimalFormatSymbols::ENumberFormatSymbol::kDecimalSeparatorSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_DECIMAL_SEPARATOR_FIELD}, + status); + } + } + + // Add the fraction digits + length += writeFractionDigits(micros, quantity, string, length + index, status); + + if (length == 0) { + // Force output of the digit for value 0 + length += utils::insertDigitFromSymbols( + string, + index, + 0, + *micros.symbols, + {UFIELD_CATEGORY_NUMBER, UNUM_INTEGER_FIELD}, + status); + } + } + + return length; +} + +int32_t NumberFormatterImpl::writeIntegerDigits( + const SimpleMicroProps& micros, + DecimalQuantity& quantity, + FormattedStringBuilder& string, + int32_t index, + UErrorCode& status) { + int length = 0; + int integerCount = quantity.getUpperDisplayMagnitude() + 1; + for (int i = 0; i < integerCount; i++) { + // Add grouping separator + if (micros.grouping.groupAtPosition(i, quantity)) { + length += string.insert( + index, + micros.useCurrency ? micros.symbols->getSymbol( + DecimalFormatSymbols::ENumberFormatSymbol::kMonetaryGroupingSeparatorSymbol) + : micros.symbols->getSymbol( + DecimalFormatSymbols::ENumberFormatSymbol::kGroupingSeparatorSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_GROUPING_SEPARATOR_FIELD}, + status); + } + + // Get and append the next digit value + int8_t nextDigit = quantity.getDigit(i); + length += utils::insertDigitFromSymbols( + string, + index, + nextDigit, + *micros.symbols, + {UFIELD_CATEGORY_NUMBER, + UNUM_INTEGER_FIELD}, + status); + } + return length; +} + +int32_t NumberFormatterImpl::writeFractionDigits( + const SimpleMicroProps& micros, + DecimalQuantity& quantity, + FormattedStringBuilder& string, + int32_t index, + UErrorCode& status) { + int length = 0; + int fractionCount = -quantity.getLowerDisplayMagnitude(); + for (int i = 0; i < fractionCount; i++) { + // Get and append the next digit value + int8_t nextDigit = quantity.getDigit(-i - 1); + length += utils::insertDigitFromSymbols( + string, + length + index, + nextDigit, + *micros.symbols, + {UFIELD_CATEGORY_NUMBER, UNUM_FRACTION_FIELD}, + status); + } + return length; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_formatimpl.h b/intl/icu/source/i18n/number_formatimpl.h new file mode 100644 index 0000000000..62d5321261 --- /dev/null +++ b/intl/icu/source/i18n/number_formatimpl.h @@ -0,0 +1,180 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_FORMATIMPL_H__ +#define __NUMBER_FORMATIMPL_H__ + +#include "number_types.h" +#include "formatted_string_builder.h" +#include "number_patternstring.h" +#include "number_usageprefs.h" +#include "number_utils.h" +#include "number_patternmodifier.h" +#include "number_longnames.h" +#include "number_compact.h" +#include "number_microprops.h" +#include "number_utypes.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +/** + * This is the "brain" of the number formatting pipeline. It ties all the pieces together, taking in a MacroProps and a + * DecimalQuantity and outputting a properly formatted number string. + */ +class NumberFormatterImpl : public UMemory { + public: + /** + * Builds a "safe" MicroPropsGenerator, which is thread-safe and can be used repeatedly. + * The caller owns the returned NumberFormatterImpl. + */ + NumberFormatterImpl(const MacroProps ¯os, UErrorCode &status); + + /** + * Default constructor; leaves the NumberFormatterImpl in an undefined state. + * Takes an error code to prevent the method from being called accidentally. + */ + NumberFormatterImpl(UErrorCode &) {} + + /** + * Builds and evaluates an "unsafe" MicroPropsGenerator, which is cheaper but can be used only once. + */ + static int32_t formatStatic(const MacroProps ¯os, UFormattedNumberData *results, + UErrorCode &status); + + /** + * Prints only the prefix and suffix; used for DecimalFormat getters. + * + * @return The index into the output at which the prefix ends and the suffix starts; in other words, + * the prefix length. + */ + static int32_t getPrefixSuffixStatic(const MacroProps& macros, Signum signum, + StandardPlural::Form plural, FormattedStringBuilder& outString, + UErrorCode& status); + + /** + * Evaluates the "safe" MicroPropsGenerator created by "fromMacros". + */ + int32_t format(UFormattedNumberData *results, UErrorCode &status) const; + + /** + * Like format(), but saves the result into an output MicroProps without additional processing. + */ + void preProcess(DecimalQuantity& inValue, MicroProps& microsOut, UErrorCode& status) const; + + /** + * Like getPrefixSuffixStatic() but uses the safe compiled object. + */ + int32_t getPrefixSuffix(Signum signum, StandardPlural::Form plural, FormattedStringBuilder& outString, + UErrorCode& status) const; + + const MicroProps& getRawMicroProps() const { + return fMicros; + } + + /** + * Synthesizes the output string from a MicroProps and DecimalQuantity. + * This method formats only the main number, not affixes. + */ + static int32_t writeNumber( + const SimpleMicroProps& micros, + DecimalQuantity& quantity, + FormattedStringBuilder& string, + int32_t index, + UErrorCode& status); + + /** + * Adds the affixes. Intended to be called immediately after formatNumber. + */ + static int32_t writeAffixes( + const MicroProps& micros, + FormattedStringBuilder& string, + int32_t start, + int32_t end, + UErrorCode& status); + + private: + // Head of the MicroPropsGenerator linked list. Subclasses' processQuantity + // methods process this list in a parent-first order, such that the last + // item added, which this points to, typically has its logic executed last. + const MicroPropsGenerator *fMicroPropsGenerator = nullptr; + + // Tail of the list: + MicroProps fMicros; + + // Other fields possibly used by the number formatting pipeline: + // TODO: Convert more of these LocalPointers to value objects to reduce the number of news? + LocalPointer fUsagePrefsHandler; + LocalPointer fUnitConversionHandler; + LocalPointer fSymbols; + LocalPointer fRules; + LocalPointer fPatternInfo; + LocalPointer fScientificHandler; + LocalPointer fPatternModifier; + LocalPointer fImmutablePatternModifier; + LocalPointer fLongNameHandler; + // TODO: use a common base class that enables fLongNameHandler, + // fLongNameMultiplexer, and fMixedUnitLongNameHandler to be merged into one + // member? + LocalPointer fMixedUnitLongNameHandler; + LocalPointer fLongNameMultiplexer; + LocalPointer fCompactHandler; + + NumberFormatterImpl(const MacroProps ¯os, bool safe, UErrorCode &status); + + MicroProps& preProcessUnsafe(DecimalQuantity &inValue, UErrorCode &status); + + int32_t getPrefixSuffixUnsafe(Signum signum, StandardPlural::Form plural, + FormattedStringBuilder& outString, UErrorCode& status); + + /** + * If rulesPtr is non-null, return it. Otherwise, return a PluralRules owned by this object for the + * specified locale, creating it if necessary. + */ + const PluralRules * + resolvePluralRules(const PluralRules *rulesPtr, const Locale &locale, UErrorCode &status); + + /** + * Synthesizes the MacroProps into a MicroPropsGenerator. All information, including the locale, is encoded into the + * MicroPropsGenerator, except for the quantity itself, which is left abstract and must be provided to the returned + * MicroPropsGenerator instance. + * + * @see MicroPropsGenerator + * @param macros + * The {@link MacroProps} to consume. This method does not mutate the MacroProps instance. + * @param safe + * If true, the returned MicroPropsGenerator will be thread-safe. If false, the returned value will + * not be thread-safe, intended for a single "one-shot" use only. Building the thread-safe + * object is more expensive. + */ + const MicroPropsGenerator * + macrosToMicroGenerator(const MacroProps ¯os, bool safe, UErrorCode &status); + + static int32_t + writeIntegerDigits( + const SimpleMicroProps& micros, + DecimalQuantity &quantity, + FormattedStringBuilder &string, + int32_t index, + UErrorCode &status); + + static int32_t + writeFractionDigits( + const SimpleMicroProps& micros, + DecimalQuantity &quantity, + FormattedStringBuilder &string, + int32_t index, + UErrorCode &status); +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_FORMATIMPL_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_grouping.cpp b/intl/icu/source/i18n/number_grouping.cpp new file mode 100644 index 0000000000..54aeffee81 --- /dev/null +++ b/intl/icu/source/i18n/number_grouping.cpp @@ -0,0 +1,109 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numberformatter.h" +#include "number_patternstring.h" +#include "uresimp.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +int16_t getMinGroupingForLocale(const Locale& locale) { + // TODO: Cache this? + UErrorCode localStatus = U_ZERO_ERROR; + LocalUResourceBundlePointer bundle(ures_open(nullptr, locale.getName(), &localStatus)); + int32_t resultLen = 0; + const char16_t* result = ures_getStringByKeyWithFallback( + bundle.getAlias(), + "NumberElements/minimumGroupingDigits", + &resultLen, + &localStatus); + // TODO: Is it safe to assume resultLen == 1? Would locales set minGrouping >= 10? + if (U_FAILURE(localStatus) || resultLen != 1) { + return 1; + } + return result[0] - u'0'; +} + +} + +Grouper Grouper::forStrategy(UNumberGroupingStrategy grouping) { + switch (grouping) { + case UNUM_GROUPING_OFF: + return {-1, -1, -2, grouping}; + case UNUM_GROUPING_AUTO: + return {-2, -2, -2, grouping}; + case UNUM_GROUPING_MIN2: + return {-2, -2, -3, grouping}; + case UNUM_GROUPING_ON_ALIGNED: + return {-4, -4, 1, grouping}; + case UNUM_GROUPING_THOUSANDS: + return {3, 3, 1, grouping}; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +Grouper Grouper::forProperties(const DecimalFormatProperties& properties) { + if (!properties.groupingUsed) { + return forStrategy(UNUM_GROUPING_OFF); + } + auto grouping1 = static_cast(properties.groupingSize); + auto grouping2 = static_cast(properties.secondaryGroupingSize); + auto minGrouping = static_cast(properties.minimumGroupingDigits); + grouping1 = grouping1 > 0 ? grouping1 : grouping2 > 0 ? grouping2 : grouping1; + grouping2 = grouping2 > 0 ? grouping2 : grouping1; + return {grouping1, grouping2, minGrouping, UNUM_GROUPING_COUNT}; +} + +void Grouper::setLocaleData(const impl::ParsedPatternInfo &patternInfo, const Locale& locale) { + if (fMinGrouping == -2) { + fMinGrouping = getMinGroupingForLocale(locale); + } else if (fMinGrouping == -3) { + fMinGrouping = static_cast(uprv_max(2, getMinGroupingForLocale(locale))); + } else { + // leave fMinGrouping alone + } + if (fGrouping1 != -2 && fGrouping2 != -4) { + return; + } + auto grouping1 = static_cast (patternInfo.positive.groupingSizes & 0xffff); + auto grouping2 = static_cast ((patternInfo.positive.groupingSizes >> 16) & 0xffff); + auto grouping3 = static_cast ((patternInfo.positive.groupingSizes >> 32) & 0xffff); + if (grouping2 == -1) { + grouping1 = fGrouping1 == -4 ? (short) 3 : (short) -1; + } + if (grouping3 == -1) { + grouping2 = grouping1; + } + fGrouping1 = grouping1; + fGrouping2 = grouping2; +} + +bool Grouper::groupAtPosition(int32_t position, const impl::DecimalQuantity &value) const { + U_ASSERT(fGrouping1 > -2); + if (fGrouping1 == -1 || fGrouping1 == 0) { + // Either -1 or 0 means "no grouping" + return false; + } + position -= fGrouping1; + return position >= 0 && (position % fGrouping2) == 0 + && value.getUpperDisplayMagnitude() - fGrouping1 + 1 >= fMinGrouping; +} + +int16_t Grouper::getPrimary() const { + return fGrouping1; +} + +int16_t Grouper::getSecondary() const { + return fGrouping2; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_integerwidth.cpp b/intl/icu/source/i18n/number_integerwidth.cpp new file mode 100644 index 0000000000..10b853423c --- /dev/null +++ b/intl/icu/source/i18n/number_integerwidth.cpp @@ -0,0 +1,71 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +IntegerWidth::IntegerWidth(digits_t minInt, digits_t maxInt, bool formatFailIfMoreThanMaxDigits) { + fUnion.minMaxInt.fMinInt = minInt; + fUnion.minMaxInt.fMaxInt = maxInt; + fUnion.minMaxInt.fFormatFailIfMoreThanMaxDigits = formatFailIfMoreThanMaxDigits; +} + +IntegerWidth IntegerWidth::zeroFillTo(int32_t minInt) { + if (minInt >= 0 && minInt <= kMaxIntFracSig) { + return {static_cast(minInt), -1, false}; + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +IntegerWidth IntegerWidth::truncateAt(int32_t maxInt) { + if (fHasError) { return *this; } // No-op on error + digits_t minInt = fUnion.minMaxInt.fMinInt; + if (maxInt >= 0 && maxInt <= kMaxIntFracSig && minInt <= maxInt) { + return {minInt, static_cast(maxInt), false}; + } else if (maxInt == -1) { + return {minInt, -1, false}; + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +void IntegerWidth::apply(impl::DecimalQuantity& quantity, UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + if (fHasError) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else if (fUnion.minMaxInt.fMaxInt == -1) { + quantity.setMinInteger(fUnion.minMaxInt.fMinInt); + } else { + // Enforce the backwards-compatibility feature "FormatFailIfMoreThanMaxDigits" + if (fUnion.minMaxInt.fFormatFailIfMoreThanMaxDigits && + fUnion.minMaxInt.fMaxInt < quantity.getMagnitude()) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } + quantity.setMinInteger(fUnion.minMaxInt.fMinInt); + quantity.applyMaxInteger(fUnion.minMaxInt.fMaxInt); + } +} + +bool IntegerWidth::operator==(const IntegerWidth& other) const { + // Private operator==; do error and bogus checking first! + U_ASSERT(!fHasError); + U_ASSERT(!other.fHasError); + U_ASSERT(!isBogus()); + U_ASSERT(!other.isBogus()); + return fUnion.minMaxInt.fMinInt == other.fUnion.minMaxInt.fMinInt && + fUnion.minMaxInt.fMaxInt == other.fUnion.minMaxInt.fMaxInt; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_longnames.cpp b/intl/icu/source/i18n/number_longnames.cpp new file mode 100644 index 0000000000..96c6ca6bf8 --- /dev/null +++ b/intl/icu/source/i18n/number_longnames.cpp @@ -0,0 +1,1766 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include + +#include "unicode/simpleformatter.h" +#include "unicode/ures.h" +#include "ureslocs.h" +#include "charstr.h" +#include "uresimp.h" +#include "measunit_impl.h" +#include "number_longnames.h" +#include "number_microprops.h" +#include +#include "cstring.h" +#include "util.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +/** + * Display Name (this format has no placeholder). + * + * Used as an index into the LongNameHandler::simpleFormats array. Units + * resources cover the normal set of PluralRules keys, as well as `dnam` and + * `per` forms. + */ +constexpr int32_t DNAM_INDEX = StandardPlural::Form::COUNT; +/** + * "per" form (e.g. "{0} per day" is day's "per" form). + * + * Used as an index into the LongNameHandler::simpleFormats array. Units + * resources cover the normal set of PluralRules keys, as well as `dnam` and + * `per` forms. + */ +constexpr int32_t PER_INDEX = StandardPlural::Form::COUNT + 1; +/** + * Gender of the word, in languages with grammatical gender. + */ +constexpr int32_t GENDER_INDEX = StandardPlural::Form::COUNT + 2; +// Number of keys in the array populated by PluralTableSink. +constexpr int32_t ARRAY_LENGTH = StandardPlural::Form::COUNT + 3; + +// TODO(icu-units#28): load this list from resources, after creating a "&set" +// function for use in ldml2icu rules. +const int32_t GENDER_COUNT = 7; +const char *gGenders[GENDER_COUNT] = {"animate", "common", "feminine", "inanimate", + "masculine", "neuter", "personal"}; + +// Converts a UnicodeString to a const char*, either pointing to a string in +// gGenders, or pointing to an empty string if an appropriate string was not +// found. +const char *getGenderString(UnicodeString uGender, UErrorCode status) { + if (uGender.length() == 0) { + return ""; + } + CharString gender; + gender.appendInvariantChars(uGender, status); + if (U_FAILURE(status)) { + return ""; + } + int32_t first = 0; + int32_t last = GENDER_COUNT; + while (first < last) { + int32_t mid = (first + last) / 2; + int32_t cmp = uprv_strcmp(gender.data(), gGenders[mid]); + if (cmp == 0) { + return gGenders[mid]; + } else if (cmp > 0) { + first = mid + 1; + } else if (cmp < 0) { + last = mid; + } + } + // We don't return an error in case our gGenders list is incomplete in + // production. + // + // TODO(icu-units#28): a unit test checking all locales' genders are covered + // by gGenders? Else load a complete list of genders found in + // grammaticalFeatures in an initOnce. + return ""; +} + +// Returns the array index that corresponds to the given pluralKeyword. +static int32_t getIndex(const char* pluralKeyword, UErrorCode& status) { + // pluralKeyword can also be "dnam", "per", or "gender" + switch (*pluralKeyword) { + case 'd': + if (uprv_strcmp(pluralKeyword + 1, "nam") == 0) { + return DNAM_INDEX; + } + break; + case 'g': + if (uprv_strcmp(pluralKeyword + 1, "ender") == 0) { + return GENDER_INDEX; + } + break; + case 'p': + if (uprv_strcmp(pluralKeyword + 1, "er") == 0) { + return PER_INDEX; + } + break; + default: + break; + } + StandardPlural::Form plural = StandardPlural::fromString(pluralKeyword, status); + return plural; +} + +// Selects a string out of the `strings` array which corresponds to the +// specified plural form, with fallback to the OTHER form. +// +// The `strings` array must have ARRAY_LENGTH items: one corresponding to each +// of the plural forms, plus a display name ("dnam") and a "per" form. +static UnicodeString getWithPlural( + const UnicodeString* strings, + StandardPlural::Form plural, + UErrorCode& status) { + UnicodeString result = strings[plural]; + if (result.isBogus()) { + result = strings[StandardPlural::Form::OTHER]; + } + if (result.isBogus()) { + // There should always be data in the "other" plural variant. + status = U_INTERNAL_PROGRAM_ERROR; + } + return result; +} + +enum PlaceholderPosition { PH_EMPTY, PH_NONE, PH_BEGINNING, PH_MIDDLE, PH_END }; + +/** + * Returns three outputs extracted from pattern. + * + * @param coreUnit is extracted as per Extract(...) in the spec: + * https://unicode.org/reports/tr35/tr35-general.html#compound-units + * @param PlaceholderPosition indicates where in the string the placeholder was + * found. + * @param joinerChar Iff the placeholder was at the beginning or end, joinerChar + * contains the space character (if any) that separated the placeholder from + * the rest of the pattern. Otherwise, joinerChar is set to NUL. Only one + * space character is considered. + */ +void extractCorePattern(const UnicodeString &pattern, + UnicodeString &coreUnit, + PlaceholderPosition &placeholderPosition, + char16_t &joinerChar) { + joinerChar = 0; + int32_t len = pattern.length(); + if (pattern.startsWith(u"{0}", 3)) { + placeholderPosition = PH_BEGINNING; + if (u_isJavaSpaceChar(pattern[3])) { + joinerChar = pattern[3]; + coreUnit.setTo(pattern, 4, len - 4); + } else { + coreUnit.setTo(pattern, 3, len - 3); + } + } else if (pattern.endsWith(u"{0}", 3)) { + placeholderPosition = PH_END; + if (u_isJavaSpaceChar(pattern[len - 4])) { + coreUnit.setTo(pattern, 0, len - 4); + joinerChar = pattern[len - 4]; + } else { + coreUnit.setTo(pattern, 0, len - 3); + } + } else if (pattern.indexOf(u"{0}", 3, 1, len - 2) == -1) { + placeholderPosition = PH_NONE; + coreUnit = pattern; + } else { + placeholderPosition = PH_MIDDLE; + coreUnit = pattern; + } +} + +////////////////////////// +/// BEGIN DATA LOADING /// +////////////////////////// + +// Gets the gender of a built-in unit: unit must be a built-in. Returns an empty +// string both in case of unknown gender and in case of unknown unit. +UnicodeString +getGenderForBuiltin(const Locale &locale, const MeasureUnit &builtinUnit, UErrorCode &status) { + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_UNIT, locale.getName(), &status)); + if (U_FAILURE(status)) { return {}; } + + // Map duration-year-person, duration-week-person, etc. to duration-year, duration-week, ... + // TODO(ICU-20400): Get duration-*-person data properly with aliases. + StringPiece subtypeForResource; + int32_t subtypeLen = static_cast(uprv_strlen(builtinUnit.getSubtype())); + if (subtypeLen > 7 && uprv_strcmp(builtinUnit.getSubtype() + subtypeLen - 7, "-person") == 0) { + subtypeForResource = {builtinUnit.getSubtype(), subtypeLen - 7}; + } else { + subtypeForResource = builtinUnit.getSubtype(); + } + + CharString key; + key.append("units/", status); + key.append(builtinUnit.getType(), status); + key.append("/", status); + key.append(subtypeForResource, status); + key.append("/gender", status); + + UErrorCode localStatus = status; + int32_t resultLen = 0; + const char16_t *result = + ures_getStringByKeyWithFallback(unitsBundle.getAlias(), key.data(), &resultLen, &localStatus); + if (U_SUCCESS(localStatus)) { + status = localStatus; + return UnicodeString(true, result, resultLen); + } else { + // TODO(icu-units#28): "$unitRes/gender" does not exist. Do we want to + // check whether the parent "$unitRes" exists? Then we could return + // U_MISSING_RESOURCE_ERROR for incorrect usage (e.g. builtinUnit not + // being a builtin). + return {}; + } +} + +// Loads data from a resource tree with paths matching +// $key/$pluralForm/$gender/$case, with lateral inheritance for missing cases +// and genders. +// +// An InflectedPluralSink is configured to load data for a specific gender and +// case. It loads all plural forms, because selection between plural forms is +// dependent upon the value being formatted. +// +// See data/unit/de.txt and data/unit/fr.txt for examples - take a look at +// units/compound/power2: German has case, French has differences for gender, +// but no case. +// +// TODO(icu-units#138): Conceptually similar to PluralTableSink, however the +// tree structures are different. After homogenizing the structures, we may be +// able to unify the two classes. +// +// TODO: Spec violation: expects presence of "count" - does not fallback to an +// absent "count"! If this fallback were added, getCompoundValue could be +// superseded? +class InflectedPluralSink : public ResourceSink { + public: + // Accepts `char*` rather than StringPiece because + // ResourceTable::findValue(...) requires a null-terminated `char*`. + // + // NOTE: outArray MUST have a length of at least ARRAY_LENGTH. No bounds + // checking is performed. + explicit InflectedPluralSink(const char *gender, const char *caseVariant, UnicodeString *outArray) + : gender(gender), caseVariant(caseVariant), outArray(outArray) { + // Initialize the array to bogus strings. + for (int32_t i = 0; i < ARRAY_LENGTH; i++) { + outArray[i].setToBogus(); + } + } + + // See ResourceSink::put(). + void put(const char *key, ResourceValue &value, UBool /*noFallback*/, UErrorCode &status) override { + int32_t pluralIndex = getIndex(key, status); + if (U_FAILURE(status)) { return; } + if (!outArray[pluralIndex].isBogus()) { + // We already have a pattern + return; + } + ResourceTable genderTable = value.getTable(status); + ResourceTable caseTable; // This instance has to outlive `value` + if (loadForPluralForm(genderTable, caseTable, value, status)) { + outArray[pluralIndex] = value.getUnicodeString(status); + } + } + + private: + // Tries to load data for the configured gender from `genderTable`. Returns + // true if found, returning the data in `value`. The returned data will be + // for the configured gender if found, falling back to "neuter" and + // no-gender if not. The caseTable parameter holds the intermediate + // ResourceTable for the sake of lifetime management. + bool loadForPluralForm(const ResourceTable &genderTable, + ResourceTable &caseTable, + ResourceValue &value, + UErrorCode &status) { + if (uprv_strcmp(gender, "") != 0) { + if (loadForGender(genderTable, gender, caseTable, value, status)) { + return true; + } + if (uprv_strcmp(gender, "neuter") != 0 && + loadForGender(genderTable, "neuter", caseTable, value, status)) { + return true; + } + } + if (loadForGender(genderTable, "_", caseTable, value, status)) { + return true; + } + return false; + } + + // Tries to load data for the given gender from `genderTable`. Returns true + // if found, returning the data in `value`. The returned data will be for + // the configured case if found, falling back to "nominative" and no-case if + // not. + bool loadForGender(const ResourceTable &genderTable, + const char *genderVal, + ResourceTable &caseTable, + ResourceValue &value, + UErrorCode &status) { + if (!genderTable.findValue(genderVal, value)) { + return false; + } + caseTable = value.getTable(status); + if (uprv_strcmp(caseVariant, "") != 0) { + if (loadForCase(caseTable, caseVariant, value)) { + return true; + } + if (uprv_strcmp(caseVariant, "nominative") != 0 && + loadForCase(caseTable, "nominative", value)) { + return true; + } + } + if (loadForCase(caseTable, "_", value)) { + return true; + } + return false; + } + + // Tries to load data for the given case from `caseTable`. Returns true if + // found, returning the data in `value`. + bool loadForCase(const ResourceTable &caseTable, const char *caseValue, ResourceValue &value) { + if (!caseTable.findValue(caseValue, value)) { + return false; + } + return true; + } + + const char *gender; + const char *caseVariant; + UnicodeString *outArray; +}; + +// Fetches localised formatting patterns for the given subKey. See documentation +// for InflectedPluralSink for details. +// +// Data is loaded for the appropriate unit width, with missing data filled in +// from unitsShort. +void getInflectedMeasureData(StringPiece subKey, + const Locale &locale, + const UNumberUnitWidth &width, + const char *gender, + const char *caseVariant, + UnicodeString *outArray, + UErrorCode &status) { + InflectedPluralSink sink(gender, caseVariant, outArray); + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_UNIT, locale.getName(), &status)); + if (U_FAILURE(status)) { return; } + + CharString key; + key.append("units", status); + if (width == UNUM_UNIT_WIDTH_NARROW) { + key.append("Narrow", status); + } else if (width == UNUM_UNIT_WIDTH_SHORT) { + key.append("Short", status); + } + key.append("/", status); + key.append(subKey, status); + + UErrorCode localStatus = status; + ures_getAllChildrenWithFallback(unitsBundle.getAlias(), key.data(), sink, localStatus); + if (width == UNUM_UNIT_WIDTH_SHORT) { + status = localStatus; + return; + } +} + +class PluralTableSink : public ResourceSink { + public: + // NOTE: outArray MUST have a length of at least ARRAY_LENGTH. No bounds + // checking is performed. + explicit PluralTableSink(UnicodeString *outArray) : outArray(outArray) { + // Initialize the array to bogus strings. + for (int32_t i = 0; i < ARRAY_LENGTH; i++) { + outArray[i].setToBogus(); + } + } + + void put(const char *key, ResourceValue &value, UBool /*noFallback*/, UErrorCode &status) override { + if (uprv_strcmp(key, "case") == 0) { + return; + } + int32_t index = getIndex(key, status); + if (U_FAILURE(status)) { return; } + if (!outArray[index].isBogus()) { + return; + } + outArray[index] = value.getUnicodeString(status); + if (U_FAILURE(status)) { return; } + } + + private: + UnicodeString *outArray; +}; + +/** + * Populates outArray with `locale`-specific values for `unit` through use of + * PluralTableSink. Only the set of basic units are supported! + * + * Reading from resources *unitsNarrow* and *unitsShort* (for width + * UNUM_UNIT_WIDTH_NARROW), or just *unitsShort* (for width + * UNUM_UNIT_WIDTH_SHORT). For other widths, it reads just "units". + * + * @param unit must be a built-in unit, i.e. must have a type and subtype, + * listed in gTypes and gSubTypes in measunit.cpp. + * @param unitDisplayCase the empty string and "nominative" are treated the + * same. For other cases, strings for the requested case are used if found. + * (For any missing case-specific data, we fall back to nominative.) + * @param outArray must be of fixed length ARRAY_LENGTH. + */ +void getMeasureData(const Locale &locale, + const MeasureUnit &unit, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + UnicodeString *outArray, + UErrorCode &status) { + PluralTableSink sink(outArray); + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_UNIT, locale.getName(), &status)); + if (U_FAILURE(status)) { return; } + + CharString subKey; + subKey.append("/", status); + subKey.append(unit.getType(), status); + subKey.append("/", status); + + // Check if unitSubType is an alias or not. + LocalUResourceBundlePointer aliasBundle(ures_open(U_ICUDATA_ALIAS, "metadata", &status)); + + UErrorCode aliasStatus = status; + StackUResourceBundle aliasFillIn; + CharString aliasKey; + aliasKey.append("alias/unit/", aliasStatus); + aliasKey.append(unit.getSubtype(), aliasStatus); + aliasKey.append("/replacement", aliasStatus); + ures_getByKeyWithFallback(aliasBundle.getAlias(), aliasKey.data(), aliasFillIn.getAlias(), + &aliasStatus); + CharString unitSubType; + if (!U_FAILURE(aliasStatus)) { + // This means the subType is an alias. Then, replace unitSubType with the replacement. + auto replacement = ures_getUnicodeString(aliasFillIn.getAlias(), &status); + unitSubType.appendInvariantChars(replacement, status); + } else { + unitSubType.append(unit.getSubtype(), status); + } + + // Map duration-year-person, duration-week-person, etc. to duration-year, duration-week, ... + // TODO(ICU-20400): Get duration-*-person data properly with aliases. + int32_t subtypeLen = static_cast(uprv_strlen(unitSubType.data())); + if (subtypeLen > 7 && uprv_strcmp(unitSubType.data() + subtypeLen - 7, "-person") == 0) { + subKey.append({unitSubType.data(), subtypeLen - 7}, status); + } else { + subKey.append({unitSubType.data(), subtypeLen}, status); + } + + if (width != UNUM_UNIT_WIDTH_FULL_NAME) { + UErrorCode localStatus = status; + CharString genderKey; + genderKey.append("units", localStatus); + genderKey.append(subKey, localStatus); + genderKey.append("/gender", localStatus); + StackUResourceBundle fillIn; + ures_getByKeyWithFallback(unitsBundle.getAlias(), genderKey.data(), fillIn.getAlias(), + &localStatus); + outArray[GENDER_INDEX] = ures_getUnicodeString(fillIn.getAlias(), &localStatus); + } + + CharString key; + key.append("units", status); + if (width == UNUM_UNIT_WIDTH_NARROW) { + key.append("Narrow", status); + } else if (width == UNUM_UNIT_WIDTH_SHORT) { + key.append("Short", status); + } + key.append(subKey, status); + + // Grab desired case first, if available. Then grab no-case data to fill in + // the gaps. + if (width == UNUM_UNIT_WIDTH_FULL_NAME && unitDisplayCase[0] != 0) { + CharString caseKey; + caseKey.append(key, status); + caseKey.append("/case/", status); + caseKey.append(unitDisplayCase, status); + + UErrorCode localStatus = U_ZERO_ERROR; + // TODO(icu-units#138): our fallback logic is not spec-compliant: + // lateral fallback should happen before locale fallback. Switch to + // getInflectedMeasureData after homogenizing data format? Find a unit + // test case that demonstrates the incorrect fallback logic (via + // regional variant of an inflected language?) + ures_getAllChildrenWithFallback(unitsBundle.getAlias(), caseKey.data(), sink, localStatus); + } + + // TODO(icu-units#138): our fallback logic is not spec-compliant: we + // check the given case, then go straight to the no-case data. The spec + // states we should first look for case="nominative". As part of #138, + // either get the spec changed, or add unit tests that warn us if + // case="nominative" data differs from no-case data? + UErrorCode localStatus = U_ZERO_ERROR; + ures_getAllChildrenWithFallback(unitsBundle.getAlias(), key.data(), sink, localStatus); + if (width == UNUM_UNIT_WIDTH_SHORT) { + if (U_FAILURE(localStatus)) { + status = localStatus; + } + return; + } +} + +// NOTE: outArray MUST have a length of at least ARRAY_LENGTH. +void getCurrencyLongNameData(const Locale &locale, const CurrencyUnit ¤cy, UnicodeString *outArray, + UErrorCode &status) { + // In ICU4J, this method gets a CurrencyData from CurrencyData.provider. + // TODO(ICU4J): Implement this without going through CurrencyData, like in ICU4C? + PluralTableSink sink(outArray); + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_CURR, locale.getName(), &status)); + if (U_FAILURE(status)) { return; } + ures_getAllChildrenWithFallback(unitsBundle.getAlias(), "CurrencyUnitPatterns", sink, status); + if (U_FAILURE(status)) { return; } + for (int32_t i = 0; i < StandardPlural::Form::COUNT; i++) { + UnicodeString &pattern = outArray[i]; + if (pattern.isBogus()) { + continue; + } + int32_t longNameLen = 0; + const char16_t *longName = ucurr_getPluralName( + currency.getISOCurrency(), + locale.getName(), + nullptr /* isChoiceFormat */, + StandardPlural::getKeyword(static_cast(i)), + &longNameLen, + &status); + // Example pattern from data: "{0} {1}" + // Example output after find-and-replace: "{0} US dollars" + pattern.findAndReplace(UnicodeString(u"{1}"), UnicodeString(longName, longNameLen)); + } +} + +UnicodeString getCompoundValue(StringPiece compoundKey, + const Locale &locale, + const UNumberUnitWidth &width, + UErrorCode &status) { + LocalUResourceBundlePointer unitsBundle(ures_open(U_ICUDATA_UNIT, locale.getName(), &status)); + if (U_FAILURE(status)) { return {}; } + CharString key; + key.append("units", status); + if (width == UNUM_UNIT_WIDTH_NARROW) { + key.append("Narrow", status); + } else if (width == UNUM_UNIT_WIDTH_SHORT) { + key.append("Short", status); + } + key.append("/compound/", status); + key.append(compoundKey, status); + + UErrorCode localStatus = status; + int32_t len = 0; + const char16_t *ptr = + ures_getStringByKeyWithFallback(unitsBundle.getAlias(), key.data(), &len, &localStatus); + if (U_FAILURE(localStatus) && width != UNUM_UNIT_WIDTH_SHORT) { + // Fall back to short, which contains more compound data + key.clear(); + key.append("unitsShort/compound/", status); + key.append(compoundKey, status); + ptr = ures_getStringByKeyWithFallback(unitsBundle.getAlias(), key.data(), &len, &status); + } else { + status = localStatus; + } + if (U_FAILURE(status)) { + return {}; + } + return UnicodeString(ptr, len); +} + +/** + * Loads and applies deriveComponent rules from CLDR's grammaticalFeatures.xml. + * + * Consider a deriveComponent rule that looks like this: + * + * + * + * Instantiating an instance as follows: + * + * DerivedComponents d(loc, "case", "per"); + * + * Applying the rule in the XML element above, `d.value0("foo")` will be "foo", + * and `d.value1("foo")` will be "nominative". + * + * The values returned by value0(...) and value1(...) are valid only while the + * instance exists. In case of any kind of failure, value0(...) and value1(...) + * will return "". + */ +class DerivedComponents { + public: + /** + * Constructor. + * + * The feature and structure parameters must be null-terminated. The string + * referenced by compoundValue must exist for longer than the + * DerivedComponents instance. + */ + DerivedComponents(const Locale &locale, const char *feature, const char *structure) { + StackUResourceBundle derivationsBundle, stackBundle; + ures_openDirectFillIn(derivationsBundle.getAlias(), nullptr, "grammaticalFeatures", &status); + ures_getByKey(derivationsBundle.getAlias(), "grammaticalData", derivationsBundle.getAlias(), + &status); + ures_getByKey(derivationsBundle.getAlias(), "derivations", derivationsBundle.getAlias(), + &status); + if (U_FAILURE(status)) { + return; + } + UErrorCode localStatus = U_ZERO_ERROR; + // TODO(icu-units#28): use standard normal locale resolution algorithms + // rather than just grabbing language: + ures_getByKey(derivationsBundle.getAlias(), locale.getLanguage(), stackBundle.getAlias(), + &localStatus); + // TODO(icu-units#28): + // - code currently assumes if the locale exists, the rules are there - + // instead of falling back to root when the requested rule is missing. + // - investigate ures.h functions, see if one that uses res_findResource() + // might be better (or use res_findResource directly), or maybe help + // improve ures documentation to guide function selection? + if (localStatus == U_MISSING_RESOURCE_ERROR) { + ures_getByKey(derivationsBundle.getAlias(), "root", stackBundle.getAlias(), &status); + } else { + status = localStatus; + } + ures_getByKey(stackBundle.getAlias(), "component", stackBundle.getAlias(), &status); + ures_getByKey(stackBundle.getAlias(), feature, stackBundle.getAlias(), &status); + ures_getByKey(stackBundle.getAlias(), structure, stackBundle.getAlias(), &status); + UnicodeString val0 = ures_getUnicodeStringByIndex(stackBundle.getAlias(), 0, &status); + UnicodeString val1 = ures_getUnicodeStringByIndex(stackBundle.getAlias(), 1, &status); + if (U_SUCCESS(status)) { + if (val0.compare(UnicodeString(u"compound")) == 0) { + compound0_ = true; + } else { + compound0_ = false; + value0_.appendInvariantChars(val0, status); + } + if (val1.compare(UnicodeString(u"compound")) == 0) { + compound1_ = true; + } else { + compound1_ = false; + value1_.appendInvariantChars(val1, status); + } + } + } + + // Returns a StringPiece that is only valid as long as the instance exists. + StringPiece value0(const StringPiece compoundValue) const { + return compound0_ ? compoundValue : value0_.toStringPiece(); + } + + // Returns a StringPiece that is only valid as long as the instance exists. + StringPiece value1(const StringPiece compoundValue) const { + return compound1_ ? compoundValue : value1_.toStringPiece(); + } + + // Returns a char* that is only valid as long as the instance exists. + const char *value0(const char *compoundValue) const { + return compound0_ ? compoundValue : value0_.data(); + } + + // Returns a char* that is only valid as long as the instance exists. + const char *value1(const char *compoundValue) const { + return compound1_ ? compoundValue : value1_.data(); + } + + private: + UErrorCode status = U_ZERO_ERROR; + + // Holds strings referred to by value0 and value1; + bool compound0_ = false, compound1_ = false; + CharString value0_, value1_; +}; + +// TODO(icu-units#28): test somehow? Associate with an ICU ticket for adding +// testsuite support for testing with synthetic data? +/** + * Loads and returns the value in rules that look like these: + * + * + * + * + * Currently a fake example, but spec compliant: + * + * + * NOTE: If U_FAILURE(status), returns an empty string. + */ +UnicodeString +getDeriveCompoundRule(Locale locale, const char *feature, const char *structure, UErrorCode &status) { + StackUResourceBundle derivationsBundle, stackBundle; + ures_openDirectFillIn(derivationsBundle.getAlias(), nullptr, "grammaticalFeatures", &status); + ures_getByKey(derivationsBundle.getAlias(), "grammaticalData", derivationsBundle.getAlias(), + &status); + ures_getByKey(derivationsBundle.getAlias(), "derivations", derivationsBundle.getAlias(), &status); + // TODO: use standard normal locale resolution algorithms rather than just grabbing language: + ures_getByKey(derivationsBundle.getAlias(), locale.getLanguage(), stackBundle.getAlias(), &status); + // TODO: + // - code currently assumes if the locale exists, the rules are there - + // instead of falling back to root when the requested rule is missing. + // - investigate ures.h functions, see if one that uses res_findResource() + // might be better (or use res_findResource directly), or maybe help + // improve ures documentation to guide function selection? + if (status == U_MISSING_RESOURCE_ERROR) { + status = U_ZERO_ERROR; + ures_getByKey(derivationsBundle.getAlias(), "root", stackBundle.getAlias(), &status); + } + ures_getByKey(stackBundle.getAlias(), "compound", stackBundle.getAlias(), &status); + ures_getByKey(stackBundle.getAlias(), feature, stackBundle.getAlias(), &status); + UnicodeString uVal = ures_getUnicodeStringByKey(stackBundle.getAlias(), structure, &status); + if (U_FAILURE(status)) { + return {}; + } + U_ASSERT(!uVal.isBogus()); + return uVal; +} + +// Returns the gender string for structures following these rules: +// +// +// +// +// Fake example: +// +// +// data0 and data1 should be pattern arrays (UnicodeString[ARRAY_SIZE]) that +// correspond to value="0" and value="1". +// +// Pass a nullptr to data1 if the structure has no concept of value="1" (e.g. +// "prefix" doesn't). +UnicodeString getDerivedGender(Locale locale, + const char *structure, + UnicodeString *data0, + UnicodeString *data1, + UErrorCode &status) { + UnicodeString val = getDeriveCompoundRule(locale, "gender", structure, status); + if (val.length() == 1) { + switch (val[0]) { + case u'0': + return data0[GENDER_INDEX]; + case u'1': + if (data1 == nullptr) { + return {}; + } + return data1[GENDER_INDEX]; + } + } + return val; +} + +//////////////////////// +/// END DATA LOADING /// +//////////////////////// + +// TODO: promote this somewhere? It's based on patternprops.cpp' trimWhitespace +const char16_t *trimSpaceChars(const char16_t *s, int32_t &length) { + if (length <= 0 || (!u_isJavaSpaceChar(s[0]) && !u_isJavaSpaceChar(s[length - 1]))) { + return s; + } + int32_t start = 0; + int32_t limit = length; + while (start < limit && u_isJavaSpaceChar(s[start])) { + ++start; + } + if (start < limit) { + // There is non-white space at start; we will not move limit below that, + // so we need not test start 0); // Else it would not be COMPOUND + if (mui.singleUnits[endSlice]->dimensionality < 0) { + // We have a -per- construct + UnicodeString perRule = getDeriveCompoundRule(locale, "gender", "per", status); + if (perRule.length() != 1) { + // Fixed gender for -per- units + return perRule; + } + if (perRule[0] == u'1') { + // Find the start of the denominator. We already know there is one. + while (mui.singleUnits[startSlice]->dimensionality >= 0) { + startSlice++; + } + } else { + // Find the end of the numerator + while (endSlice >= 0 && mui.singleUnits[endSlice]->dimensionality < 0) { + endSlice--; + } + if (endSlice < 0) { + // We have only a denominator, e.g. "per-second". + // TODO(icu-units#28): find out what gender to use in the + // absence of a first value - mentioned in CLDR-14253. + return {}; + } + } + } + if (endSlice > startSlice) { + // We have a -times- construct + UnicodeString timesRule = getDeriveCompoundRule(locale, "gender", "times", status); + if (timesRule.length() != 1) { + // Fixed gender for -times- units + return timesRule; + } + if (timesRule[0] == u'0') { + endSlice = startSlice; + } else { + // We assume timesRule[0] == u'1' + startSlice = endSlice; + } + } + U_ASSERT(startSlice == endSlice); + singleUnitIndex = startSlice; + } else if (mui.complexity == UMEASURE_UNIT_MIXED) { + status = U_INTERNAL_PROGRAM_ERROR; + return {}; + } else { + U_ASSERT(mui.complexity == UMEASURE_UNIT_SINGLE); + U_ASSERT(mui.singleUnits.length() == 1); + } + + // Now we know which singleUnit's gender we want + const SingleUnitImpl *singleUnit = mui.singleUnits[singleUnitIndex]; + // Check for any power-prefix gender override: + if (std::abs(singleUnit->dimensionality) != 1) { + UnicodeString powerRule = getDeriveCompoundRule(locale, "gender", "power", status); + if (powerRule.length() != 1) { + // Fixed gender for -powN- units + return powerRule; + } + // powerRule[0] == u'0'; u'1' not currently in spec. + } + // Check for any SI and binary prefix gender override: + if (std::abs(singleUnit->dimensionality) != 1) { + UnicodeString prefixRule = getDeriveCompoundRule(locale, "gender", "prefix", status); + if (prefixRule.length() != 1) { + // Fixed gender for -powN- units + return prefixRule; + } + // prefixRule[0] == u'0'; u'1' not currently in spec. + } + // Now we've boiled it down to the gender of one simple unit identifier: + return getGenderForBuiltin(locale, MeasureUnit::forIdentifier(singleUnit->getSimpleUnitID(), status), + status); +} + +void maybeCalculateGender(const Locale &locale, + const MeasureUnit &unitRef, + UnicodeString *outArray, + UErrorCode &status) { + if (outArray[GENDER_INDEX].isBogus()) { + UnicodeString meterGender = getGenderForBuiltin(locale, MeasureUnit::getMeter(), status); + if (meterGender.isEmpty()) { + // No gender for meter: assume ungendered language + return; + } + // We have a gendered language, but are lacking gender for unitRef. + outArray[GENDER_INDEX] = calculateGenderForUnit(locale, unitRef, status); + } +} + +} // namespace + +void LongNameHandler::forMeasureUnit(const Locale &loc, + const MeasureUnit &unitRef, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + LongNameHandler *fillIn, + UErrorCode &status) { + // From https://unicode.org/reports/tr35/tr35-general.html#compound-units - + // Points 1 and 2 are mostly handled by MeasureUnit: + // + // 1. If the unitId is empty or invalid, fail + // 2. Put the unitId into normalized order + U_ASSERT(fillIn != nullptr); + + if (uprv_strcmp(unitRef.getType(), "") != 0) { + // Handling built-in units: + // + // 3. Set result to be getValue(unitId with length, pluralCategory, caseVariant) + // - If result is not empty, return it + UnicodeString simpleFormats[ARRAY_LENGTH]; + getMeasureData(loc, unitRef, width, unitDisplayCase, simpleFormats, status); + maybeCalculateGender(loc, unitRef, simpleFormats, status); + if (U_FAILURE(status)) { + return; + } + fillIn->rules = rules; + fillIn->parent = parent; + fillIn->simpleFormatsToModifiers(simpleFormats, + {UFIELD_CATEGORY_NUMBER, UNUM_MEASURE_UNIT_FIELD}, status); + if (!simpleFormats[GENDER_INDEX].isBogus()) { + fillIn->gender = getGenderString(simpleFormats[GENDER_INDEX], status); + } + return; + + // TODO(icu-units#145): figure out why this causes a failure in + // format/MeasureFormatTest/TestIndividualPluralFallback and other + // tests, when it should have been an alternative for the lines above: + + // forArbitraryUnit(loc, unitRef, width, unitDisplayCase, fillIn, status); + // fillIn->rules = rules; + // fillIn->parent = parent; + // return; + } else { + // Check if it is a MeasureUnit this constructor handles: this + // constructor does not handle mixed units + U_ASSERT(unitRef.getComplexity(status) != UMEASURE_UNIT_MIXED); + forArbitraryUnit(loc, unitRef, width, unitDisplayCase, fillIn, status); + fillIn->rules = rules; + fillIn->parent = parent; + return; + } +} + +void LongNameHandler::forArbitraryUnit(const Locale &loc, + const MeasureUnit &unitRef, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + LongNameHandler *fillIn, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + if (fillIn == nullptr) { + status = U_INTERNAL_PROGRAM_ERROR; + return; + } + + // Numbered list items are from the algorithms at + // https://unicode.org/reports/tr35/tr35-general.html#compound-units: + // + // 4. Divide the unitId into numerator (the part before the "-per-") and + // denominator (the part after the "-per-). If both are empty, fail + MeasureUnitImpl unit; + MeasureUnitImpl perUnit; + { + MeasureUnitImpl fullUnit = MeasureUnitImpl::forMeasureUnitMaybeCopy(unitRef, status); + if (U_FAILURE(status)) { + return; + } + for (int32_t i = 0; i < fullUnit.singleUnits.length(); i++) { + SingleUnitImpl *subUnit = fullUnit.singleUnits[i]; + if (subUnit->dimensionality > 0) { + unit.appendSingleUnit(*subUnit, status); + } else { + subUnit->dimensionality *= -1; + perUnit.appendSingleUnit(*subUnit, status); + } + } + } + + // TODO(icu-units#28): check placeholder logic, see if it needs to be + // present here instead of only in processPatternTimes: + // + // 5. Set both globalPlaceholder and globalPlaceholderPosition to be empty + + DerivedComponents derivedPerCases(loc, "case", "per"); + + // 6. numeratorUnitString + UnicodeString numeratorUnitData[ARRAY_LENGTH]; + processPatternTimes(std::move(unit), loc, width, derivedPerCases.value0(unitDisplayCase), + numeratorUnitData, status); + + // 7. denominatorUnitString + UnicodeString denominatorUnitData[ARRAY_LENGTH]; + processPatternTimes(std::move(perUnit), loc, width, derivedPerCases.value1(unitDisplayCase), + denominatorUnitData, status); + + // TODO(icu-units#139): + // - implement DerivedComponents for "plural/times" and "plural/power": + // French has different rules, we'll be producing the wrong results + // currently. (Prove via tests!) + // - implement DerivedComponents for "plural/per", "plural/prefix", + // "case/times", "case/power", and "case/prefix" - although they're + // currently hardcoded. Languages with different rules are surely on the + // way. + // + // Currently we only use "case/per", "plural/times", "case/times", and + // "case/power". + // + // This may have impact on multiSimpleFormatsToModifiers(...) below too? + // These rules are currently (ICU 69) all the same and hard-coded below. + UnicodeString perUnitPattern; + if (!denominatorUnitData[PER_INDEX].isBogus()) { + // If we have no denominator, we obtain the empty string: + perUnitPattern = denominatorUnitData[PER_INDEX]; + } else { + // 8. Set perPattern to be getValue([per], locale, length) + UnicodeString rawPerUnitFormat = getCompoundValue("per", loc, width, status); + // rawPerUnitFormat is something like "{0} per {1}"; we need to substitute in the secondary unit. + SimpleFormatter perPatternFormatter(rawPerUnitFormat, 2, 2, status); + if (U_FAILURE(status)) { + return; + } + // Plural and placeholder handling for 7. denominatorUnitString: + // TODO(icu-units#139): hardcoded: + // + UnicodeString denominatorFormat = + getWithPlural(denominatorUnitData, StandardPlural::Form::ONE, status); + // Some "one" pattern may not contain "{0}". For example in "ar" or "ne" locale. + SimpleFormatter denominatorFormatter(denominatorFormat, 0, 1, status); + if (U_FAILURE(status)) { + return; + } + UnicodeString denominatorPattern = denominatorFormatter.getTextWithNoArguments(); + int32_t trimmedLen = denominatorPattern.length(); + const char16_t *trimmed = trimSpaceChars(denominatorPattern.getBuffer(), trimmedLen); + UnicodeString denominatorString(false, trimmed, trimmedLen); + // 9. If the denominatorString is empty, set result to + // [numeratorString], otherwise set result to format(perPattern, + // numeratorString, denominatorString) + // + // TODO(icu-units#28): Why does UnicodeString need to be explicit in the + // following line? + perPatternFormatter.format(UnicodeString(u"{0}"), denominatorString, perUnitPattern, status); + if (U_FAILURE(status)) { + return; + } + } + if (perUnitPattern.length() == 0) { + fillIn->simpleFormatsToModifiers(numeratorUnitData, + {UFIELD_CATEGORY_NUMBER, UNUM_MEASURE_UNIT_FIELD}, status); + } else { + fillIn->multiSimpleFormatsToModifiers(numeratorUnitData, perUnitPattern, + {UFIELD_CATEGORY_NUMBER, UNUM_MEASURE_UNIT_FIELD}, status); + } + + // Gender + // + // TODO(icu-units#28): find out what gender to use in the absence of a first + // value - e.g. what's the gender of "per-second"? Mentioned in CLDR-14253. + // + // gender/per deriveCompound rules don't say: + // + fillIn->gender = getGenderString( + getDerivedGender(loc, "per", numeratorUnitData, denominatorUnitData, status), status); +} + +void LongNameHandler::processPatternTimes(MeasureUnitImpl &&productUnit, + Locale loc, + const UNumberUnitWidth &width, + const char *caseVariant, + UnicodeString *outArray, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + if (productUnit.complexity == UMEASURE_UNIT_MIXED) { + // These are handled by MixedUnitLongNameHandler + status = U_UNSUPPORTED_ERROR; + return; + } + +#if U_DEBUG + for (int32_t pluralIndex = 0; pluralIndex < ARRAY_LENGTH; pluralIndex++) { + U_ASSERT(outArray[pluralIndex].length() == 0); + U_ASSERT(!outArray[pluralIndex].isBogus()); + } +#endif + + if (productUnit.identifier.isEmpty()) { + // TODO(icu-units#28): consider when serialize should be called. + // identifier might also be empty for MeasureUnit(). + productUnit.serialize(status); + } + if (U_FAILURE(status)) { + return; + } + if (productUnit.identifier.length() == 0) { + // MeasureUnit(): no units: return empty strings. + return; + } + + MeasureUnit builtinUnit; + if (MeasureUnit::findBySubType(productUnit.identifier.toStringPiece(), &builtinUnit)) { + // TODO(icu-units#145): spec doesn't cover builtin-per-builtin, it + // breaks them all down. Do we want to drop this? + // - findBySubType isn't super efficient, if we skip it and go to basic + // singles, we don't have to construct MeasureUnit's anymore. + // - Check all the existing unit tests that fail without this: is it due + // to incorrect fallback via getMeasureData? + // - Do those unit tests cover this code path representatively? + if (builtinUnit != MeasureUnit()) { + getMeasureData(loc, builtinUnit, width, caseVariant, outArray, status); + maybeCalculateGender(loc, builtinUnit, outArray, status); + } + return; + } + + // 2. Set timesPattern to be getValue(times, locale, length) + UnicodeString timesPattern = getCompoundValue("times", loc, width, status); + SimpleFormatter timesPatternFormatter(timesPattern, 2, 2, status); + if (U_FAILURE(status)) { + return; + } + + PlaceholderPosition globalPlaceholder[ARRAY_LENGTH]; + char16_t globalJoinerChar = 0; + // Numbered list items are from the algorithms at + // https://unicode.org/reports/tr35/tr35-general.html#compound-units: + // + // pattern(...) point 5: + // - Set both globalPlaceholder and globalPlaceholderPosition to be empty + // + // 3. Set result to be empty + for (int32_t pluralIndex = 0; pluralIndex < ARRAY_LENGTH; pluralIndex++) { + // Initial state: empty string pattern, via all falling back to OTHER: + if (pluralIndex == StandardPlural::Form::OTHER) { + outArray[pluralIndex].remove(); + } else { + outArray[pluralIndex].setToBogus(); + } + globalPlaceholder[pluralIndex] = PH_EMPTY; + } + + // Empty string represents "compound" (propagate the plural form). + const char *pluralCategory = ""; + DerivedComponents derivedTimesPlurals(loc, "plural", "times"); + DerivedComponents derivedTimesCases(loc, "case", "times"); + DerivedComponents derivedPowerCases(loc, "case", "power"); + + // 4. For each single_unit in product_unit + for (int32_t singleUnitIndex = 0; singleUnitIndex < productUnit.singleUnits.length(); + singleUnitIndex++) { + SingleUnitImpl *singleUnit = productUnit.singleUnits[singleUnitIndex]; + const char *singlePluralCategory; + const char *singleCaseVariant; + // TODO(icu-units#28): ensure we have unit tests that change/fail if we + // assign incorrect case variants here: + if (singleUnitIndex < productUnit.singleUnits.length() - 1) { + // 4.1. If hasMultiple + singlePluralCategory = derivedTimesPlurals.value0(pluralCategory); + singleCaseVariant = derivedTimesCases.value0(caseVariant); + pluralCategory = derivedTimesPlurals.value1(pluralCategory); + caseVariant = derivedTimesCases.value1(caseVariant); + } else { + singlePluralCategory = derivedTimesPlurals.value1(pluralCategory); + singleCaseVariant = derivedTimesCases.value1(caseVariant); + } + + // 4.2. Get the gender of that single_unit + MeasureUnit simpleUnit; + if (!MeasureUnit::findBySubType(singleUnit->getSimpleUnitID(), &simpleUnit)) { + // Ideally all simple units should be known, but they're not: + // 100-kilometer is internally treated as a simple unit, but it is + // not a built-in unit and does not have formatting data in CLDR 39. + // + // TODO(icu-units#28): test (desirable) invariants in unit tests. + status = U_UNSUPPORTED_ERROR; + return; + } + const char *gender = getGenderString(getGenderForBuiltin(loc, simpleUnit, status), status); + + // 4.3. If singleUnit starts with a dimensionality_prefix, such as 'square-' + U_ASSERT(singleUnit->dimensionality > 0); + int32_t dimensionality = singleUnit->dimensionality; + UnicodeString dimensionalityPrefixPatterns[ARRAY_LENGTH]; + if (dimensionality != 1) { + // 4.3.1. set dimensionalityPrefixPattern to be + // getValue(that dimensionality_prefix, locale, length, singlePluralCategory, singleCaseVariant, gender), + // such as "{0} kwadratowym" + CharString dimensionalityKey("compound/power", status); + dimensionalityKey.appendNumber(dimensionality, status); + getInflectedMeasureData(dimensionalityKey.toStringPiece(), loc, width, gender, + singleCaseVariant, dimensionalityPrefixPatterns, status); + if (U_FAILURE(status)) { + // At the time of writing, only pow2 and pow3 are supported. + // Attempting to format other powers results in a + // U_RESOURCE_TYPE_MISMATCH. We convert the error if we + // understand it: + if (status == U_RESOURCE_TYPE_MISMATCH && dimensionality > 3) { + status = U_UNSUPPORTED_ERROR; + } + return; + } + + // TODO(icu-units#139): + // 4.3.2. set singlePluralCategory to be power0(singlePluralCategory) + + // 4.3.3. set singleCaseVariant to be power0(singleCaseVariant) + singleCaseVariant = derivedPowerCases.value0(singleCaseVariant); + // 4.3.4. remove the dimensionality_prefix from singleUnit + singleUnit->dimensionality = 1; + } + + // 4.4. if singleUnit starts with an si_prefix, such as 'centi' + UMeasurePrefix prefix = singleUnit->unitPrefix; + UnicodeString prefixPattern; + if (prefix != UMEASURE_PREFIX_ONE) { + // 4.4.1. set siPrefixPattern to be getValue(that si_prefix, locale, + // length), such as "centy{0}" + CharString prefixKey; + // prefixKey looks like "1024p3" or "10p-2": + prefixKey.appendNumber(umeas_getPrefixBase(prefix), status); + prefixKey.append('p', status); + prefixKey.appendNumber(umeas_getPrefixPower(prefix), status); + // Contains a pattern like "centy{0}". + prefixPattern = getCompoundValue(prefixKey.toStringPiece(), loc, width, status); + + // 4.4.2. set singlePluralCategory to be prefix0(singlePluralCategory) + // + // TODO(icu-units#139): that refers to these rules: + // + // though I'm not sure what other value they might end up having. + // + // 4.4.3. set singleCaseVariant to be prefix0(singleCaseVariant) + // + // TODO(icu-units#139): that refers to: + // but the prefix (value0) doesn't have case, the rest simply + // propagates. + + // 4.4.4. remove the si_prefix from singleUnit + singleUnit->unitPrefix = UMEASURE_PREFIX_ONE; + } + + // 4.5. Set corePattern to be the getValue(singleUnit, locale, length, + // singlePluralCategory, singleCaseVariant), such as "{0} metrem" + UnicodeString singleUnitArray[ARRAY_LENGTH]; + // At this point we are left with a Simple Unit: + U_ASSERT(uprv_strcmp(singleUnit->build(status).getIdentifier(), singleUnit->getSimpleUnitID()) == + 0); + getMeasureData(loc, singleUnit->build(status), width, singleCaseVariant, singleUnitArray, + status); + if (U_FAILURE(status)) { + // Shouldn't happen if we have data for all single units + return; + } + + // Calculate output gender + if (!singleUnitArray[GENDER_INDEX].isBogus()) { + U_ASSERT(!singleUnitArray[GENDER_INDEX].isEmpty()); + UnicodeString uVal; + + if (prefix != UMEASURE_PREFIX_ONE) { + singleUnitArray[GENDER_INDEX] = + getDerivedGender(loc, "prefix", singleUnitArray, nullptr, status); + } + + if (dimensionality != 1) { + singleUnitArray[GENDER_INDEX] = + getDerivedGender(loc, "power", singleUnitArray, nullptr, status); + } + + UnicodeString timesGenderRule = getDeriveCompoundRule(loc, "gender", "times", status); + if (timesGenderRule.length() == 1) { + switch (timesGenderRule[0]) { + case u'0': + if (singleUnitIndex == 0) { + U_ASSERT(outArray[GENDER_INDEX].isBogus()); + outArray[GENDER_INDEX] = singleUnitArray[GENDER_INDEX]; + } + break; + case u'1': + if (singleUnitIndex == productUnit.singleUnits.length() - 1) { + U_ASSERT(outArray[GENDER_INDEX].isBogus()); + outArray[GENDER_INDEX] = singleUnitArray[GENDER_INDEX]; + } + } + } else { + if (outArray[GENDER_INDEX].isBogus()) { + outArray[GENDER_INDEX] = timesGenderRule; + } + } + } + + // Calculate resulting patterns for each plural form + for (int32_t pluralIndex = 0; pluralIndex < StandardPlural::Form::COUNT; pluralIndex++) { + StandardPlural::Form plural = static_cast(pluralIndex); + + // singleUnitArray[pluralIndex] looks something like "{0} Meter" + if (outArray[pluralIndex].isBogus()) { + if (singleUnitArray[pluralIndex].isBogus()) { + // Let the usual plural fallback mechanism take care of this + // plural form + continue; + } else { + // Since our singleUnit can have a plural form that outArray + // doesn't yet have (relying on fallback to OTHER), we start + // by grabbing it with the normal plural fallback mechanism + outArray[pluralIndex] = getWithPlural(outArray, plural, status); + if (U_FAILURE(status)) { + return; + } + } + } + + if (uprv_strcmp(singlePluralCategory, "") != 0) { + plural = static_cast(getIndex(singlePluralCategory, status)); + } + + // 4.6. Extract(corePattern, coreUnit, placeholder, placeholderPosition) from that pattern. + UnicodeString coreUnit; + PlaceholderPosition placeholderPosition; + char16_t joinerChar; + extractCorePattern(getWithPlural(singleUnitArray, plural, status), coreUnit, + placeholderPosition, joinerChar); + + // 4.7 If the position is middle, then fail + if (placeholderPosition == PH_MIDDLE) { + status = U_UNSUPPORTED_ERROR; + return; + } + + // 4.8. If globalPlaceholder is empty + if (globalPlaceholder[pluralIndex] == PH_EMPTY) { + globalPlaceholder[pluralIndex] = placeholderPosition; + globalJoinerChar = joinerChar; + } else { + // Expect all units involved to have the same placeholder position + U_ASSERT(globalPlaceholder[pluralIndex] == placeholderPosition); + // TODO(icu-units#28): Do we want to add a unit test that checks + // for consistent joiner chars? Probably not, given how + // inconsistent they are. File a CLDR ticket with examples? + } + // Now coreUnit would be just "Meter" + + // 4.9. If siPrefixPattern is not empty + if (prefix != UMEASURE_PREFIX_ONE) { + SimpleFormatter prefixCompiled(prefixPattern, 1, 1, status); + if (U_FAILURE(status)) { + return; + } + + // 4.9.1. Set coreUnit to be the combineLowercasing(locale, length, siPrefixPattern, + // coreUnit) + UnicodeString tmp; + // combineLowercasing(locale, length, prefixPattern, coreUnit) + // + // TODO(icu-units#28): run this only if prefixPattern does not + // contain space characters - do languages "as", "bn", "hi", + // "kk", etc have concepts of upper and lower case?: + if (width == UNUM_UNIT_WIDTH_FULL_NAME) { + coreUnit.toLower(loc); + } + prefixCompiled.format(coreUnit, tmp, status); + if (U_FAILURE(status)) { + return; + } + coreUnit = tmp; + } + + // 4.10. If dimensionalityPrefixPattern is not empty + if (dimensionality != 1) { + SimpleFormatter dimensionalityCompiled( + getWithPlural(dimensionalityPrefixPatterns, plural, status), 1, 1, status); + if (U_FAILURE(status)) { + return; + } + + // 4.10.1. Set coreUnit to be the combineLowercasing(locale, length, + // dimensionalityPrefixPattern, coreUnit) + UnicodeString tmp; + // combineLowercasing(locale, length, prefixPattern, coreUnit) + // + // TODO(icu-units#28): run this only if prefixPattern does not + // contain space characters - do languages "as", "bn", "hi", + // "kk", etc have concepts of upper and lower case?: + if (width == UNUM_UNIT_WIDTH_FULL_NAME) { + coreUnit.toLower(loc); + } + dimensionalityCompiled.format(coreUnit, tmp, status); + if (U_FAILURE(status)) { + return; + } + coreUnit = tmp; + } + + if (outArray[pluralIndex].length() == 0) { + // 4.11. If the result is empty, set result to be coreUnit + outArray[pluralIndex] = coreUnit; + } else { + // 4.12. Otherwise set result to be format(timesPattern, result, coreUnit) + UnicodeString tmp; + timesPatternFormatter.format(outArray[pluralIndex], coreUnit, tmp, status); + outArray[pluralIndex] = tmp; + } + } + } + for (int32_t pluralIndex = 0; pluralIndex < StandardPlural::Form::COUNT; pluralIndex++) { + if (globalPlaceholder[pluralIndex] == PH_BEGINNING) { + UnicodeString tmp; + tmp.append(u"{0}", 3); + if (globalJoinerChar != 0) { + tmp.append(globalJoinerChar); + } + tmp.append(outArray[pluralIndex]); + outArray[pluralIndex] = tmp; + } else if (globalPlaceholder[pluralIndex] == PH_END) { + if (globalJoinerChar != 0) { + outArray[pluralIndex].append(globalJoinerChar); + } + outArray[pluralIndex].append(u"{0}", 3); + } + } +} + +UnicodeString LongNameHandler::getUnitDisplayName( + const Locale& loc, + const MeasureUnit& unit, + UNumberUnitWidth width, + UErrorCode& status) { + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + UnicodeString simpleFormats[ARRAY_LENGTH]; + getMeasureData(loc, unit, width, "", simpleFormats, status); + return simpleFormats[DNAM_INDEX]; +} + +UnicodeString LongNameHandler::getUnitPattern( + const Locale& loc, + const MeasureUnit& unit, + UNumberUnitWidth width, + StandardPlural::Form pluralForm, + UErrorCode& status) { + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + UnicodeString simpleFormats[ARRAY_LENGTH]; + getMeasureData(loc, unit, width, "", simpleFormats, status); + // The above already handles fallback from other widths to short + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + // Now handle fallback from other plural forms to OTHER + return (!(simpleFormats[pluralForm]).isBogus())? simpleFormats[pluralForm]: + simpleFormats[StandardPlural::Form::OTHER]; +} + +LongNameHandler* LongNameHandler::forCurrencyLongNames(const Locale &loc, const CurrencyUnit ¤cy, + const PluralRules *rules, + const MicroPropsGenerator *parent, + UErrorCode &status) { + auto* result = new LongNameHandler(rules, parent); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + UnicodeString simpleFormats[ARRAY_LENGTH]; + getCurrencyLongNameData(loc, currency, simpleFormats, status); + if (U_FAILURE(status)) { return nullptr; } + result->simpleFormatsToModifiers(simpleFormats, {UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}, status); + // TODO(icu-units#28): currency gender? + return result; +} + +void LongNameHandler::simpleFormatsToModifiers(const UnicodeString *simpleFormats, Field field, + UErrorCode &status) { + for (int32_t i = 0; i < StandardPlural::Form::COUNT; i++) { + StandardPlural::Form plural = static_cast(i); + UnicodeString simpleFormat = getWithPlural(simpleFormats, plural, status); + if (U_FAILURE(status)) { return; } + SimpleFormatter compiledFormatter(simpleFormat, 0, 1, status); + if (U_FAILURE(status)) { return; } + fModifiers[i] = SimpleModifier(compiledFormatter, field, false, {this, SIGNUM_POS_ZERO, plural}); + } +} + +void LongNameHandler::multiSimpleFormatsToModifiers(const UnicodeString *leadFormats, UnicodeString trailFormat, + Field field, UErrorCode &status) { + SimpleFormatter trailCompiled(trailFormat, 1, 1, status); + if (U_FAILURE(status)) { return; } + for (int32_t i = 0; i < StandardPlural::Form::COUNT; i++) { + StandardPlural::Form plural = static_cast(i); + UnicodeString leadFormat = getWithPlural(leadFormats, plural, status); + if (U_FAILURE(status)) { return; } + UnicodeString compoundFormat; + if (leadFormat.length() == 0) { + compoundFormat = trailFormat; + } else { + trailCompiled.format(leadFormat, compoundFormat, status); + if (U_FAILURE(status)) { return; } + } + SimpleFormatter compoundCompiled(compoundFormat, 0, 1, status); + if (U_FAILURE(status)) { return; } + fModifiers[i] = SimpleModifier(compoundCompiled, field, false, {this, SIGNUM_POS_ZERO, plural}); + } +} + +void LongNameHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + if (parent != nullptr) { + parent->processQuantity(quantity, micros, status); + } + StandardPlural::Form pluralForm = utils::getPluralSafe(micros.rounder, rules, quantity, status); + micros.modOuter = &fModifiers[pluralForm]; + micros.gender = gender; +} + +const Modifier* LongNameHandler::getModifier(Signum /*signum*/, StandardPlural::Form plural) const { + return &fModifiers[plural]; +} + +void MixedUnitLongNameHandler::forMeasureUnit(const Locale &loc, + const MeasureUnit &mixedUnit, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + MixedUnitLongNameHandler *fillIn, + UErrorCode &status) { + U_ASSERT(mixedUnit.getComplexity(status) == UMEASURE_UNIT_MIXED); + U_ASSERT(fillIn != nullptr); + if (U_FAILURE(status)) { + return; + } + + MeasureUnitImpl temp; + const MeasureUnitImpl &impl = MeasureUnitImpl::forMeasureUnit(mixedUnit, temp, status); + // Defensive, for production code: + if (impl.complexity != UMEASURE_UNIT_MIXED) { + // Should be using the normal LongNameHandler + status = U_UNSUPPORTED_ERROR; + return; + } + + fillIn->fMixedUnitCount = impl.singleUnits.length(); + fillIn->fMixedUnitData.adoptInstead(new UnicodeString[fillIn->fMixedUnitCount * ARRAY_LENGTH]); + for (int32_t i = 0; i < fillIn->fMixedUnitCount; i++) { + // Grab data for each of the components. + UnicodeString *unitData = &fillIn->fMixedUnitData[i * ARRAY_LENGTH]; + // TODO(CLDR-14582): check from the CLDR-14582 ticket whether this + // propagation of unitDisplayCase is correct: + getMeasureData(loc, impl.singleUnits[i]->build(status), width, unitDisplayCase, unitData, + status); + // TODO(ICU-21494): if we add support for gender for mixed units, we may + // need maybeCalculateGender() here. + } + + // TODO(icu-units#120): Make sure ICU doesn't output zero-valued + // high-magnitude fields + // * for mixed units count N, produce N listFormatters, one for each subset + // that might be formatted. + UListFormatterWidth listWidth = ULISTFMT_WIDTH_SHORT; + if (width == UNUM_UNIT_WIDTH_NARROW) { + listWidth = ULISTFMT_WIDTH_NARROW; + } else if (width == UNUM_UNIT_WIDTH_FULL_NAME) { + // This might be the same as SHORT in most languages: + listWidth = ULISTFMT_WIDTH_WIDE; + } + fillIn->fListFormatter.adoptInsteadAndCheckErrorCode( + ListFormatter::createInstance(loc, ULISTFMT_TYPE_UNITS, listWidth, status), status); + // TODO(ICU-21494): grab gender of each unit, calculate the gender + // associated with this list formatter, save it for later. + fillIn->rules = rules; + fillIn->parent = parent; + + // We need a localised NumberFormatter for the numbers of the bigger units + // (providing Arabic numerals, for example). + fillIn->fNumberFormatter = NumberFormatter::withLocale(loc); +} + +void MixedUnitLongNameHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + U_ASSERT(fMixedUnitCount > 1); + if (parent != nullptr) { + parent->processQuantity(quantity, micros, status); + } + micros.modOuter = getMixedUnitModifier(quantity, micros, status); +} + +const Modifier *MixedUnitLongNameHandler::getMixedUnitModifier(DecimalQuantity &quantity, + MicroProps µs, + UErrorCode &status) const { + if (micros.mixedMeasuresCount == 0) { + U_ASSERT(micros.mixedMeasuresCount > 0); // Mixed unit: we must have more than one unit value + status = U_UNSUPPORTED_ERROR; + return µs.helpers.emptyWeakModifier; + } + + // Algorithm: + // + // For the mixed-units measurement of: "3 yard, 1 foot, 2.6 inch", we should + // find "3 yard" and "1 foot" in micros.mixedMeasures. + // + // Obtain long-names with plural forms corresponding to measure values: + // * {0} yards, {0} foot, {0} inches + // + // Format the integer values appropriately and modify with the format + // strings: + // - 3 yards, 1 foot + // + // Use ListFormatter to combine, with one placeholder: + // - 3 yards, 1 foot and {0} inches + // + // Return a SimpleModifier for this pattern, letting the rest of the + // pipeline take care of the remaining inches. + + LocalArray outputMeasuresList(new UnicodeString[fMixedUnitCount], status); + if (U_FAILURE(status)) { + return µs.helpers.emptyWeakModifier; + } + + StandardPlural::Form quantityPlural = StandardPlural::Form::OTHER; + for (int32_t i = 0; i < micros.mixedMeasuresCount; i++) { + DecimalQuantity fdec; + + // If numbers are negative, only the first number needs to have its + // negative sign formatted. + int64_t number = i > 0 ? std::abs(micros.mixedMeasures[i]) : micros.mixedMeasures[i]; + + if (micros.indexOfQuantity == i) { // Insert placeholder for `quantity` + // If quantity is not the first value and quantity is negative + if (micros.indexOfQuantity > 0 && quantity.isNegative()) { + quantity.negate(); + } + + StandardPlural::Form quantityPlural = + utils::getPluralSafe(micros.rounder, rules, quantity, status); + UnicodeString quantityFormatWithPlural = + getWithPlural(&fMixedUnitData[i * ARRAY_LENGTH], quantityPlural, status); + SimpleFormatter quantityFormatter(quantityFormatWithPlural, 0, 1, status); + quantityFormatter.format(UnicodeString(u"{0}"), outputMeasuresList[i], status); + } else { + fdec.setToLong(number); + StandardPlural::Form pluralForm = utils::getStandardPlural(rules, fdec); + UnicodeString simpleFormat = + getWithPlural(&fMixedUnitData[i * ARRAY_LENGTH], pluralForm, status); + SimpleFormatter compiledFormatter(simpleFormat, 0, 1, status); + UnicodeString num; + auto appendable = UnicodeStringAppendable(num); + + fNumberFormatter.formatDecimalQuantity(fdec, status).appendTo(appendable, status); + compiledFormatter.format(num, outputMeasuresList[i], status); + } + } + + // TODO(ICU-21494): implement gender for lists of mixed units. Presumably we + // can set micros.gender to the gender associated with the list formatter in + // use below (once we have correct support for that). And then document this + // appropriately? "getMixedUnitModifier" doesn't sound like it would do + // something like this. + + // Combine list into a "premixed" pattern + UnicodeString premixedFormatPattern; + fListFormatter->format(outputMeasuresList.getAlias(), fMixedUnitCount, premixedFormatPattern, + status); + SimpleFormatter premixedCompiled(premixedFormatPattern, 0, 1, status); + if (U_FAILURE(status)) { + return µs.helpers.emptyWeakModifier; + } + + micros.helpers.mixedUnitModifier = + SimpleModifier(premixedCompiled, kUndefinedField, false, {this, SIGNUM_POS_ZERO, quantityPlural}); + return µs.helpers.mixedUnitModifier; +} + +const Modifier *MixedUnitLongNameHandler::getModifier(Signum /*signum*/, + StandardPlural::Form /*plural*/) const { + // TODO(icu-units#28): investigate this method when investigating where + // ModifierStore::getModifier() gets used. To be sure it remains + // unreachable: + UPRV_UNREACHABLE_EXIT; + return nullptr; +} + +LongNameMultiplexer *LongNameMultiplexer::forMeasureUnits(const Locale &loc, + const MaybeStackVector &units, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + UErrorCode &status) { + LocalPointer result(new LongNameMultiplexer(parent), status); + if (U_FAILURE(status)) { + return nullptr; + } + U_ASSERT(units.length() > 0); + if (result->fHandlers.resize(units.length()) == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + result->fMeasureUnits.adoptInstead(new MeasureUnit[units.length()]); + for (int32_t i = 0, length = units.length(); i < length; i++) { + const MeasureUnit &unit = *units[i]; + result->fMeasureUnits[i] = unit; + if (unit.getComplexity(status) == UMEASURE_UNIT_MIXED) { + MixedUnitLongNameHandler *mlnh = result->fMixedUnitHandlers.createAndCheckErrorCode(status); + MixedUnitLongNameHandler::forMeasureUnit(loc, unit, width, unitDisplayCase, rules, nullptr, + mlnh, status); + result->fHandlers[i] = mlnh; + } else { + LongNameHandler *lnh = result->fLongNameHandlers.createAndCheckErrorCode(status); + LongNameHandler::forMeasureUnit(loc, unit, width, unitDisplayCase, rules, nullptr, lnh, status); + result->fHandlers[i] = lnh; + } + if (U_FAILURE(status)) { + return nullptr; + } + } + return result.orphan(); +} + +void LongNameMultiplexer::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + // We call parent->processQuantity() from the Multiplexer, instead of + // letting LongNameHandler handle it: we don't know which LongNameHandler to + // call until we've called the parent! + fParent->processQuantity(quantity, micros, status); + + // Call the correct LongNameHandler based on outputUnit + for (int i = 0; i < fHandlers.getCapacity(); i++) { + if (fMeasureUnits[i] == micros.outputUnit) { + fHandlers[i]->processQuantity(quantity, micros, status); + return; + } + } + if (U_FAILURE(status)) { + return; + } + // We shouldn't receive any outputUnit for which we haven't already got a + // LongNameHandler: + status = U_INTERNAL_PROGRAM_ERROR; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_longnames.h b/intl/icu/source/i18n/number_longnames.h new file mode 100644 index 0000000000..56d8c9b24e --- /dev/null +++ b/intl/icu/source/i18n/number_longnames.h @@ -0,0 +1,272 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_LONGNAMES_H__ +#define __NUMBER_LONGNAMES_H__ + +#include "cmemory.h" +#include "unicode/listformatter.h" +#include "unicode/uversion.h" +#include "number_utils.h" +#include "number_modifiers.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +// LongNameHandler takes care of formatting currency and measurement unit names, +// as well as populating the gender of measure units. +class LongNameHandler : public MicroPropsGenerator, public ModifierStore, public UMemory { + public: + static UnicodeString getUnitDisplayName( + const Locale& loc, + const MeasureUnit& unit, + UNumberUnitWidth width, + UErrorCode& status); + + // This function does not support inflections or other newer NumberFormatter + // features: it exists to support the older not-recommended MeasureFormat. + static UnicodeString getUnitPattern( + const Locale& loc, + const MeasureUnit& unit, + UNumberUnitWidth width, + StandardPlural::Form pluralForm, + UErrorCode& status); + + static LongNameHandler* + forCurrencyLongNames(const Locale &loc, const CurrencyUnit ¤cy, const PluralRules *rules, + const MicroPropsGenerator *parent, UErrorCode &status); + + /** + * Construct a localized LongNameHandler for the specified MeasureUnit. + * + * Mixed units are not supported, use MixedUnitLongNameHandler::forMeasureUnit. + * + * This function uses a fillIn instead of returning a pointer, because we + * want to fill in instances in a MemoryPool (which cannot adopt pointers it + * didn't create itself). + * + * @param loc The desired locale. + * @param unitRef The measure unit to construct a LongNameHandler for. + * @param width Specifies the desired unit rendering. + * @param unitDisplayCase Specifies the desired grammatical case. If the + * specified case is not found, we fall back to nominative or no-case. + * @param rules Does not take ownership. + * @param parent Does not take ownership. + * @param fillIn Required. + */ + static void forMeasureUnit(const Locale &loc, + const MeasureUnit &unitRef, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + LongNameHandler *fillIn, + UErrorCode &status); + + /** + * Selects the plural-appropriate Modifier from the set of fModifiers based + * on the plural form. + */ + void + processQuantity(DecimalQuantity &quantity, MicroProps µs, UErrorCode &status) const override; + + const Modifier* getModifier(Signum signum, StandardPlural::Form plural) const override; + + private: + // A set of pre-computed modifiers, one for each plural form. + SimpleModifier fModifiers[StandardPlural::Form::COUNT]; + // Not owned + const PluralRules *rules; + // Not owned + const MicroPropsGenerator *parent; + // Grammatical gender of the formatted result. Not owned: must point at + // static or global strings. + const char *gender = ""; + + LongNameHandler(const PluralRules *rules, const MicroPropsGenerator *parent) + : rules(rules), parent(parent) { + } + + LongNameHandler() : rules(nullptr), parent(nullptr) { + } + + // Enables MemoryPool::emplaceBack(): requires access to + // the private constructors. + friend class MemoryPool; + + // Allow macrosToMicroGenerator to call the private default constructor. + friend class NumberFormatterImpl; + + // Fills in LongNameHandler fields for formatting units identified `unit`. + static void forArbitraryUnit(const Locale &loc, + const MeasureUnit &unit, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + LongNameHandler *fillIn, + UErrorCode &status); + + // Roughly corresponds to patternTimes(...) in the spec: + // https://unicode.org/reports/tr35/tr35-general.html#compound-units + // + // productUnit is an rvalue reference to indicate this function consumes it, + // leaving it in a not-useful / undefined state. + static void processPatternTimes(MeasureUnitImpl &&productUnit, + Locale loc, + const UNumberUnitWidth &width, + const char *caseVariant, + UnicodeString *outArray, + UErrorCode &status); + + // Sets fModifiers to use the patterns from `simpleFormats`. + void simpleFormatsToModifiers(const UnicodeString *simpleFormats, Field field, UErrorCode &status); + + // Sets fModifiers to a combination of `leadFormats` (one per plural form) + // and `trailFormat` appended to each. + // + // With a leadFormat of "{0}m" and a trailFormat of "{0}/s", it produces a + // pattern of "{0}m/s" by inserting each leadFormat pattern into trailFormat. + void multiSimpleFormatsToModifiers(const UnicodeString *leadFormats, UnicodeString trailFormat, + Field field, UErrorCode &status); +}; + +// Similar to LongNameHandler, but only for MIXED units. +class MixedUnitLongNameHandler : public MicroPropsGenerator, public ModifierStore, public UMemory { + public: + /** + * Construct a localized MixedUnitLongNameHandler for the specified + * MeasureUnit. It must be a MIXED unit. + * + * This function uses a fillIn instead of returning a pointer, because we + * want to fill in instances in a MemoryPool (which cannot adopt pointers it + * didn't create itself). + * + * @param loc The desired locale. + * @param mixedUnit The mixed measure unit to construct a + * MixedUnitLongNameHandler for. + * @param width Specifies the desired unit rendering. + * @param unitDisplayCase Specifies the desired grammatical case. If the + * specified case is not found, we fall back to nominative or no-case. + * @param rules Does not take ownership. + * @param parent Does not take ownership. + * @param fillIn Required. + */ + static void forMeasureUnit(const Locale &loc, + const MeasureUnit &mixedUnit, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + MixedUnitLongNameHandler *fillIn, + UErrorCode &status); + + /** + * Produces a plural-appropriate Modifier for a mixed unit: `quantity` is + * taken as the final smallest unit, while the larger unit values must be + * provided via `micros.mixedMeasures`. + */ + void processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const override; + + // Required for ModifierStore. And ModifierStore is required by + // SimpleModifier constructor's last parameter. We assert his will never get + // called though. + const Modifier *getModifier(Signum signum, StandardPlural::Form plural) const override; + + private: + // Not owned + const PluralRules *rules; + + // Not owned + const MicroPropsGenerator *parent; + + // Total number of units in the MeasureUnit this handler was configured for: + // for "foot-and-inch", this will be 2. + int32_t fMixedUnitCount = 1; + + // Stores unit data for each of the individual units. For each unit, it + // stores ARRAY_LENGTH strings, as returned by getMeasureData. (Each unit + // with index `i` has ARRAY_LENGTH strings starting at index + // `i*ARRAY_LENGTH` in this array.) + LocalArray fMixedUnitData; + + // Formats the larger units of Mixed Unit measurements. + LocalizedNumberFormatter fNumberFormatter; + + // Joins mixed units together. + LocalPointer fListFormatter; + + MixedUnitLongNameHandler(const PluralRules *rules, const MicroPropsGenerator *parent) + : rules(rules), parent(parent) { + } + + MixedUnitLongNameHandler() : rules(nullptr), parent(nullptr) { + } + + // Allow macrosToMicroGenerator to call the private default constructor. + friend class NumberFormatterImpl; + + // Enables MemoryPool::emplaceBack(): requires access to + // the private constructors. + friend class MemoryPool; + + // For a mixed unit, returns a Modifier that takes only one parameter: the + // smallest and final unit of the set. The bigger units' values and labels + // get baked into this Modifier, together with the unit label of the final + // unit. + const Modifier *getMixedUnitModifier(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const; +}; + +/** + * A MicroPropsGenerator that multiplexes between different LongNameHandlers, + * depending on the outputUnit. + * + * See processQuantity() for the input requirements. + */ +class LongNameMultiplexer : public MicroPropsGenerator, public UMemory { + public: + // Produces a multiplexer for LongNameHandlers, one for each unit in + // `units`. An individual unit might be a mixed unit. + static LongNameMultiplexer *forMeasureUnits(const Locale &loc, + const MaybeStackVector &units, + const UNumberUnitWidth &width, + const char *unitDisplayCase, + const PluralRules *rules, + const MicroPropsGenerator *parent, + UErrorCode &status); + + // The output unit must be provided via `micros.outputUnit`, it must match + // one of the units provided to the factory function. + void processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const override; + + private: + /** + * Because we only know which LongNameHandler we wish to call after calling + * earlier MicroPropsGenerators in the chain, LongNameMultiplexer keeps the + * parent link, while the LongNameHandlers are given no parents. + */ + MemoryPool fLongNameHandlers; + MemoryPool fMixedUnitHandlers; + // Unowned pointers to instances owned by MaybeStackVectors. + MaybeStackArray fHandlers; + // Each MeasureUnit corresponds to the same-index MicroPropsGenerator + // pointed to in fHandlers. + LocalArray fMeasureUnits; + + const MicroPropsGenerator *fParent; + + LongNameMultiplexer(const MicroPropsGenerator *parent) : fParent(parent) { + } +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_LONGNAMES_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_mapper.cpp b/intl/icu/source/i18n/number_mapper.cpp new file mode 100644 index 0000000000..2f398d4a93 --- /dev/null +++ b/intl/icu/source/i18n/number_mapper.cpp @@ -0,0 +1,525 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "number_mapper.h" +#include "number_patternstring.h" +#include "unicode/errorcode.h" +#include "number_utils.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +UnlocalizedNumberFormatter NumberPropertyMapper::create(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, + DecimalFormatWarehouse& warehouse, + UErrorCode& status) { + return NumberFormatter::with().macros(oldToNew(properties, symbols, warehouse, nullptr, status)); +} + +UnlocalizedNumberFormatter NumberPropertyMapper::create(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, + DecimalFormatWarehouse& warehouse, + DecimalFormatProperties& exportedProperties, + UErrorCode& status) { + return NumberFormatter::with().macros( + oldToNew( + properties, symbols, warehouse, &exportedProperties, status)); +} + +MacroProps NumberPropertyMapper::oldToNew(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, + DecimalFormatWarehouse& warehouse, + DecimalFormatProperties* exportedProperties, + UErrorCode& status) { + MacroProps macros; + Locale locale = symbols.getLocale(); + + ///////////// + // SYMBOLS // + ///////////// + + macros.symbols.setTo(symbols); + + ////////////////// + // PLURAL RULES // + ////////////////// + + if (!properties.currencyPluralInfo.fPtr.isNull()) { + macros.rules = properties.currencyPluralInfo.fPtr->getPluralRules(); + } + + ///////////// + // AFFIXES // + ///////////// + + warehouse.affixProvider.setTo(properties, status); + macros.affixProvider = &warehouse.affixProvider.get(); + + /////////// + // UNITS // + /////////// + + bool useCurrency = ( + !properties.currency.isNull() || + !properties.currencyPluralInfo.fPtr.isNull() || + !properties.currencyUsage.isNull() || + warehouse.affixProvider.get().hasCurrencySign()); + CurrencyUnit currency = resolveCurrency(properties, locale, status); + UCurrencyUsage currencyUsage = properties.currencyUsage.getOrDefault(UCURR_USAGE_STANDARD); + if (useCurrency) { + // NOTE: Slicing is OK. + macros.unit = currency; // NOLINT + } + + /////////////////////// + // ROUNDING STRATEGY // + /////////////////////// + + int32_t maxInt = properties.maximumIntegerDigits; + int32_t minInt = properties.minimumIntegerDigits; + int32_t maxFrac = properties.maximumFractionDigits; + int32_t minFrac = properties.minimumFractionDigits; + int32_t minSig = properties.minimumSignificantDigits; + int32_t maxSig = properties.maximumSignificantDigits; + double roundingIncrement = properties.roundingIncrement; + // Not assigning directly to macros.roundingMode here: we change + // roundingMode if and when we also change macros.precision. + RoundingMode roundingMode = properties.roundingMode.getOrDefault(UNUM_ROUND_HALFEVEN); + bool explicitMinMaxFrac = minFrac != -1 || maxFrac != -1; + bool explicitMinMaxSig = minSig != -1 || maxSig != -1; + // Resolve min/max frac for currencies, required for the validation logic and for when minFrac or + // maxFrac was + // set (but not both) on a currency instance. + // NOTE: Increments are handled in "Precision.constructCurrency()". + if (useCurrency && (minFrac == -1 || maxFrac == -1)) { + int32_t digits = ucurr_getDefaultFractionDigitsForUsage( + currency.getISOCurrency(), currencyUsage, &status); + if (minFrac == -1 && maxFrac == -1) { + minFrac = digits; + maxFrac = digits; + } else if (minFrac == -1) { + minFrac = std::min(maxFrac, digits); + } else /* if (maxFrac == -1) */ { + maxFrac = std::max(minFrac, digits); + } + } + // Validate min/max int/frac. + // For backwards compatibility, minimum overrides maximum if the two conflict. + if (minInt == 0 && maxFrac != 0) { + minFrac = (minFrac < 0 || (minFrac == 0 && maxInt == 0)) ? 1 : minFrac; + maxFrac = maxFrac < 0 ? -1 : maxFrac < minFrac ? minFrac : maxFrac; + minInt = 0; + maxInt = maxInt < 0 ? -1 : maxInt > kMaxIntFracSig ? -1 : maxInt; + } else { + // Force a digit before the decimal point. + minFrac = minFrac < 0 ? 0 : minFrac; + maxFrac = maxFrac < 0 ? -1 : maxFrac < minFrac ? minFrac : maxFrac; + minInt = minInt <= 0 ? 1 : minInt > kMaxIntFracSig ? 1 : minInt; + maxInt = maxInt < 0 ? -1 : maxInt < minInt ? minInt : maxInt > kMaxIntFracSig ? -1 : maxInt; + } + Precision precision; + if (!properties.currencyUsage.isNull()) { + precision = Precision::constructCurrency(currencyUsage).withCurrency(currency); + } else if (roundingIncrement != 0.0) { + if (PatternStringUtils::ignoreRoundingIncrement(roundingIncrement, maxFrac)) { + precision = Precision::constructFraction(minFrac, maxFrac); + } else { + // Convert the double increment to an integer increment + precision = Precision::increment(roundingIncrement).withMinFraction(minFrac); + } + } else if (explicitMinMaxSig) { + minSig = minSig < 1 ? 1 : minSig > kMaxIntFracSig ? kMaxIntFracSig : minSig; + maxSig = maxSig < 0 ? kMaxIntFracSig : maxSig < minSig ? minSig : maxSig > kMaxIntFracSig + ? kMaxIntFracSig : maxSig; + precision = Precision::constructSignificant(minSig, maxSig); + } else if (explicitMinMaxFrac) { + precision = Precision::constructFraction(minFrac, maxFrac); + } else if (useCurrency) { + precision = Precision::constructCurrency(currencyUsage); + } + if (!precision.isBogus()) { + macros.roundingMode = roundingMode; + macros.precision = precision; + } + + /////////////////// + // INTEGER WIDTH // + /////////////////// + + macros.integerWidth = IntegerWidth( + static_cast(minInt), + static_cast(maxInt), + properties.formatFailIfMoreThanMaxDigits); + + /////////////////////// + // GROUPING STRATEGY // + /////////////////////// + + macros.grouper = Grouper::forProperties(properties); + + ///////////// + // PADDING // + ///////////// + + if (properties.formatWidth > 0) { + macros.padder = Padder::forProperties(properties); + } + + /////////////////////////////// + // DECIMAL MARK ALWAYS SHOWN // + /////////////////////////////// + + macros.decimal = properties.decimalSeparatorAlwaysShown ? UNUM_DECIMAL_SEPARATOR_ALWAYS + : UNUM_DECIMAL_SEPARATOR_AUTO; + + /////////////////////// + // SIGN ALWAYS SHOWN // + /////////////////////// + + macros.sign = properties.signAlwaysShown ? UNUM_SIGN_ALWAYS : UNUM_SIGN_AUTO; + + ///////////////////////// + // SCIENTIFIC NOTATION // + ///////////////////////// + + if (properties.minimumExponentDigits != -1) { + // Scientific notation is required. + // This whole section feels like a hack, but it is needed for regression tests. + // The mapping from property bag to scientific notation is nontrivial due to LDML rules. + if (maxInt > 8) { + // But #13110: The maximum of 8 digits has unknown origins and is not in the spec. + // If maxInt is greater than 8, it is set to minInt, even if minInt is greater than 8. + maxInt = minInt; + macros.integerWidth = IntegerWidth::zeroFillTo(minInt).truncateAt(maxInt); + } else if (maxInt > minInt && minInt > 1) { + // Bug #13289: if maxInt > minInt > 1, then minInt should be 1. + minInt = 1; + macros.integerWidth = IntegerWidth::zeroFillTo(minInt).truncateAt(maxInt); + } + int engineering = maxInt < 0 ? -1 : maxInt; + macros.notation = ScientificNotation( + // Engineering interval: + static_cast(engineering), + // Enforce minimum integer digits (for patterns like "000.00E0"): + (engineering == minInt), + // Minimum exponent digits: + static_cast(properties.minimumExponentDigits), + // Exponent sign always shown: + properties.exponentSignAlwaysShown ? UNUM_SIGN_ALWAYS : UNUM_SIGN_AUTO); + // Scientific notation also involves overriding the rounding mode. + // TODO: Overriding here is a bit of a hack. Should this logic go earlier? + if (macros.precision.fType == Precision::PrecisionType::RND_FRACTION) { + // For the purposes of rounding, get the original min/max int/frac, since the local + // variables have been manipulated for display purposes. + int maxInt_ = properties.maximumIntegerDigits; + int minInt_ = properties.minimumIntegerDigits; + int minFrac_ = properties.minimumFractionDigits; + int maxFrac_ = properties.maximumFractionDigits; + if (minInt_ == 0 && maxFrac_ == 0) { + // Patterns like "#E0" and "##E0", which mean no rounding! + macros.precision = Precision::unlimited(); + } else if (minInt_ == 0 && minFrac_ == 0) { + // Patterns like "#.##E0" (no zeros in the mantissa), which mean round to maxFrac+1 + macros.precision = Precision::constructSignificant(1, maxFrac_ + 1); + } else { + int maxSig_ = minInt_ + maxFrac_; + // Bug #20058: if maxInt_ > minInt_ > 1, then minInt_ should be 1. + if (maxInt_ > minInt_ && minInt_ > 1) { + minInt_ = 1; + } + int minSig_ = minInt_ + minFrac_; + // To avoid regression, maxSig is not reset when minInt_ set to 1. + // TODO: Reset maxSig_ = 1 + minFrac_ to follow the spec. + macros.precision = Precision::constructSignificant(minSig_, maxSig_); + } + macros.roundingMode = roundingMode; + } + } + + ////////////////////// + // COMPACT NOTATION // + ////////////////////// + + if (!properties.compactStyle.isNull()) { + if (properties.compactStyle.getNoError() == UNumberCompactStyle::UNUM_LONG) { + macros.notation = Notation::compactLong(); + } else { + macros.notation = Notation::compactShort(); + } + } + + ///////////////// + // MULTIPLIERS // + ///////////////// + + macros.scale = scaleFromProperties(properties); + + ////////////////////// + // PROPERTY EXPORTS // + ////////////////////// + + if (exportedProperties != nullptr) { + + exportedProperties->currency = currency; + exportedProperties->roundingMode = roundingMode; + exportedProperties->minimumIntegerDigits = minInt; + exportedProperties->maximumIntegerDigits = maxInt == -1 ? INT32_MAX : maxInt; + + Precision rounding_; + if (precision.fType == Precision::PrecisionType::RND_CURRENCY) { + rounding_ = precision.withCurrency(currency, status); + } else { + rounding_ = precision; + } + int minFrac_ = minFrac; + int maxFrac_ = maxFrac; + int minSig_ = minSig; + int maxSig_ = maxSig; + double increment_ = 0.0; + if (rounding_.fType == Precision::PrecisionType::RND_FRACTION) { + minFrac_ = rounding_.fUnion.fracSig.fMinFrac; + maxFrac_ = rounding_.fUnion.fracSig.fMaxFrac; + } else if (rounding_.fType == Precision::PrecisionType::RND_INCREMENT + || rounding_.fType == Precision::PrecisionType::RND_INCREMENT_ONE + || rounding_.fType == Precision::PrecisionType::RND_INCREMENT_FIVE) { + minFrac_ = rounding_.fUnion.increment.fMinFrac; + // If incrementRounding is used, maxFrac is set equal to minFrac + maxFrac_ = rounding_.fUnion.increment.fMinFrac; + // Convert the integer increment to a double + DecimalQuantity dq; + dq.setToLong(rounding_.fUnion.increment.fIncrement); + dq.adjustMagnitude(rounding_.fUnion.increment.fIncrementMagnitude); + increment_ = dq.toDouble(); + } else if (rounding_.fType == Precision::PrecisionType::RND_SIGNIFICANT) { + minSig_ = rounding_.fUnion.fracSig.fMinSig; + maxSig_ = rounding_.fUnion.fracSig.fMaxSig; + } + + exportedProperties->minimumFractionDigits = minFrac_; + exportedProperties->maximumFractionDigits = maxFrac_; + exportedProperties->minimumSignificantDigits = minSig_; + exportedProperties->maximumSignificantDigits = maxSig_; + exportedProperties->roundingIncrement = increment_; + } + + return macros; +} + + +void PropertiesAffixPatternProvider::setTo(const DecimalFormatProperties& properties, UErrorCode& status) { + fBogus = false; + + // There are two ways to set affixes in DecimalFormat: via the pattern string (applyPattern), and via the + // explicit setters (setPositivePrefix and friends). The way to resolve the settings is as follows: + // + // 1) If the explicit setting is present for the field, use it. + // 2) Otherwise, follows UTS 35 rules based on the pattern string. + // + // Importantly, the explicit setters affect only the one field they override. If you set the positive + // prefix, that should not affect the negative prefix. + + // Convenience: Extract the properties into local variables. + // Variables are named with three chars: [p/n][p/s][o/p] + // [p/n] => p for positive, n for negative + // [p/s] => p for prefix, s for suffix + // [o/p] => o for escaped custom override string, p for pattern string + UnicodeString ppo = AffixUtils::escape(properties.positivePrefix); + UnicodeString pso = AffixUtils::escape(properties.positiveSuffix); + UnicodeString npo = AffixUtils::escape(properties.negativePrefix); + UnicodeString nso = AffixUtils::escape(properties.negativeSuffix); + const UnicodeString& ppp = properties.positivePrefixPattern; + const UnicodeString& psp = properties.positiveSuffixPattern; + const UnicodeString& npp = properties.negativePrefixPattern; + const UnicodeString& nsp = properties.negativeSuffixPattern; + + if (!properties.positivePrefix.isBogus()) { + posPrefix = ppo; + } else if (!ppp.isBogus()) { + posPrefix = ppp; + } else { + // UTS 35: Default positive prefix is empty string. + posPrefix = u""; + } + + if (!properties.positiveSuffix.isBogus()) { + posSuffix = pso; + } else if (!psp.isBogus()) { + posSuffix = psp; + } else { + // UTS 35: Default positive suffix is empty string. + posSuffix = u""; + } + + if (!properties.negativePrefix.isBogus()) { + negPrefix = npo; + } else if (!npp.isBogus()) { + negPrefix = npp; + } else { + // UTS 35: Default negative prefix is "-" with positive prefix. + // Important: We prepend the "-" to the pattern, not the override! + negPrefix = ppp.isBogus() ? u"-" : u"-" + ppp; + } + + if (!properties.negativeSuffix.isBogus()) { + negSuffix = nso; + } else if (!nsp.isBogus()) { + negSuffix = nsp; + } else { + // UTS 35: Default negative prefix is the positive prefix. + negSuffix = psp.isBogus() ? u"" : psp; + } + + // For declaring if this is a currency pattern, we need to look at the + // original pattern, not at any user-specified overrides. + isCurrencyPattern = ( + AffixUtils::hasCurrencySymbols(ppp, status) || + AffixUtils::hasCurrencySymbols(psp, status) || + AffixUtils::hasCurrencySymbols(npp, status) || + AffixUtils::hasCurrencySymbols(nsp, status) || + properties.currencyAsDecimal); + + fCurrencyAsDecimal = properties.currencyAsDecimal; +} + +char16_t PropertiesAffixPatternProvider::charAt(int flags, int i) const { + return getStringInternal(flags).charAt(i); +} + +int PropertiesAffixPatternProvider::length(int flags) const { + return getStringInternal(flags).length(); +} + +UnicodeString PropertiesAffixPatternProvider::getString(int32_t flags) const { + return getStringInternal(flags); +} + +const UnicodeString& PropertiesAffixPatternProvider::getStringInternal(int32_t flags) const { + bool prefix = (flags & AFFIX_PREFIX) != 0; + bool negative = (flags & AFFIX_NEGATIVE_SUBPATTERN) != 0; + if (prefix && negative) { + return negPrefix; + } else if (prefix) { + return posPrefix; + } else if (negative) { + return negSuffix; + } else { + return posSuffix; + } +} + +bool PropertiesAffixPatternProvider::positiveHasPlusSign() const { + // TODO: Change the internal APIs to propagate out the error? + ErrorCode localStatus; + return AffixUtils::containsType(posPrefix, TYPE_PLUS_SIGN, localStatus) || + AffixUtils::containsType(posSuffix, TYPE_PLUS_SIGN, localStatus); +} + +bool PropertiesAffixPatternProvider::hasNegativeSubpattern() const { + return ( + (negSuffix != posSuffix) || + negPrefix.tempSubString(1) != posPrefix || + negPrefix.charAt(0) != u'-' + ); +} + +bool PropertiesAffixPatternProvider::negativeHasMinusSign() const { + ErrorCode localStatus; + return AffixUtils::containsType(negPrefix, TYPE_MINUS_SIGN, localStatus) || + AffixUtils::containsType(negSuffix, TYPE_MINUS_SIGN, localStatus); +} + +bool PropertiesAffixPatternProvider::hasCurrencySign() const { + return isCurrencyPattern; +} + +bool PropertiesAffixPatternProvider::containsSymbolType(AffixPatternType type, UErrorCode& status) const { + return AffixUtils::containsType(posPrefix, type, status) || + AffixUtils::containsType(posSuffix, type, status) || + AffixUtils::containsType(negPrefix, type, status) || + AffixUtils::containsType(negSuffix, type, status); +} + +bool PropertiesAffixPatternProvider::hasBody() const { + return true; +} + +bool PropertiesAffixPatternProvider::currencyAsDecimal() const { + return fCurrencyAsDecimal; +} + + +void CurrencyPluralInfoAffixProvider::setTo(const CurrencyPluralInfo& cpi, + const DecimalFormatProperties& properties, + UErrorCode& status) { + // We need to use a PropertiesAffixPatternProvider, not the simpler version ParsedPatternInfo, + // because user-specified affix overrides still need to work. + fBogus = false; + DecimalFormatProperties pluralProperties(properties); + for (int32_t plural = 0; plural < StandardPlural::COUNT; plural++) { + const char* keyword = StandardPlural::getKeyword(static_cast(plural)); + UnicodeString patternString; + patternString = cpi.getCurrencyPluralPattern(keyword, patternString); + PatternParser::parseToExistingProperties( + patternString, + pluralProperties, + IGNORE_ROUNDING_NEVER, + status); + affixesByPlural[plural].setTo(pluralProperties, status); + } +} + +char16_t CurrencyPluralInfoAffixProvider::charAt(int32_t flags, int32_t i) const { + int32_t pluralOrdinal = (flags & AFFIX_PLURAL_MASK); + return affixesByPlural[pluralOrdinal].charAt(flags, i); +} + +int32_t CurrencyPluralInfoAffixProvider::length(int32_t flags) const { + int32_t pluralOrdinal = (flags & AFFIX_PLURAL_MASK); + return affixesByPlural[pluralOrdinal].length(flags); +} + +UnicodeString CurrencyPluralInfoAffixProvider::getString(int32_t flags) const { + int32_t pluralOrdinal = (flags & AFFIX_PLURAL_MASK); + return affixesByPlural[pluralOrdinal].getString(flags); +} + +bool CurrencyPluralInfoAffixProvider::positiveHasPlusSign() const { + return affixesByPlural[StandardPlural::OTHER].positiveHasPlusSign(); +} + +bool CurrencyPluralInfoAffixProvider::hasNegativeSubpattern() const { + return affixesByPlural[StandardPlural::OTHER].hasNegativeSubpattern(); +} + +bool CurrencyPluralInfoAffixProvider::negativeHasMinusSign() const { + return affixesByPlural[StandardPlural::OTHER].negativeHasMinusSign(); +} + +bool CurrencyPluralInfoAffixProvider::hasCurrencySign() const { + return affixesByPlural[StandardPlural::OTHER].hasCurrencySign(); +} + +bool CurrencyPluralInfoAffixProvider::containsSymbolType(AffixPatternType type, UErrorCode& status) const { + return affixesByPlural[StandardPlural::OTHER].containsSymbolType(type, status); +} + +bool CurrencyPluralInfoAffixProvider::hasBody() const { + return affixesByPlural[StandardPlural::OTHER].hasBody(); +} + +bool CurrencyPluralInfoAffixProvider::currencyAsDecimal() const { + return affixesByPlural[StandardPlural::OTHER].currencyAsDecimal(); +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_mapper.h b/intl/icu/source/i18n/number_mapper.h new file mode 100644 index 0000000000..5c5d28734b --- /dev/null +++ b/intl/icu/source/i18n/number_mapper.h @@ -0,0 +1,277 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_MAPPER_H__ +#define __NUMBER_MAPPER_H__ + +#include "number_types.h" +#include "unicode/currpinf.h" +#include "standardplural.h" +#include "number_patternstring.h" +#include "number_currencysymbols.h" +#include "numparse_impl.h" + +#ifndef __wasi__ +#include +#endif + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { + + +class AutoAffixPatternProvider; +class CurrencyPluralInfoAffixProvider; + + +class PropertiesAffixPatternProvider : public AffixPatternProvider, public UMemory { + public: + bool isBogus() const { + return fBogus; + } + + void setToBogus() { + fBogus = true; + } + + void setTo(const DecimalFormatProperties& properties, UErrorCode& status); + + // AffixPatternProvider Methods: + + char16_t charAt(int32_t flags, int32_t i) const override; + + int32_t length(int32_t flags) const override; + + UnicodeString getString(int32_t flags) const override; + + bool hasCurrencySign() const override; + + bool positiveHasPlusSign() const override; + + bool hasNegativeSubpattern() const override; + + bool negativeHasMinusSign() const override; + + bool containsSymbolType(AffixPatternType, UErrorCode&) const override; + + bool hasBody() const override; + + bool currencyAsDecimal() const override; + + private: + UnicodeString posPrefix; + UnicodeString posSuffix; + UnicodeString negPrefix; + UnicodeString negSuffix; + bool isCurrencyPattern; + bool fCurrencyAsDecimal; + + PropertiesAffixPatternProvider() = default; // puts instance in valid but undefined state + + const UnicodeString& getStringInternal(int32_t flags) const; + + bool fBogus{true}; + + friend class AutoAffixPatternProvider; + friend class CurrencyPluralInfoAffixProvider; +}; + + +class CurrencyPluralInfoAffixProvider : public AffixPatternProvider, public UMemory { + public: + bool isBogus() const { + return fBogus; + } + + void setToBogus() { + fBogus = true; + } + + void setTo(const CurrencyPluralInfo& cpi, const DecimalFormatProperties& properties, + UErrorCode& status); + + // AffixPatternProvider Methods: + + char16_t charAt(int32_t flags, int32_t i) const override; + + int32_t length(int32_t flags) const override; + + UnicodeString getString(int32_t flags) const override; + + bool hasCurrencySign() const override; + + bool positiveHasPlusSign() const override; + + bool hasNegativeSubpattern() const override; + + bool negativeHasMinusSign() const override; + + bool containsSymbolType(AffixPatternType, UErrorCode&) const override; + + bool hasBody() const override; + + bool currencyAsDecimal() const override; + + private: + PropertiesAffixPatternProvider affixesByPlural[StandardPlural::COUNT]; + + CurrencyPluralInfoAffixProvider() = default; + + bool fBogus{true}; + + friend class AutoAffixPatternProvider; +}; + + +class AutoAffixPatternProvider { + public: + inline AutoAffixPatternProvider() = default; + + inline AutoAffixPatternProvider(const DecimalFormatProperties& properties, UErrorCode& status) { + setTo(properties, status); + } + + inline void setTo(const DecimalFormatProperties& properties, UErrorCode& status) { + if (properties.currencyPluralInfo.fPtr.isNull()) { + propertiesAPP.setTo(properties, status); + currencyPluralInfoAPP.setToBogus(); + } else { + propertiesAPP.setToBogus(); + currencyPluralInfoAPP.setTo(*properties.currencyPluralInfo.fPtr, properties, status); + } + } + + inline void setTo(const AffixPatternProvider* provider, UErrorCode& status) { + if (auto ptr = dynamic_cast(provider)) { + propertiesAPP = *ptr; + } else if (auto ptr = dynamic_cast(provider)) { + currencyPluralInfoAPP = *ptr; + } else { + status = U_INTERNAL_PROGRAM_ERROR; + } + } + + inline const AffixPatternProvider& get() const { + if (!currencyPluralInfoAPP.isBogus()) { + return currencyPluralInfoAPP; + } else { + return propertiesAPP; + } + } + + private: + PropertiesAffixPatternProvider propertiesAPP; + CurrencyPluralInfoAffixProvider currencyPluralInfoAPP; +}; + + +/** + * A struct for ownership of a few objects needed for formatting. + */ +struct DecimalFormatWarehouse : public UMemory { + AutoAffixPatternProvider affixProvider; + LocalPointer rules; +}; + + +/** +* Internal fields for DecimalFormat. +* TODO: Make some of these fields by value instead of by LocalPointer? +*/ +struct DecimalFormatFields : public UMemory { + + DecimalFormatFields() {} + + DecimalFormatFields(const DecimalFormatProperties& propsToCopy) + : properties(propsToCopy) {} + + /** The property bag corresponding to user-specified settings and settings from the pattern string. */ + DecimalFormatProperties properties; + + /** The symbols for the current locale. */ + LocalPointer symbols; + + /** + * The pre-computed formatter object. Setters cause this to be re-computed atomically. The {@link + * #format} method uses the formatter directly without needing to synchronize. + */ + LocalizedNumberFormatter formatter; + + /** The lazy-computed parser for .parse() */ +#ifndef __wasi__ + std::atomic<::icu::numparse::impl::NumberParserImpl*> atomicParser = {}; +#else + ::icu::numparse::impl::NumberParserImpl* atomicParser = nullptr; +#endif + + /** The lazy-computed parser for .parseCurrency() */ +#ifndef __wasi__ + std::atomic<::icu::numparse::impl::NumberParserImpl*> atomicCurrencyParser = {}; +#else + ::icu::numparse::impl::NumberParserImpl* atomicCurrencyParser = {}; +#endif + + /** Small object ownership warehouse for the formatter and parser */ + DecimalFormatWarehouse warehouse; + + /** The effective properties as exported from the formatter object. Used by some getters. */ + DecimalFormatProperties exportedProperties; + + // Data for fastpath + bool canUseFastFormat = false; + struct FastFormatData { + char16_t cpZero; + char16_t cpGroupingSeparator; + char16_t cpMinusSign; + int8_t minInt; + int8_t maxInt; + } fastData; +}; + + +/** + * Utilities for converting between a DecimalFormatProperties and a MacroProps. + */ +class NumberPropertyMapper { + public: + /** Convenience method to create a NumberFormatter directly from Properties. */ + static UnlocalizedNumberFormatter create(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, + DecimalFormatWarehouse& warehouse, UErrorCode& status); + + /** Convenience method to create a NumberFormatter directly from Properties. */ + static UnlocalizedNumberFormatter create(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, + DecimalFormatWarehouse& warehouse, + DecimalFormatProperties& exportedProperties, + UErrorCode& status); + + /** + * Creates a new {@link MacroProps} object based on the content of a {@link DecimalFormatProperties} + * object. In other words, maps Properties to MacroProps. This function is used by the + * JDK-compatibility API to call into the ICU 60 fluent number formatting pipeline. + * + * @param properties + * The property bag to be mapped. + * @param symbols + * The symbols associated with the property bag. + * @param exportedProperties + * A property bag in which to store validated properties. Used by some DecimalFormat + * getters. + * @return A new MacroProps containing all of the information in the Properties. + */ + static MacroProps oldToNew(const DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, DecimalFormatWarehouse& warehouse, + DecimalFormatProperties* exportedProperties, UErrorCode& status); +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMBER_MAPPER_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_microprops.h b/intl/icu/source/i18n/number_microprops.h new file mode 100644 index 0000000000..18addaae08 --- /dev/null +++ b/intl/icu/source/i18n/number_microprops.h @@ -0,0 +1,197 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_MICROPROPS_H__ +#define __NUMBER_MICROPROPS_H__ + +// TODO: minimize includes +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "number_scientific.h" +#include "number_patternstring.h" +#include "number_modifiers.h" +#include "number_multiplier.h" +#include "number_roundingutils.h" +#include "decNumber.h" +#include "charstr.h" +#include "util.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +/** + * A copyable container for the integer values of mixed unit measurements. + * + * If memory allocation fails during copying, no values are copied and status is + * set to U_MEMORY_ALLOCATION_ERROR. + */ +class IntMeasures : public MaybeStackArray { + public: + /** + * Default constructor initializes with internal T[stackCapacity] buffer. + * + * Stack Capacity: most mixed units are expected to consist of two or three + * subunits, so one or two integer measures should be enough. + */ + IntMeasures() : MaybeStackArray() {} + + /** + * Copy constructor. + * + * If memory allocation fails during copying, no values are copied and + * status is set to U_MEMORY_ALLOCATION_ERROR. + */ + IntMeasures(const IntMeasures &other) : MaybeStackArray() { + this->operator=(other); + } + + // Assignment operator + IntMeasures &operator=(const IntMeasures &rhs) { + if (this == &rhs) { + return *this; + } + copyFrom(rhs, status); + return *this; + } + + /** Move constructor */ + IntMeasures(IntMeasures &&src) = default; + + /** Move assignment */ + IntMeasures &operator=(IntMeasures &&src) = default; + + UErrorCode status = U_ZERO_ERROR; +}; + +struct SimpleMicroProps : public UMemory { + Grouper grouping; + bool useCurrency = false; + UNumberDecimalSeparatorDisplay decimal = UNUM_DECIMAL_SEPARATOR_AUTO; + + // Currency symbol to be used as the decimal separator + UnicodeString currencyAsDecimal = ICU_Utility::makeBogusString(); + + // Note: This struct has no direct ownership of the following pointer. + const DecimalFormatSymbols* symbols = nullptr; +}; + +/** + * MicroProps is the first MicroPropsGenerator that should be should be called, + * producing an initialized MicroProps instance that will be passed on and + * modified throughout the rest of the chain of MicroPropsGenerator instances. + */ +struct MicroProps : public MicroPropsGenerator { + SimpleMicroProps simple; + + // NOTE: All of these fields are properly initialized in NumberFormatterImpl. + RoundingImpl rounder; + Padder padding; + IntegerWidth integerWidth; + UNumberSignDisplay sign; + char nsName[9]; + + // No ownership: must point at a string which will outlive MicroProps + // instances, e.g. a string with static storage duration, or just a string + // that will never be deallocated or modified. + const char *gender; + + // Note: This struct has no direct ownership of the following pointers. + + // Pointers to Modifiers provided by the number formatting pipeline (when + // the value is known): + + // A Modifier provided by LongNameHandler, used for currency long names and + // units. If there is no LongNameHandler needed, this should be an + // EmptyModifier. (This is typically the third modifier applied.) + const Modifier* modOuter; + // A Modifier for short currencies and compact notation. (This is typically + // the second modifier applied.) + const Modifier* modMiddle = nullptr; + // A Modifier provided by ScientificHandler, used for scientific notation. + // This is typically the first modifier applied. + const Modifier* modInner; + + // The following "helper" fields may optionally be used during the MicroPropsGenerator. + // They live here to retain memory. + struct { + // The ScientificModifier for which ScientificHandler is responsible. + // ScientificHandler::processQuantity() modifies this Modifier. + ScientificModifier scientificModifier; + // EmptyModifier used for modOuter + EmptyModifier emptyWeakModifier{false}; + // EmptyModifier used for modInner + EmptyModifier emptyStrongModifier{true}; + MultiplierFormatHandler multiplier; + // A Modifier used for Mixed Units. When formatting mixed units, + // LongNameHandler assigns this Modifier. + SimpleModifier mixedUnitModifier; + } helpers; + + // The MeasureUnit with which the output is represented. May also have + // UMEASURE_UNIT_MIXED complexity, in which case mixedMeasures comes into + // play. + MeasureUnit outputUnit; + + // Contains all the values of each unit in mixed units. For quantity (which is the floating value of + // the smallest unit in the mixed unit), the value stores in `quantity`. + // NOTE: the value of quantity in `mixedMeasures` will be left unset. + IntMeasures mixedMeasures; + + // Points to quantity position, -1 if the position is not set yet. + int32_t indexOfQuantity = -1; + + // Number of mixedMeasures that have been populated + int32_t mixedMeasuresCount = 0; + + MicroProps() = default; + + MicroProps(const MicroProps& other) = default; + + MicroProps& operator=(const MicroProps& other) = default; + + /** + * As MicroProps is the "base instance", this implementation of + * MicroPropsGenerator::processQuantity() just ensures that the output + * `micros` is correctly initialized. + * + * For the "safe" invocation of this function, micros must not be *this, + * such that a copy of the base instance is made. For the "unsafe" path, + * this function can be used only once, because the base MicroProps instance + * will be modified and thus not be available for re-use. + * + * @param quantity The quantity for consideration and optional mutation. + * @param micros The MicroProps instance to populate. If this parameter is + * not already `*this`, it will be overwritten with a copy of `*this`. + */ + void processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const override { + (void) quantity; + (void) status; + if (this == µs) { + // Unsafe path: no need to perform a copy. + U_ASSERT(!exhausted); + micros.exhausted = true; + U_ASSERT(exhausted); + } else { + // Safe path: copy self into the output micros. + U_ASSERT(!exhausted); + micros = *this; + } + } + + private: + // Internal fields: + bool exhausted = false; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif // __NUMBER_MICROPROPS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_modifiers.cpp b/intl/icu/source/i18n/number_modifiers.cpp new file mode 100644 index 0000000000..0f6fdafb09 --- /dev/null +++ b/intl/icu/source/i18n/number_modifiers.cpp @@ -0,0 +1,494 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "umutex.h" +#include "ucln_cmn.h" +#include "ucln_in.h" +#include "number_modifiers.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +// TODO: This is copied from simpleformatter.cpp +const int32_t ARG_NUM_LIMIT = 0x100; + +// These are the default currency spacing UnicodeSets in CLDR. +// Pre-compute them for performance. +// The Java unit test testCurrencySpacingPatternStability() will start failing if these change in CLDR. +icu::UInitOnce gDefaultCurrencySpacingInitOnce {}; + +UnicodeSet *UNISET_DIGIT = nullptr; +UnicodeSet *UNISET_NOTSZ = nullptr; + +UBool U_CALLCONV cleanupDefaultCurrencySpacing() { + delete UNISET_DIGIT; + UNISET_DIGIT = nullptr; + delete UNISET_NOTSZ; + UNISET_NOTSZ = nullptr; + gDefaultCurrencySpacingInitOnce.reset(); + return true; +} + +void U_CALLCONV initDefaultCurrencySpacing(UErrorCode &status) { + ucln_i18n_registerCleanup(UCLN_I18N_CURRENCY_SPACING, cleanupDefaultCurrencySpacing); + UNISET_DIGIT = new UnicodeSet(UnicodeString(u"[:digit:]"), status); + UNISET_NOTSZ = new UnicodeSet(UnicodeString(u"[[:^S:]&[:^Z:]]"), status); + if (UNISET_DIGIT == nullptr || UNISET_NOTSZ == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + UNISET_DIGIT->freeze(); + UNISET_NOTSZ->freeze(); +} + +} // namespace + + +Modifier::~Modifier() = default; + +Modifier::Parameters::Parameters() + : obj(nullptr) {} + +Modifier::Parameters::Parameters( + const ModifierStore* _obj, Signum _signum, StandardPlural::Form _plural) + : obj(_obj), signum(_signum), plural(_plural) {} + +ModifierStore::~ModifierStore() = default; + +AdoptingSignumModifierStore::~AdoptingSignumModifierStore() { + for (const Modifier *mod : mods) { + delete mod; + } +} + +AdoptingSignumModifierStore& +AdoptingSignumModifierStore::operator=(AdoptingSignumModifierStore&& other) noexcept { + for (size_t i=0; imods[i] = other.mods[i]; + other.mods[i] = nullptr; + } + return *this; +} + + +int32_t ConstantAffixModifier::apply(FormattedStringBuilder &output, int leftIndex, int rightIndex, + UErrorCode &status) const { + // Insert the suffix first since inserting the prefix will change the rightIndex + int length = output.insert(rightIndex, fSuffix, fField, status); + length += output.insert(leftIndex, fPrefix, fField, status); + return length; +} + +int32_t ConstantAffixModifier::getPrefixLength() const { + return fPrefix.length(); +} + +int32_t ConstantAffixModifier::getCodePointCount() const { + return fPrefix.countChar32() + fSuffix.countChar32(); +} + +bool ConstantAffixModifier::isStrong() const { + return fStrong; +} + +bool ConstantAffixModifier::containsField(Field field) const { + (void)field; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +void ConstantAffixModifier::getParameters(Parameters& output) const { + (void)output; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +bool ConstantAffixModifier::semanticallyEquivalent(const Modifier& other) const { + auto* _other = dynamic_cast(&other); + if (_other == nullptr) { + return false; + } + return fPrefix == _other->fPrefix + && fSuffix == _other->fSuffix + && fField == _other->fField + && fStrong == _other->fStrong; +} + + +SimpleModifier::SimpleModifier(const SimpleFormatter &simpleFormatter, Field field, bool strong) + : SimpleModifier(simpleFormatter, field, strong, {}) {} + +SimpleModifier::SimpleModifier(const SimpleFormatter &simpleFormatter, Field field, bool strong, + const Modifier::Parameters parameters) + : fCompiledPattern(simpleFormatter.compiledPattern), fField(field), fStrong(strong), + fParameters(parameters) { + int32_t argLimit = SimpleFormatter::getArgumentLimit( + fCompiledPattern.getBuffer(), fCompiledPattern.length()); + if (argLimit == 0) { + // No arguments in compiled pattern + fPrefixLength = fCompiledPattern.charAt(1) - ARG_NUM_LIMIT; + U_ASSERT(2 + fPrefixLength == fCompiledPattern.length()); + // Set suffixOffset = -1 to indicate no arguments in compiled pattern. + fSuffixOffset = -1; + fSuffixLength = 0; + } else { + U_ASSERT(argLimit == 1); + if (fCompiledPattern.charAt(1) != 0) { + // Found prefix + fPrefixLength = fCompiledPattern.charAt(1) - ARG_NUM_LIMIT; + fSuffixOffset = 3 + fPrefixLength; + } else { + // No prefix + fPrefixLength = 0; + fSuffixOffset = 2; + } + if (3 + fPrefixLength < fCompiledPattern.length()) { + // Found suffix + fSuffixLength = fCompiledPattern.charAt(fSuffixOffset) - ARG_NUM_LIMIT; + } else { + // No suffix + fSuffixLength = 0; + } + } +} + +SimpleModifier::SimpleModifier() + : fField(kUndefinedField), fStrong(false), fPrefixLength(0), fSuffixLength(0) { +} + +int32_t SimpleModifier::apply(FormattedStringBuilder &output, int leftIndex, int rightIndex, + UErrorCode &status) const { + return formatAsPrefixSuffix(output, leftIndex, rightIndex, status); +} + +int32_t SimpleModifier::getPrefixLength() const { + return fPrefixLength; +} + +int32_t SimpleModifier::getCodePointCount() const { + int32_t count = 0; + if (fPrefixLength > 0) { + count += fCompiledPattern.countChar32(2, fPrefixLength); + } + if (fSuffixLength > 0) { + count += fCompiledPattern.countChar32(1 + fSuffixOffset, fSuffixLength); + } + return count; +} + +bool SimpleModifier::isStrong() const { + return fStrong; +} + +bool SimpleModifier::containsField(Field field) const { + (void)field; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +void SimpleModifier::getParameters(Parameters& output) const { + output = fParameters; +} + +bool SimpleModifier::semanticallyEquivalent(const Modifier& other) const { + auto* _other = dynamic_cast(&other); + if (_other == nullptr) { + return false; + } + if (fParameters.obj != nullptr) { + return fParameters.obj == _other->fParameters.obj; + } + return fCompiledPattern == _other->fCompiledPattern + && fField == _other->fField + && fStrong == _other->fStrong; +} + + +int32_t +SimpleModifier::formatAsPrefixSuffix(FormattedStringBuilder &result, int32_t startIndex, int32_t endIndex, + UErrorCode &status) const { + if (fSuffixOffset == -1 && fPrefixLength + fSuffixLength > 0) { + // There is no argument for the inner number; overwrite the entire segment with our string. + return result.splice(startIndex, endIndex, fCompiledPattern, 2, 2 + fPrefixLength, fField, status); + } else { + if (fPrefixLength > 0) { + result.insert(startIndex, fCompiledPattern, 2, 2 + fPrefixLength, fField, status); + } + if (fSuffixLength > 0) { + result.insert( + endIndex + fPrefixLength, + fCompiledPattern, + 1 + fSuffixOffset, + 1 + fSuffixOffset + fSuffixLength, + fField, + status); + } + return fPrefixLength + fSuffixLength; + } +} + + +int32_t +SimpleModifier::formatTwoArgPattern(const SimpleFormatter& compiled, FormattedStringBuilder& result, + int32_t index, int32_t* outPrefixLength, int32_t* outSuffixLength, + Field field, UErrorCode& status) { + const UnicodeString& compiledPattern = compiled.compiledPattern; + int32_t argLimit = SimpleFormatter::getArgumentLimit( + compiledPattern.getBuffer(), compiledPattern.length()); + if (argLimit != 2) { + status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + int32_t offset = 1; // offset into compiledPattern + int32_t length = 0; // chars added to result + + int32_t prefixLength = compiledPattern.charAt(offset); + offset++; + if (prefixLength < ARG_NUM_LIMIT) { + // No prefix + prefixLength = 0; + } else { + prefixLength -= ARG_NUM_LIMIT; + result.insert(index + length, compiledPattern, offset, offset + prefixLength, field, status); + offset += prefixLength; + length += prefixLength; + offset++; + } + + int32_t infixLength = compiledPattern.charAt(offset); + offset++; + if (infixLength < ARG_NUM_LIMIT) { + // No infix + infixLength = 0; + } else { + infixLength -= ARG_NUM_LIMIT; + result.insert(index + length, compiledPattern, offset, offset + infixLength, field, status); + offset += infixLength; + length += infixLength; + offset++; + } + + int32_t suffixLength; + if (offset == compiledPattern.length()) { + // No suffix + suffixLength = 0; + } else { + suffixLength = compiledPattern.charAt(offset) - ARG_NUM_LIMIT; + offset++; + result.insert(index + length, compiledPattern, offset, offset + suffixLength, field, status); + length += suffixLength; + } + + *outPrefixLength = prefixLength; + *outSuffixLength = suffixLength; + + return length; +} + + +int32_t ConstantMultiFieldModifier::apply(FormattedStringBuilder &output, int leftIndex, int rightIndex, + UErrorCode &status) const { + int32_t length = output.insert(leftIndex, fPrefix, status); + if (fOverwrite) { + length += output.splice( + leftIndex + length, + rightIndex + length, + UnicodeString(), 0, 0, + kUndefinedField, status); + } + length += output.insert(rightIndex + length, fSuffix, status); + return length; +} + +int32_t ConstantMultiFieldModifier::getPrefixLength() const { + return fPrefix.length(); +} + +int32_t ConstantMultiFieldModifier::getCodePointCount() const { + return fPrefix.codePointCount() + fSuffix.codePointCount(); +} + +bool ConstantMultiFieldModifier::isStrong() const { + return fStrong; +} + +bool ConstantMultiFieldModifier::containsField(Field field) const { + return fPrefix.containsField(field) || fSuffix.containsField(field); +} + +void ConstantMultiFieldModifier::getParameters(Parameters& output) const { + output = fParameters; +} + +bool ConstantMultiFieldModifier::semanticallyEquivalent(const Modifier& other) const { + auto* _other = dynamic_cast(&other); + if (_other == nullptr) { + return false; + } + if (fParameters.obj != nullptr) { + return fParameters.obj == _other->fParameters.obj; + } + return fPrefix.contentEquals(_other->fPrefix) + && fSuffix.contentEquals(_other->fSuffix) + && fOverwrite == _other->fOverwrite + && fStrong == _other->fStrong; +} + + +CurrencySpacingEnabledModifier::CurrencySpacingEnabledModifier(const FormattedStringBuilder &prefix, + const FormattedStringBuilder &suffix, + bool overwrite, + bool strong, + const DecimalFormatSymbols &symbols, + UErrorCode &status) + : ConstantMultiFieldModifier(prefix, suffix, overwrite, strong) { + // Check for currency spacing. Do not build the UnicodeSets unless there is + // a currency code point at a boundary. + if (prefix.length() > 0 && prefix.fieldAt(prefix.length() - 1) == Field(UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD)) { + int prefixCp = prefix.getLastCodePoint(); + UnicodeSet prefixUnicodeSet = getUnicodeSet(symbols, IN_CURRENCY, PREFIX, status); + if (prefixUnicodeSet.contains(prefixCp)) { + fAfterPrefixUnicodeSet = getUnicodeSet(symbols, IN_NUMBER, PREFIX, status); + fAfterPrefixUnicodeSet.freeze(); + fAfterPrefixInsert = getInsertString(symbols, PREFIX, status); + } else { + fAfterPrefixUnicodeSet.setToBogus(); + fAfterPrefixInsert.setToBogus(); + } + } else { + fAfterPrefixUnicodeSet.setToBogus(); + fAfterPrefixInsert.setToBogus(); + } + if (suffix.length() > 0 && suffix.fieldAt(0) == Field(UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD)) { + int suffixCp = suffix.getFirstCodePoint(); + UnicodeSet suffixUnicodeSet = getUnicodeSet(symbols, IN_CURRENCY, SUFFIX, status); + if (suffixUnicodeSet.contains(suffixCp)) { + fBeforeSuffixUnicodeSet = getUnicodeSet(symbols, IN_NUMBER, SUFFIX, status); + fBeforeSuffixUnicodeSet.freeze(); + fBeforeSuffixInsert = getInsertString(symbols, SUFFIX, status); + } else { + fBeforeSuffixUnicodeSet.setToBogus(); + fBeforeSuffixInsert.setToBogus(); + } + } else { + fBeforeSuffixUnicodeSet.setToBogus(); + fBeforeSuffixInsert.setToBogus(); + } +} + +int32_t CurrencySpacingEnabledModifier::apply(FormattedStringBuilder &output, int leftIndex, int rightIndex, + UErrorCode &status) const { + // Currency spacing logic + int length = 0; + if (rightIndex - leftIndex > 0 && !fAfterPrefixUnicodeSet.isBogus() && + fAfterPrefixUnicodeSet.contains(output.codePointAt(leftIndex))) { + // TODO: Should we use the CURRENCY field here? + length += output.insert( + leftIndex, + fAfterPrefixInsert, + kUndefinedField, + status); + } + if (rightIndex - leftIndex > 0 && !fBeforeSuffixUnicodeSet.isBogus() && + fBeforeSuffixUnicodeSet.contains(output.codePointBefore(rightIndex))) { + // TODO: Should we use the CURRENCY field here? + length += output.insert( + rightIndex + length, + fBeforeSuffixInsert, + kUndefinedField, + status); + } + + // Call super for the remaining logic + length += ConstantMultiFieldModifier::apply(output, leftIndex, rightIndex + length, status); + return length; +} + +int32_t +CurrencySpacingEnabledModifier::applyCurrencySpacing(FormattedStringBuilder &output, int32_t prefixStart, + int32_t prefixLen, int32_t suffixStart, + int32_t suffixLen, + const DecimalFormatSymbols &symbols, + UErrorCode &status) { + int length = 0; + bool hasPrefix = (prefixLen > 0); + bool hasSuffix = (suffixLen > 0); + bool hasNumber = (suffixStart - prefixStart - prefixLen > 0); // could be empty string + if (hasPrefix && hasNumber) { + length += applyCurrencySpacingAffix(output, prefixStart + prefixLen, PREFIX, symbols, status); + } + if (hasSuffix && hasNumber) { + length += applyCurrencySpacingAffix(output, suffixStart + length, SUFFIX, symbols, status); + } + return length; +} + +int32_t +CurrencySpacingEnabledModifier::applyCurrencySpacingAffix(FormattedStringBuilder &output, int32_t index, + EAffix affix, + const DecimalFormatSymbols &symbols, + UErrorCode &status) { + // NOTE: For prefix, output.fieldAt(index-1) gets the last field type in the prefix. + // This works even if the last code point in the prefix is 2 code units because the + // field value gets populated to both indices in the field array. + Field affixField = (affix == PREFIX) ? output.fieldAt(index - 1) : output.fieldAt(index); + if (affixField != Field(UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD)) { + return 0; + } + int affixCp = (affix == PREFIX) ? output.codePointBefore(index) : output.codePointAt(index); + UnicodeSet affixUniset = getUnicodeSet(symbols, IN_CURRENCY, affix, status); + if (!affixUniset.contains(affixCp)) { + return 0; + } + int numberCp = (affix == PREFIX) ? output.codePointAt(index) : output.codePointBefore(index); + UnicodeSet numberUniset = getUnicodeSet(symbols, IN_NUMBER, affix, status); + if (!numberUniset.contains(numberCp)) { + return 0; + } + UnicodeString spacingString = getInsertString(symbols, affix, status); + + // NOTE: This next line *inserts* the spacing string, triggering an arraycopy. + // It would be more efficient if this could be done before affixes were attached, + // so that it could be prepended/appended instead of inserted. + // However, the build code path is more efficient, and this is the most natural + // place to put currency spacing in the non-build code path. + // TODO: Should we use the CURRENCY field here? + return output.insert(index, spacingString, kUndefinedField, status); +} + +UnicodeSet +CurrencySpacingEnabledModifier::getUnicodeSet(const DecimalFormatSymbols &symbols, EPosition position, + EAffix affix, UErrorCode &status) { + // Ensure the static defaults are initialized: + umtx_initOnce(gDefaultCurrencySpacingInitOnce, &initDefaultCurrencySpacing, status); + if (U_FAILURE(status)) { + return UnicodeSet(); + } + + const UnicodeString& pattern = symbols.getPatternForCurrencySpacing( + position == IN_CURRENCY ? UNUM_CURRENCY_MATCH : UNUM_CURRENCY_SURROUNDING_MATCH, + affix == SUFFIX, + status); + if (pattern.compare(u"[:digit:]", -1) == 0) { + return *UNISET_DIGIT; + } else if (pattern.compare(u"[[:^S:]&[:^Z:]]", -1) == 0) { + return *UNISET_NOTSZ; + } else { + return UnicodeSet(pattern, status); + } +} + +UnicodeString +CurrencySpacingEnabledModifier::getInsertString(const DecimalFormatSymbols &symbols, EAffix affix, + UErrorCode &status) { + return symbols.getPatternForCurrencySpacing(UNUM_CURRENCY_INSERT, affix == SUFFIX, status); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_modifiers.h b/intl/icu/source/i18n/number_modifiers.h new file mode 100644 index 0000000000..da6956bcfb --- /dev/null +++ b/intl/icu/source/i18n/number_modifiers.h @@ -0,0 +1,360 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_MODIFIERS_H__ +#define __NUMBER_MODIFIERS_H__ + +#include +#include +#include "unicode/uniset.h" +#include "unicode/simpleformatter.h" +#include "standardplural.h" +#include "formatted_string_builder.h" +#include "number_types.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +/** + * The canonical implementation of {@link Modifier}, containing a prefix and suffix string. + * TODO: This is not currently being used by real code and could be removed. + */ +class U_I18N_API ConstantAffixModifier : public Modifier, public UObject { + public: + ConstantAffixModifier(const UnicodeString &prefix, const UnicodeString &suffix, Field field, + bool strong) + : fPrefix(prefix), fSuffix(suffix), fField(field), fStrong(strong) {} + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + int32_t getPrefixLength() const override; + + int32_t getCodePointCount() const override; + + bool isStrong() const override; + + bool containsField(Field field) const override; + + void getParameters(Parameters& output) const override; + + bool semanticallyEquivalent(const Modifier& other) const override; + + private: + UnicodeString fPrefix; + UnicodeString fSuffix; + Field fField; + bool fStrong; +}; + +/** + * The second primary implementation of {@link Modifier}, this one consuming a {@link SimpleFormatter} + * pattern. + */ +class U_I18N_API SimpleModifier : public Modifier, public UMemory { + public: + SimpleModifier(const SimpleFormatter &simpleFormatter, Field field, bool strong); + + SimpleModifier(const SimpleFormatter &simpleFormatter, Field field, bool strong, + const Modifier::Parameters parameters); + + // Default constructor for LongNameHandler.h + SimpleModifier(); + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + int32_t getPrefixLength() const override; + + int32_t getCodePointCount() const override; + + bool isStrong() const override; + + bool containsField(Field field) const override; + + void getParameters(Parameters& output) const override; + + bool semanticallyEquivalent(const Modifier& other) const override; + + /** + * TODO: This belongs in SimpleFormatterImpl. The only reason I haven't moved it there yet is because + * FormattedStringBuilder is an internal class and SimpleFormatterImpl feels like it should not depend on it. + * + *

    + * Formats a value that is already stored inside the StringBuilder result between the indices + * startIndex and endIndex by inserting characters before the start index and after the + * end index. + * + *

    + * This is well-defined only for patterns with exactly one argument. + * + * @param result + * The StringBuilder containing the value argument. + * @param startIndex + * The left index of the value within the string builder. + * @param endIndex + * The right index of the value within the string builder. + * @return The number of characters (UTF-16 code points) that were added to the StringBuilder. + */ + int32_t + formatAsPrefixSuffix(FormattedStringBuilder& result, int32_t startIndex, int32_t endIndex, + UErrorCode& status) const; + + /** + * TODO: Like above, this belongs with the rest of the SimpleFormatterImpl code. + * I put it here so that the SimpleFormatter uses in FormattedStringBuilder are near each other. + * + *

    + * Applies the compiled two-argument pattern to the FormattedStringBuilder. + * + *

    + * This method is optimized for the case where the prefix and suffix are often empty, such as + * in the range pattern like "{0}-{1}". + */ + static int32_t + formatTwoArgPattern(const SimpleFormatter& compiled, FormattedStringBuilder& result, + int32_t index, int32_t* outPrefixLength, int32_t* outSuffixLength, + Field field, UErrorCode& status); + + private: + UnicodeString fCompiledPattern; + Field fField; + bool fStrong = false; + int32_t fPrefixLength = 0; + int32_t fSuffixOffset = -1; + int32_t fSuffixLength = 0; + Modifier::Parameters fParameters; +}; + +/** + * An implementation of {@link Modifier} that allows for multiple types of fields in the same modifier. Constructed + * based on the contents of two {@link FormattedStringBuilder} instances (one for the prefix, one for the suffix). + */ +class U_I18N_API ConstantMultiFieldModifier : public Modifier, public UMemory { + public: + ConstantMultiFieldModifier( + const FormattedStringBuilder &prefix, + const FormattedStringBuilder &suffix, + bool overwrite, + bool strong, + const Modifier::Parameters parameters) + : fPrefix(prefix), + fSuffix(suffix), + fOverwrite(overwrite), + fStrong(strong), + fParameters(parameters) {} + + ConstantMultiFieldModifier( + const FormattedStringBuilder &prefix, + const FormattedStringBuilder &suffix, + bool overwrite, + bool strong) + : fPrefix(prefix), + fSuffix(suffix), + fOverwrite(overwrite), + fStrong(strong) {} + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + int32_t getPrefixLength() const override; + + int32_t getCodePointCount() const override; + + bool isStrong() const override; + + bool containsField(Field field) const override; + + void getParameters(Parameters& output) const override; + + bool semanticallyEquivalent(const Modifier& other) const override; + + protected: + // NOTE: In Java, these are stored as array pointers. In C++, the FormattedStringBuilder is stored by + // value and is treated internally as immutable. + FormattedStringBuilder fPrefix; + FormattedStringBuilder fSuffix; + bool fOverwrite; + bool fStrong; + Modifier::Parameters fParameters; +}; + +/** Identical to {@link ConstantMultiFieldModifier}, but supports currency spacing. */ +class U_I18N_API CurrencySpacingEnabledModifier : public ConstantMultiFieldModifier { + public: + /** Safe code path */ + CurrencySpacingEnabledModifier( + const FormattedStringBuilder &prefix, + const FormattedStringBuilder &suffix, + bool overwrite, + bool strong, + const DecimalFormatSymbols &symbols, + UErrorCode &status); + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + /** Unsafe code path */ + static int32_t + applyCurrencySpacing(FormattedStringBuilder &output, int32_t prefixStart, int32_t prefixLen, + int32_t suffixStart, int32_t suffixLen, const DecimalFormatSymbols &symbols, + UErrorCode &status); + + private: + UnicodeSet fAfterPrefixUnicodeSet; + UnicodeString fAfterPrefixInsert; + UnicodeSet fBeforeSuffixUnicodeSet; + UnicodeString fBeforeSuffixInsert; + + enum EAffix { + PREFIX, SUFFIX + }; + + enum EPosition { + IN_CURRENCY, IN_NUMBER + }; + + /** Unsafe code path */ + static int32_t applyCurrencySpacingAffix(FormattedStringBuilder &output, int32_t index, EAffix affix, + const DecimalFormatSymbols &symbols, UErrorCode &status); + + static UnicodeSet + getUnicodeSet(const DecimalFormatSymbols &symbols, EPosition position, EAffix affix, + UErrorCode &status); + + static UnicodeString + getInsertString(const DecimalFormatSymbols &symbols, EAffix affix, UErrorCode &status); +}; + +/** A Modifier that does not do anything. */ +class U_I18N_API EmptyModifier : public Modifier, public UMemory { + public: + explicit EmptyModifier(bool isStrong) : fStrong(isStrong) {} + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override { + (void)output; + (void)leftIndex; + (void)rightIndex; + (void)status; + return 0; + } + + int32_t getPrefixLength() const override { + return 0; + } + + int32_t getCodePointCount() const override { + return 0; + } + + bool isStrong() const override { + return fStrong; + } + + bool containsField(Field field) const override { + (void)field; + return false; + } + + void getParameters(Parameters& output) const override { + output.obj = nullptr; + } + + bool semanticallyEquivalent(const Modifier& other) const override { + return other.getCodePointCount() == 0; + } + + private: + bool fStrong; +}; + +/** An adopting Modifier store that varies by signum but not plural form. */ +class U_I18N_API AdoptingSignumModifierStore : public UMemory { + public: + virtual ~AdoptingSignumModifierStore(); + + AdoptingSignumModifierStore() = default; + + // No copying! + AdoptingSignumModifierStore(const AdoptingSignumModifierStore &other) = delete; + AdoptingSignumModifierStore& operator=(const AdoptingSignumModifierStore& other) = delete; + + // Moving is OK + AdoptingSignumModifierStore(AdoptingSignumModifierStore &&other) noexcept { + *this = std::move(other); + } + AdoptingSignumModifierStore& operator=(AdoptingSignumModifierStore&& other) noexcept; + + /** Take ownership of the Modifier and slot it in at the given Signum. */ + void adoptModifier(Signum signum, const Modifier* mod) { + U_ASSERT(mods[signum] == nullptr); + mods[signum] = mod; + } + + inline const Modifier*& operator[](Signum signum) { + return mods[signum]; + } + inline Modifier const* operator[](Signum signum) const { + return mods[signum]; + } + + private: + const Modifier* mods[SIGNUM_COUNT] = {}; +}; + +/** + * This implementation of ModifierStore adopts Modifier pointers. + */ +class U_I18N_API AdoptingModifierStore : public ModifierStore, public UMemory { + public: + static constexpr StandardPlural::Form DEFAULT_STANDARD_PLURAL = StandardPlural::OTHER; + + AdoptingModifierStore() = default; + + // No copying! + AdoptingModifierStore(const AdoptingModifierStore &other) = delete; + + // Moving is OK + AdoptingModifierStore(AdoptingModifierStore &&other) = default; + + /** Sets the modifiers for a specific plural form. */ + void adoptSignumModifierStore(StandardPlural::Form plural, AdoptingSignumModifierStore other) { + mods[plural] = std::move(other); + } + + /** Sets the modifiers for the default plural form. */ + void adoptSignumModifierStoreNoPlural(AdoptingSignumModifierStore other) { + mods[DEFAULT_STANDARD_PLURAL] = std::move(other); + } + + /** Returns a reference to the modifier; no ownership change. */ + const Modifier *getModifier(Signum signum, StandardPlural::Form plural) const override { + const Modifier* modifier = mods[plural][signum]; + if (modifier == nullptr && plural != DEFAULT_STANDARD_PLURAL) { + modifier = mods[DEFAULT_STANDARD_PLURAL][signum]; + } + return modifier; + } + + /** Returns a reference to the modifier; no ownership change. */ + const Modifier *getModifierWithoutPlural(Signum signum) const { + return mods[DEFAULT_STANDARD_PLURAL][signum]; + } + + private: + // NOTE: mods is zero-initialized (to nullptr) + AdoptingSignumModifierStore mods[StandardPlural::COUNT] = {}; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_MODIFIERS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_multiplier.cpp b/intl/icu/source/i18n/number_multiplier.cpp new file mode 100644 index 0000000000..dd5b3f8e06 --- /dev/null +++ b/intl/icu/source/i18n/number_multiplier.cpp @@ -0,0 +1,160 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "number_decnum.h" +#include "number_types.h" +#include "number_multiplier.h" +#include "numparse_validators.h" +#include "number_utils.h" +#include "decNumber.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using namespace icu::numparse::impl; + + +Scale::Scale(int32_t magnitude, DecNum* arbitraryToAdopt) + : fMagnitude(magnitude), fArbitrary(arbitraryToAdopt), fError(U_ZERO_ERROR) { + if (fArbitrary != nullptr) { + // Attempt to convert the DecNum to a magnitude multiplier. + fArbitrary->normalize(); + if (fArbitrary->getRawDecNumber()->digits == 1 && fArbitrary->getRawDecNumber()->lsu[0] == 1 && + !fArbitrary->isNegative()) { + // Success! + fMagnitude += fArbitrary->getRawDecNumber()->exponent; + delete fArbitrary; + fArbitrary = nullptr; + } + } +} + +Scale::Scale(const Scale& other) + : fMagnitude(other.fMagnitude), fArbitrary(nullptr), fError(other.fError) { + if (other.fArbitrary != nullptr) { + UErrorCode localStatus = U_ZERO_ERROR; + fArbitrary = new DecNum(*other.fArbitrary, localStatus); + } +} + +Scale& Scale::operator=(const Scale& other) { + if (this == &other) { return *this; } // self-assignment: no-op + fMagnitude = other.fMagnitude; + if (other.fArbitrary != nullptr) { + UErrorCode localStatus = U_ZERO_ERROR; + fArbitrary = new DecNum(*other.fArbitrary, localStatus); + } else { + fArbitrary = nullptr; + } + fError = other.fError; + return *this; +} + +Scale::Scale(Scale&& src) noexcept + : fMagnitude(src.fMagnitude), fArbitrary(src.fArbitrary), fError(src.fError) { + // Take ownership away from src if necessary + src.fArbitrary = nullptr; +} + +Scale& Scale::operator=(Scale&& src) noexcept { + fMagnitude = src.fMagnitude; + if (fArbitrary != nullptr) { + delete fArbitrary; + } + fArbitrary = src.fArbitrary; + fError = src.fError; + // Take ownership away from src if necessary + src.fArbitrary = nullptr; + return *this; +} + +Scale::~Scale() { + delete fArbitrary; +} + + +Scale Scale::none() { + return {0, nullptr}; +} + +Scale Scale::powerOfTen(int32_t power) { + return {power, nullptr}; +} + +Scale Scale::byDecimal(StringPiece multiplicand) { + UErrorCode localError = U_ZERO_ERROR; + LocalPointer decnum(new DecNum(), localError); + if (U_FAILURE(localError)) { + return {localError}; + } + decnum->setTo(multiplicand, localError); + if (U_FAILURE(localError)) { + return {localError}; + } + return {0, decnum.orphan()}; +} + +Scale Scale::byDouble(double multiplicand) { + UErrorCode localError = U_ZERO_ERROR; + LocalPointer decnum(new DecNum(), localError); + if (U_FAILURE(localError)) { + return {localError}; + } + decnum->setTo(multiplicand, localError); + if (U_FAILURE(localError)) { + return {localError}; + } + return {0, decnum.orphan()}; +} + +Scale Scale::byDoubleAndPowerOfTen(double multiplicand, int32_t power) { + UErrorCode localError = U_ZERO_ERROR; + LocalPointer decnum(new DecNum(), localError); + if (U_FAILURE(localError)) { + return {localError}; + } + decnum->setTo(multiplicand, localError); + if (U_FAILURE(localError)) { + return {localError}; + } + return {power, decnum.orphan()}; +} + +void Scale::applyTo(impl::DecimalQuantity& quantity) const { + quantity.adjustMagnitude(fMagnitude); + if (fArbitrary != nullptr) { + UErrorCode localStatus = U_ZERO_ERROR; + quantity.multiplyBy(*fArbitrary, localStatus); + } +} + +void Scale::applyReciprocalTo(impl::DecimalQuantity& quantity) const { + quantity.adjustMagnitude(-fMagnitude); + if (fArbitrary != nullptr) { + UErrorCode localStatus = U_ZERO_ERROR; + quantity.divideBy(*fArbitrary, localStatus); + } +} + + +void +MultiplierFormatHandler::setAndChain(const Scale& multiplier, const MicroPropsGenerator* parent) { + fMultiplier = multiplier; + fParent = parent; +} + +void MultiplierFormatHandler::processQuantity(DecimalQuantity& quantity, MicroProps& micros, + UErrorCode& status) const { + fParent->processQuantity(quantity, micros, status); + fMultiplier.applyTo(quantity); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_multiplier.h b/intl/icu/source/i18n/number_multiplier.h new file mode 100644 index 0000000000..c752935b78 --- /dev/null +++ b/intl/icu/source/i18n/number_multiplier.h @@ -0,0 +1,57 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMBER_MULTIPLIER_H__ +#define __SOURCE_NUMBER_MULTIPLIER_H__ + +#include "numparse_types.h" +#include "number_decimfmtprops.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + + +/** + * Wraps a {@link Multiplier} for use in the number formatting pipeline. + */ +// Exported as U_I18N_API for tests +class U_I18N_API MultiplierFormatHandler : public MicroPropsGenerator, public UMemory { + public: + MultiplierFormatHandler() = default; // WARNING: Leaves object in an unusable state; call setAndChain() + + void setAndChain(const Scale& multiplier, const MicroPropsGenerator* parent); + + void processQuantity(DecimalQuantity& quantity, MicroProps& micros, + UErrorCode& status) const override; + + private: + Scale fMultiplier; + const MicroPropsGenerator *fParent; +}; + + +/** Gets a Scale from a DecimalFormatProperties. In Java, defined in RoundingUtils.java */ +static inline Scale scaleFromProperties(const DecimalFormatProperties& properties) { + int32_t magnitudeMultiplier = properties.magnitudeMultiplier + properties.multiplierScale; + int32_t arbitraryMultiplier = properties.multiplier; + if (magnitudeMultiplier != 0 && arbitraryMultiplier != 1) { + return Scale::byDoubleAndPowerOfTen(arbitraryMultiplier, magnitudeMultiplier); + } else if (magnitudeMultiplier != 0) { + return Scale::powerOfTen(magnitudeMultiplier); + } else if (arbitraryMultiplier != 1) { + return Scale::byDouble(arbitraryMultiplier); + } else { + return Scale::none(); + } +} + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__SOURCE_NUMBER_MULTIPLIER_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_notation.cpp b/intl/icu/source/i18n/number_notation.cpp new file mode 100644 index 0000000000..b3cabb57a5 --- /dev/null +++ b/intl/icu/source/i18n/number_notation.cpp @@ -0,0 +1,88 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numberformatter.h" +#include "number_types.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +ScientificNotation Notation::scientific() { + // NOTE: ISO C++ does not allow C99 designated initializers. + ScientificSettings settings; + settings.fEngineeringInterval = 1; + settings.fRequireMinInt = false; + settings.fMinExponentDigits = 1; + settings.fExponentSignDisplay = UNUM_SIGN_AUTO; + NotationUnion union_; + union_.scientific = settings; + return {NTN_SCIENTIFIC, union_}; +} + +ScientificNotation Notation::engineering() { + ScientificSettings settings; + settings.fEngineeringInterval = 3; + settings.fRequireMinInt = false; + settings.fMinExponentDigits = 1; + settings.fExponentSignDisplay = UNUM_SIGN_AUTO; + NotationUnion union_; + union_.scientific = settings; + return {NTN_SCIENTIFIC, union_}; +} + +ScientificNotation::ScientificNotation(int8_t fEngineeringInterval, bool fRequireMinInt, + impl::digits_t fMinExponentDigits, + UNumberSignDisplay fExponentSignDisplay) { + ScientificSettings settings; + settings.fEngineeringInterval = fEngineeringInterval; + settings.fRequireMinInt = fRequireMinInt; + settings.fMinExponentDigits = fMinExponentDigits; + settings.fExponentSignDisplay = fExponentSignDisplay; + NotationUnion union_; + union_.scientific = settings; + *this = {NTN_SCIENTIFIC, union_}; +} + +Notation Notation::compactShort() { + NotationUnion union_; + union_.compactStyle = CompactStyle::UNUM_SHORT; + return {NTN_COMPACT, union_}; +} + +Notation Notation::compactLong() { + NotationUnion union_; + union_.compactStyle = CompactStyle::UNUM_LONG; + return {NTN_COMPACT, union_}; +} + +Notation Notation::simple() { + return {}; +} + +ScientificNotation +ScientificNotation::withMinExponentDigits(int32_t minExponentDigits) const { + if (minExponentDigits >= 1 && minExponentDigits <= kMaxIntFracSig) { + ScientificSettings settings = fUnion.scientific; + settings.fMinExponentDigits = static_cast(minExponentDigits); + NotationUnion union_ = {settings}; + return {NTN_SCIENTIFIC, union_}; + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +ScientificNotation +ScientificNotation::withExponentSignDisplay(UNumberSignDisplay exponentSignDisplay) const { + ScientificSettings settings = fUnion.scientific; + settings.fExponentSignDisplay = exponentSignDisplay; + NotationUnion union_ = {settings}; + return {NTN_SCIENTIFIC, union_}; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_output.cpp b/intl/icu/source/i18n/number_output.cpp new file mode 100644 index 0000000000..729a2cd5e6 --- /dev/null +++ b/intl/icu/source/i18n/number_output.cpp @@ -0,0 +1,86 @@ +// © 2019 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/measunit.h" +#include "unicode/numberformatter.h" +#include "number_utypes.h" +#include "util.h" +#include "number_decimalquantity.h" +#include "number_decnum.h" +#include "numrange_impl.h" + +U_NAMESPACE_BEGIN +namespace number { + + +UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(FormattedNumber) + +#define UPRV_NOARG + +void FormattedNumber::toDecimalNumber(ByteSink& sink, UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UPRV_NOARG) + impl::DecNum decnum; + fData->quantity.toDecNum(decnum, status); + decnum.toString(sink, status); +} + +void FormattedNumber::getAllFieldPositionsImpl(FieldPositionIteratorHandler& fpih, + UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UPRV_NOARG) + fData->getAllFieldPositions(fpih, status); +} + +MeasureUnit FormattedNumber::getOutputUnit(UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(MeasureUnit()) + return fData->outputUnit; +} + +UDisplayOptionsNounClass FormattedNumber::getNounClass(UErrorCode &status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UDISPOPT_NOUN_CLASS_UNDEFINED); + const char *nounClass = fData->gender; + return udispopt_fromNounClassIdentifier(nounClass); +} + +void FormattedNumber::getDecimalQuantity(impl::DecimalQuantity& output, UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UPRV_NOARG) + output = fData->quantity; +} + + +impl::UFormattedNumberData::~UFormattedNumberData() = default; + + +UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(FormattedNumberRange) + +#define UPRV_NOARG + +void FormattedNumberRange::getDecimalNumbers(ByteSink& sink1, ByteSink& sink2, UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UPRV_NOARG) + impl::DecNum decnum1; + impl::DecNum decnum2; + fData->quantity1.toDecNum(decnum1, status).toString(sink1, status); + fData->quantity2.toDecNum(decnum2, status).toString(sink2, status); +} + +UNumberRangeIdentityResult FormattedNumberRange::getIdentityResult(UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(UNUM_IDENTITY_RESULT_NOT_EQUAL) + return fData->identityResult; +} + +const impl::UFormattedNumberRangeData* FormattedNumberRange::getData(UErrorCode& status) const { + UPRV_FORMATTED_VALUE_METHOD_GUARD(nullptr) + return fData; +} + + +impl::UFormattedNumberRangeData::~UFormattedNumberRangeData() = default; + + +} // namespace number +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_padding.cpp b/intl/icu/source/i18n/number_padding.cpp new file mode 100644 index 0000000000..c320c3ffb6 --- /dev/null +++ b/intl/icu/source/i18n/number_padding.cpp @@ -0,0 +1,96 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "formatted_string_builder.h" +#include "number_decimfmtprops.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +int32_t +addPaddingHelper(UChar32 paddingCp, int32_t requiredPadding, FormattedStringBuilder &string, int32_t index, + UErrorCode &status) { + for (int32_t i = 0; i < requiredPadding; i++) { + // TODO: If appending to the end, this will cause actual insertion operations. Improve. + string.insertCodePoint(index, paddingCp, kUndefinedField, status); + } + return U16_LENGTH(paddingCp) * requiredPadding; +} + +} + +Padder::Padder(UChar32 cp, int32_t width, UNumberFormatPadPosition position) : fWidth(width) { + // TODO(13034): Consider making this a string instead of code point. + fUnion.padding.fCp = cp; + fUnion.padding.fPosition = position; +} + +Padder::Padder(int32_t width) : fWidth(width) {} + +Padder Padder::none() { + return {-1}; +} + +Padder Padder::codePoints(UChar32 cp, int32_t targetWidth, UNumberFormatPadPosition position) { + // TODO: Validate the code point? + if (targetWidth >= 0) { + return {cp, targetWidth, position}; + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Padder Padder::forProperties(const DecimalFormatProperties& properties) { + UChar32 padCp; + if (properties.padString.length() > 0) { + padCp = properties.padString.char32At(0); + } else { + padCp = kFallbackPaddingString[0]; + } + return {padCp, properties.formatWidth, properties.padPosition.getOrDefault(UNUM_PAD_BEFORE_PREFIX)}; +} + +int32_t Padder::padAndApply(const Modifier &mod1, const Modifier &mod2, + FormattedStringBuilder &string, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const { + int32_t modLength = mod1.getCodePointCount() + mod2.getCodePointCount(); + int32_t requiredPadding = fWidth - modLength - string.codePointCount(); + U_ASSERT(leftIndex == 0 && + rightIndex == string.length()); // fix the previous line to remove this assertion + + int length = 0; + if (requiredPadding <= 0) { + // Padding is not required. + length += mod1.apply(string, leftIndex, rightIndex, status); + length += mod2.apply(string, leftIndex, rightIndex + length, status); + return length; + } + + PadPosition position = fUnion.padding.fPosition; + UChar32 paddingCp = fUnion.padding.fCp; + if (position == UNUM_PAD_AFTER_PREFIX) { + length += addPaddingHelper(paddingCp, requiredPadding, string, leftIndex, status); + } else if (position == UNUM_PAD_BEFORE_SUFFIX) { + length += addPaddingHelper(paddingCp, requiredPadding, string, rightIndex + length, status); + } + length += mod1.apply(string, leftIndex, rightIndex + length, status); + length += mod2.apply(string, leftIndex, rightIndex + length, status); + if (position == UNUM_PAD_BEFORE_PREFIX) { + length += addPaddingHelper(paddingCp, requiredPadding, string, leftIndex, status); + } else if (position == UNUM_PAD_AFTER_SUFFIX) { + length += addPaddingHelper(paddingCp, requiredPadding, string, rightIndex + length, status); + } + + return length; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_patternmodifier.cpp b/intl/icu/source/i18n/number_patternmodifier.cpp new file mode 100644 index 0000000000..6f398c6acf --- /dev/null +++ b/intl/icu/source/i18n/number_patternmodifier.cpp @@ -0,0 +1,348 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "cstring.h" +#include "number_patternmodifier.h" +#include "unicode/dcfmtsym.h" +#include "unicode/ucurr.h" +#include "unicode/unistr.h" +#include "number_microprops.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +AffixPatternProvider::~AffixPatternProvider() = default; + + +MutablePatternModifier::MutablePatternModifier(bool isStrong) + : fStrong(isStrong) {} + +void MutablePatternModifier::setPatternInfo(const AffixPatternProvider* patternInfo, Field field) { + fPatternInfo = patternInfo; + fField = field; +} + +void MutablePatternModifier::setPatternAttributes( + UNumberSignDisplay signDisplay, + bool perMille, + bool approximately) { + fSignDisplay = signDisplay; + fPerMilleReplacesPercent = perMille; + fApproximately = approximately; +} + +void MutablePatternModifier::setSymbols(const DecimalFormatSymbols* symbols, + const CurrencyUnit& currency, + const UNumberUnitWidth unitWidth, + const PluralRules* rules, + UErrorCode& status) { + U_ASSERT((rules != nullptr) == needsPlurals()); + fSymbols = symbols; + fCurrencySymbols = {currency, symbols->getLocale(), *symbols, status}; + fUnitWidth = unitWidth; + fRules = rules; +} + +void MutablePatternModifier::setNumberProperties(Signum signum, StandardPlural::Form plural) { + fSignum = signum; + fPlural = plural; +} + +bool MutablePatternModifier::needsPlurals() const { + UErrorCode statusLocal = U_ZERO_ERROR; + return fPatternInfo->containsSymbolType(AffixPatternType::TYPE_CURRENCY_TRIPLE, statusLocal); + // Silently ignore any error codes. +} + +AdoptingSignumModifierStore MutablePatternModifier::createImmutableForPlural(StandardPlural::Form plural, UErrorCode& status) { + AdoptingSignumModifierStore pm; + + setNumberProperties(SIGNUM_POS, plural); + pm.adoptModifier(SIGNUM_POS, createConstantModifier(status)); + setNumberProperties(SIGNUM_NEG_ZERO, plural); + pm.adoptModifier(SIGNUM_NEG_ZERO, createConstantModifier(status)); + setNumberProperties(SIGNUM_POS_ZERO, plural); + pm.adoptModifier(SIGNUM_POS_ZERO, createConstantModifier(status)); + setNumberProperties(SIGNUM_NEG, plural); + pm.adoptModifier(SIGNUM_NEG, createConstantModifier(status)); + + return pm; +} + +ImmutablePatternModifier* MutablePatternModifier::createImmutable(UErrorCode& status) { + // TODO: Move StandardPlural VALUES to standardplural.h + static const StandardPlural::Form STANDARD_PLURAL_VALUES[] = { + StandardPlural::Form::ZERO, + StandardPlural::Form::ONE, + StandardPlural::Form::TWO, + StandardPlural::Form::FEW, + StandardPlural::Form::MANY, + StandardPlural::Form::OTHER}; + + auto pm = new AdoptingModifierStore(); + if (pm == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + + if (needsPlurals()) { + // Slower path when we require the plural keyword. + for (StandardPlural::Form plural : STANDARD_PLURAL_VALUES) { + pm->adoptSignumModifierStore(plural, createImmutableForPlural(plural, status)); + } + if (U_FAILURE(status)) { + delete pm; + return nullptr; + } + return new ImmutablePatternModifier(pm, fRules); // adopts pm + } else { + // Faster path when plural keyword is not needed. + pm->adoptSignumModifierStoreNoPlural(createImmutableForPlural(StandardPlural::Form::COUNT, status)); + if (U_FAILURE(status)) { + delete pm; + return nullptr; + } + return new ImmutablePatternModifier(pm, nullptr); // adopts pm + } +} + +ConstantMultiFieldModifier* MutablePatternModifier::createConstantModifier(UErrorCode& status) { + FormattedStringBuilder a; + FormattedStringBuilder b; + insertPrefix(a, 0, status); + insertSuffix(b, 0, status); + if (fPatternInfo->hasCurrencySign()) { + return new CurrencySpacingEnabledModifier( + a, b, !fPatternInfo->hasBody(), fStrong, *fSymbols, status); + } else { + return new ConstantMultiFieldModifier(a, b, !fPatternInfo->hasBody(), fStrong); + } +} + +ImmutablePatternModifier::ImmutablePatternModifier(AdoptingModifierStore* pm, const PluralRules* rules) + : pm(pm), rules(rules), parent(nullptr) {} + +void ImmutablePatternModifier::processQuantity(DecimalQuantity& quantity, MicroProps& micros, + UErrorCode& status) const { + parent->processQuantity(quantity, micros, status); + micros.rounder.apply(quantity, status); + if (micros.modMiddle != nullptr) { + return; + } + applyToMicros(micros, quantity, status); +} + +void ImmutablePatternModifier::applyToMicros( + MicroProps& micros, const DecimalQuantity& quantity, UErrorCode& status) const { + if (rules == nullptr) { + micros.modMiddle = pm->getModifierWithoutPlural(quantity.signum()); + } else { + StandardPlural::Form pluralForm = utils::getPluralSafe(micros.rounder, rules, quantity, status); + micros.modMiddle = pm->getModifier(quantity.signum(), pluralForm); + } +} + +const Modifier* ImmutablePatternModifier::getModifier(Signum signum, StandardPlural::Form plural) const { + if (rules == nullptr) { + return pm->getModifierWithoutPlural(signum); + } else { + return pm->getModifier(signum, plural); + } +} + +void ImmutablePatternModifier::addToChain(const MicroPropsGenerator* parent) { + this->parent = parent; +} + + +/** Used by the unsafe code path. */ +MicroPropsGenerator& MutablePatternModifier::addToChain(const MicroPropsGenerator* parent) { + fParent = parent; + return *this; +} + +void MutablePatternModifier::processQuantity(DecimalQuantity& fq, MicroProps& micros, + UErrorCode& status) const { + fParent->processQuantity(fq, micros, status); + micros.rounder.apply(fq, status); + if (micros.modMiddle != nullptr) { + return; + } + // The unsafe code path performs self-mutation, so we need a const_cast. + // This method needs to be const because it overrides a const method in the parent class. + auto nonConstThis = const_cast(this); + if (needsPlurals()) { + StandardPlural::Form pluralForm = utils::getPluralSafe(micros.rounder, fRules, fq, status); + nonConstThis->setNumberProperties(fq.signum(), pluralForm); + } else { + nonConstThis->setNumberProperties(fq.signum(), StandardPlural::Form::COUNT); + } + micros.modMiddle = this; +} + +int32_t MutablePatternModifier::apply(FormattedStringBuilder& output, int32_t leftIndex, int32_t rightIndex, + UErrorCode& status) const { + // The unsafe code path performs self-mutation, so we need a const_cast. + // This method needs to be const because it overrides a const method in the parent class. + auto nonConstThis = const_cast(this); + int32_t prefixLen = nonConstThis->insertPrefix(output, leftIndex, status); + int32_t suffixLen = nonConstThis->insertSuffix(output, rightIndex + prefixLen, status); + // If the pattern had no decimal stem body (like #,##0.00), overwrite the value. + int32_t overwriteLen = 0; + if (!fPatternInfo->hasBody()) { + overwriteLen = output.splice( + leftIndex + prefixLen, + rightIndex + prefixLen, + UnicodeString(), + 0, + 0, + kUndefinedField, + status); + } + CurrencySpacingEnabledModifier::applyCurrencySpacing( + output, + leftIndex, + prefixLen, + rightIndex + overwriteLen + prefixLen, + suffixLen, + *fSymbols, + status); + return prefixLen + overwriteLen + suffixLen; +} + +int32_t MutablePatternModifier::getPrefixLength() const { + // The unsafe code path performs self-mutation, so we need a const_cast. + // This method needs to be const because it overrides a const method in the parent class. + auto nonConstThis = const_cast(this); + + // Enter and exit CharSequence Mode to get the length. + UErrorCode status = U_ZERO_ERROR; // status fails only with an iilegal argument exception + nonConstThis->prepareAffix(true); + int result = AffixUtils::unescapedCodePointCount(currentAffix, *this, status); // prefix length + return result; +} + +int32_t MutablePatternModifier::getCodePointCount() const { + // The unsafe code path performs self-mutation, so we need a const_cast. + // This method needs to be const because it overrides a const method in the parent class. + auto nonConstThis = const_cast(this); + + // Render the affixes to get the length + UErrorCode status = U_ZERO_ERROR; // status fails only with an iilegal argument exception + nonConstThis->prepareAffix(true); + int result = AffixUtils::unescapedCodePointCount(currentAffix, *this, status); // prefix length + nonConstThis->prepareAffix(false); + result += AffixUtils::unescapedCodePointCount(currentAffix, *this, status); // suffix length + return result; +} + +bool MutablePatternModifier::isStrong() const { + return fStrong; +} + +bool MutablePatternModifier::containsField(Field field) const { + (void)field; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +void MutablePatternModifier::getParameters(Parameters& output) const { + (void)output; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +bool MutablePatternModifier::semanticallyEquivalent(const Modifier& other) const { + (void)other; + // This method is not currently used. + UPRV_UNREACHABLE_EXIT; +} + +int32_t MutablePatternModifier::insertPrefix(FormattedStringBuilder& sb, int position, UErrorCode& status) { + prepareAffix(true); + int32_t length = AffixUtils::unescape(currentAffix, sb, position, *this, fField, status); + return length; +} + +int32_t MutablePatternModifier::insertSuffix(FormattedStringBuilder& sb, int position, UErrorCode& status) { + prepareAffix(false); + int32_t length = AffixUtils::unescape(currentAffix, sb, position, *this, fField, status); + return length; +} + +/** This method contains the heart of the logic for rendering LDML affix strings. */ +void MutablePatternModifier::prepareAffix(bool isPrefix) { + PatternStringUtils::patternInfoToStringBuilder( + *fPatternInfo, + isPrefix, + PatternStringUtils::resolveSignDisplay(fSignDisplay, fSignum), + fApproximately, + fPlural, + fPerMilleReplacesPercent, + false, // dropCurrencySymbols + currentAffix); +} + +UnicodeString MutablePatternModifier::getSymbol(AffixPatternType type) const { + UErrorCode localStatus = U_ZERO_ERROR; + switch (type) { + case AffixPatternType::TYPE_MINUS_SIGN: + return fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kMinusSignSymbol); + case AffixPatternType::TYPE_PLUS_SIGN: + return fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kPlusSignSymbol); + case AffixPatternType::TYPE_APPROXIMATELY_SIGN: + return fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kApproximatelySignSymbol); + case AffixPatternType::TYPE_PERCENT: + return fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kPercentSymbol); + case AffixPatternType::TYPE_PERMILLE: + return fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kPerMillSymbol); + case AffixPatternType::TYPE_CURRENCY_SINGLE: + return getCurrencySymbolForUnitWidth(localStatus); + case AffixPatternType::TYPE_CURRENCY_DOUBLE: + return fCurrencySymbols.getIntlCurrencySymbol(localStatus); + case AffixPatternType::TYPE_CURRENCY_TRIPLE: + // NOTE: This is the code path only for patterns containing "¤¤¤". + // Plural currencies set via the API are formatted in LongNameHandler. + // This code path is used by DecimalFormat via CurrencyPluralInfo. + U_ASSERT(fPlural != StandardPlural::Form::COUNT); + return fCurrencySymbols.getPluralName(fPlural, localStatus); + case AffixPatternType::TYPE_CURRENCY_QUAD: + return UnicodeString(u"\uFFFD"); + case AffixPatternType::TYPE_CURRENCY_QUINT: + return UnicodeString(u"\uFFFD"); + default: + UPRV_UNREACHABLE_EXIT; + } +} + +UnicodeString MutablePatternModifier::getCurrencySymbolForUnitWidth(UErrorCode& status) const { + switch (fUnitWidth) { + case UNumberUnitWidth::UNUM_UNIT_WIDTH_NARROW: + return fCurrencySymbols.getNarrowCurrencySymbol(status); + case UNumberUnitWidth::UNUM_UNIT_WIDTH_SHORT: + return fCurrencySymbols.getCurrencySymbol(status); + case UNumberUnitWidth::UNUM_UNIT_WIDTH_ISO_CODE: + return fCurrencySymbols.getIntlCurrencySymbol(status); + case UNumberUnitWidth::UNUM_UNIT_WIDTH_FORMAL: + return fCurrencySymbols.getFormalCurrencySymbol(status); + case UNumberUnitWidth::UNUM_UNIT_WIDTH_VARIANT: + return fCurrencySymbols.getVariantCurrencySymbol(status); + case UNumberUnitWidth::UNUM_UNIT_WIDTH_HIDDEN: + return UnicodeString(); + default: + return fCurrencySymbols.getCurrencySymbol(status); + } +} + +UnicodeString MutablePatternModifier::toUnicodeString() const { + // Never called by AffixUtils + UPRV_UNREACHABLE_EXIT; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_patternmodifier.h b/intl/icu/source/i18n/number_patternmodifier.h new file mode 100644 index 0000000000..ee38c20c9c --- /dev/null +++ b/intl/icu/source/i18n/number_patternmodifier.h @@ -0,0 +1,265 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_PATTERNMODIFIER_H__ +#define __NUMBER_PATTERNMODIFIER_H__ + +#include "standardplural.h" +#include "unicode/numberformatter.h" +#include "number_patternstring.h" +#include "number_types.h" +#include "number_modifiers.h" +#include "number_utils.h" +#include "number_currencysymbols.h" + +U_NAMESPACE_BEGIN + +// Export an explicit template instantiation of the LocalPointer that is used as a +// data member of AdoptingModifierStore. +// (When building DLLs for Windows this is required.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#if defined(_MSC_VER) +// Ignore warning 4661 as LocalPointerBase does not use operator== or operator!= +#pragma warning(push) +#pragma warning(disable : 4661) +#endif +template class U_I18N_API LocalPointerBase; +template class U_I18N_API LocalPointer; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + +namespace number { +namespace impl { + +// Forward declaration +class MutablePatternModifier; + +// Exported as U_I18N_API because it is needed for the unit test PatternModifierTest +class U_I18N_API ImmutablePatternModifier : public MicroPropsGenerator, public UMemory { + public: + ~ImmutablePatternModifier() override = default; + + void processQuantity(DecimalQuantity&, MicroProps& micros, UErrorCode& status) const override; + + void applyToMicros(MicroProps& micros, const DecimalQuantity& quantity, UErrorCode& status) const; + + const Modifier* getModifier(Signum signum, StandardPlural::Form plural) const; + + // Non-const method: + void addToChain(const MicroPropsGenerator* parent); + + private: + ImmutablePatternModifier(AdoptingModifierStore* pm, const PluralRules* rules); + + const LocalPointer pm; + const PluralRules* rules; + const MicroPropsGenerator* parent; + + friend class MutablePatternModifier; +}; + +/** + * This class is a {@link Modifier} that wraps a decimal format pattern. It applies the pattern's affixes in + * {@link Modifier#apply}. + * + *

    + * In addition to being a Modifier, this class contains the business logic for substituting the correct locale symbols + * into the affixes of the decimal format pattern. + * + *

    + * In order to use this class, create a new instance and call the following four setters: {@link #setPatternInfo}, + * {@link #setPatternAttributes}, {@link #setSymbols}, and {@link #setNumberProperties}. After calling these four + * setters, the instance will be ready for use as a Modifier. + * + *

    + * This is a MUTABLE, NON-THREAD-SAFE class designed for performance. Do NOT save references to this or attempt to use + * it from multiple threads! Instead, you can obtain a safe, immutable decimal format pattern modifier by calling + * {@link MutablePatternModifier#createImmutable}, in effect treating this instance as a builder for the immutable + * variant. + */ +class U_I18N_API MutablePatternModifier + : public MicroPropsGenerator, + public Modifier, + public SymbolProvider, + public UMemory { + public: + + ~MutablePatternModifier() override = default; + + /** + * @param isStrong + * Whether the modifier should be considered strong. For more information, see + * {@link Modifier#isStrong()}. Most of the time, decimal format pattern modifiers should be considered + * as non-strong. + */ + explicit MutablePatternModifier(bool isStrong); + + /** + * Sets a reference to the parsed decimal format pattern, usually obtained from + * {@link PatternStringParser#parseToPatternInfo(String)}, but any implementation of {@link AffixPatternProvider} is + * accepted. + * + * @param field + * Which field to use for literal characters in the pattern. + */ + void setPatternInfo(const AffixPatternProvider *patternInfo, Field field); + + /** + * Sets attributes that imply changes to the literal interpretation of the pattern string affixes. + * + * @param signDisplay + * Whether to force a plus sign on positive numbers. + * @param perMille + * Whether to substitute the percent sign in the pattern with a permille sign. + * @param approximately + * Whether to prepend approximately to the sign + */ + void setPatternAttributes(UNumberSignDisplay signDisplay, bool perMille, bool approximately); + + /** + * Sets locale-specific details that affect the symbols substituted into the pattern string affixes. + * + * @param symbols + * The desired instance of DecimalFormatSymbols. + * @param currency + * The currency to be used when substituting currency values into the affixes. + * @param unitWidth + * The width used to render currencies. + * @param rules + * Required if the triple currency sign, "¤¤¤", appears in the pattern, which can be determined from the + * convenience method {@link #needsPlurals()}. + * @param status + * Set if an error occurs while loading currency data. + */ + void setSymbols(const DecimalFormatSymbols* symbols, const CurrencyUnit& currency, + UNumberUnitWidth unitWidth, const PluralRules* rules, UErrorCode& status); + + /** + * Sets attributes of the current number being processed. + * + * @param signum + * -1 if negative; +1 if positive; or 0 if zero. + * @param plural + * The plural form of the number, required only if the pattern contains the triple + * currency sign, "¤¤¤" (and as indicated by {@link #needsPlurals()}). + */ + void setNumberProperties(Signum signum, StandardPlural::Form plural); + + /** + * Returns true if the pattern represented by this MurkyModifier requires a plural keyword in order to localize. + * This is currently true only if there is a currency long name placeholder in the pattern ("¤¤¤"). + */ + bool needsPlurals() const; + + /** Creates a quantity-dependent Modifier for the specified plural form. */ + AdoptingSignumModifierStore createImmutableForPlural(StandardPlural::Form plural, UErrorCode& status); + + /** + * Creates a new quantity-dependent Modifier that behaves the same as the current instance, but which is immutable + * and can be saved for future use. The number properties in the current instance are mutated; all other properties + * are left untouched. + * + *

    + * The resulting modifier cannot be used in a QuantityChain. + * + *

    + * CREATES A NEW HEAP OBJECT; THE CALLER GETS OWNERSHIP. + * + * @return An immutable that supports both positive and negative numbers. + */ + ImmutablePatternModifier *createImmutable(UErrorCode &status); + + MicroPropsGenerator &addToChain(const MicroPropsGenerator *parent); + + void processQuantity(DecimalQuantity &, MicroProps µs, UErrorCode &status) const override; + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + int32_t getPrefixLength() const override; + + int32_t getCodePointCount() const override; + + bool isStrong() const override; + + bool containsField(Field field) const override; + + void getParameters(Parameters& output) const override; + + bool semanticallyEquivalent(const Modifier& other) const override; + + /** + * Returns the string that substitutes a given symbol type in a pattern. + */ + UnicodeString getSymbol(AffixPatternType type) const override; + + /** + * Returns the currency symbol for the unit width specified in setSymbols() + */ + UnicodeString getCurrencySymbolForUnitWidth(UErrorCode& status) const; + + UnicodeString toUnicodeString() const; + + private: + // Modifier details (initialized in constructor) + const bool fStrong; + + // Pattern details (initialized in setPatternInfo and setPatternAttributes) + const AffixPatternProvider *fPatternInfo; + Field fField; + UNumberSignDisplay fSignDisplay; + bool fPerMilleReplacesPercent; + bool fApproximately; + + // Symbol details (initialized in setSymbols) + const DecimalFormatSymbols *fSymbols; + UNumberUnitWidth fUnitWidth; + CurrencySymbols fCurrencySymbols; + const PluralRules *fRules; + + // Number details (initialized in setNumberProperties) + Signum fSignum; + StandardPlural::Form fPlural; + + // QuantityChain details (initialized in addToChain) + const MicroPropsGenerator *fParent; + + // Transient fields for rendering + UnicodeString currentAffix; + + /** + * Uses the current properties to create a single {@link ConstantMultiFieldModifier} with currency spacing support + * if required. + * + *

    + * CREATES A NEW HEAP OBJECT; THE CALLER GETS OWNERSHIP. + * + * @param a + * A working FormattedStringBuilder object; passed from the outside to prevent the need to create many new + * instances if this method is called in a loop. + * @param b + * Another working FormattedStringBuilder object. + * @return The constant modifier object. + */ + ConstantMultiFieldModifier *createConstantModifier(UErrorCode &status); + + int32_t insertPrefix(FormattedStringBuilder &sb, int position, UErrorCode &status); + + int32_t insertSuffix(FormattedStringBuilder &sb, int position, UErrorCode &status); + + void prepareAffix(bool isPrefix); +}; + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_PATTERNMODIFIER_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_patternstring.cpp b/intl/icu/source/i18n/number_patternstring.cpp new file mode 100644 index 0000000000..aa082be5a8 --- /dev/null +++ b/intl/icu/source/i18n/number_patternstring.cpp @@ -0,0 +1,1212 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT +#define UNISTR_FROM_CHAR_EXPLICIT + +#include "uassert.h" +#include "number_patternstring.h" +#include "unicode/utf16.h" +#include "number_utils.h" +#include "number_roundingutils.h" +#include "number_mapper.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +void PatternParser::parseToPatternInfo(const UnicodeString& patternString, ParsedPatternInfo& patternInfo, + UErrorCode& status) { + patternInfo.consumePattern(patternString, status); +} + +DecimalFormatProperties +PatternParser::parseToProperties(const UnicodeString& pattern, IgnoreRounding ignoreRounding, + UErrorCode& status) { + DecimalFormatProperties properties; + parseToExistingPropertiesImpl(pattern, properties, ignoreRounding, status); + return properties; +} + +DecimalFormatProperties PatternParser::parseToProperties(const UnicodeString& pattern, + UErrorCode& status) { + return parseToProperties(pattern, IGNORE_ROUNDING_NEVER, status); +} + +void +PatternParser::parseToExistingProperties(const UnicodeString& pattern, DecimalFormatProperties& properties, + IgnoreRounding ignoreRounding, UErrorCode& status) { + parseToExistingPropertiesImpl(pattern, properties, ignoreRounding, status); +} + + +char16_t ParsedPatternInfo::charAt(int32_t flags, int32_t index) const { + const Endpoints& endpoints = getEndpoints(flags); + if (index < 0 || index >= endpoints.end - endpoints.start) { + UPRV_UNREACHABLE_EXIT; + } + return pattern.charAt(endpoints.start + index); +} + +int32_t ParsedPatternInfo::length(int32_t flags) const { + return getLengthFromEndpoints(getEndpoints(flags)); +} + +int32_t ParsedPatternInfo::getLengthFromEndpoints(const Endpoints& endpoints) { + return endpoints.end - endpoints.start; +} + +UnicodeString ParsedPatternInfo::getString(int32_t flags) const { + const Endpoints& endpoints = getEndpoints(flags); + if (endpoints.start == endpoints.end) { + return UnicodeString(); + } + // Create a new UnicodeString + return UnicodeString(pattern, endpoints.start, endpoints.end - endpoints.start); +} + +const Endpoints& ParsedPatternInfo::getEndpoints(int32_t flags) const { + bool prefix = (flags & AFFIX_PREFIX) != 0; + bool isNegative = (flags & AFFIX_NEGATIVE_SUBPATTERN) != 0; + bool padding = (flags & AFFIX_PADDING) != 0; + if (isNegative && padding) { + return negative.paddingEndpoints; + } else if (padding) { + return positive.paddingEndpoints; + } else if (prefix && isNegative) { + return negative.prefixEndpoints; + } else if (prefix) { + return positive.prefixEndpoints; + } else if (isNegative) { + return negative.suffixEndpoints; + } else { + return positive.suffixEndpoints; + } +} + +bool ParsedPatternInfo::positiveHasPlusSign() const { + return positive.hasPlusSign; +} + +bool ParsedPatternInfo::hasNegativeSubpattern() const { + return fHasNegativeSubpattern; +} + +bool ParsedPatternInfo::negativeHasMinusSign() const { + return negative.hasMinusSign; +} + +bool ParsedPatternInfo::hasCurrencySign() const { + return positive.hasCurrencySign || (fHasNegativeSubpattern && negative.hasCurrencySign); +} + +bool ParsedPatternInfo::containsSymbolType(AffixPatternType type, UErrorCode& status) const { + return AffixUtils::containsType(pattern, type, status); +} + +bool ParsedPatternInfo::hasBody() const { + return positive.integerTotal > 0; +} + +bool ParsedPatternInfo::currencyAsDecimal() const { + return positive.hasCurrencyDecimal; +} + +///////////////////////////////////////////////////// +/// BEGIN RECURSIVE DESCENT PARSER IMPLEMENTATION /// +///////////////////////////////////////////////////// + +UChar32 ParsedPatternInfo::ParserState::peek() { + if (offset == pattern.length()) { + return -1; + } else { + return pattern.char32At(offset); + } +} + +UChar32 ParsedPatternInfo::ParserState::peek2() { + if (offset == pattern.length()) { + return -1; + } + int32_t cp1 = pattern.char32At(offset); + int32_t offset2 = offset + U16_LENGTH(cp1); + if (offset2 == pattern.length()) { + return -1; + } + return pattern.char32At(offset2); +} + +UChar32 ParsedPatternInfo::ParserState::next() { + int32_t codePoint = peek(); + offset += U16_LENGTH(codePoint); + return codePoint; +} + +void ParsedPatternInfo::consumePattern(const UnicodeString& patternString, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + this->pattern = patternString; + + // This class is not intended for writing twice! + // Use move assignment to overwrite instead. + U_ASSERT(state.offset == 0); + + // pattern := subpattern (';' subpattern)? + currentSubpattern = &positive; + consumeSubpattern(status); + if (U_FAILURE(status)) { return; } + if (state.peek() == u';') { + state.next(); // consume the ';' + // Don't consume the negative subpattern if it is empty (trailing ';') + if (state.peek() != -1) { + fHasNegativeSubpattern = true; + currentSubpattern = &negative; + consumeSubpattern(status); + if (U_FAILURE(status)) { return; } + } + } + if (state.peek() != -1) { + state.toParseException(u"Found unquoted special character"); + status = U_UNQUOTED_SPECIAL; + } +} + +void ParsedPatternInfo::consumeSubpattern(UErrorCode& status) { + // subpattern := literals? number exponent? literals? + consumePadding(PadPosition::UNUM_PAD_BEFORE_PREFIX, status); + if (U_FAILURE(status)) { return; } + consumeAffix(currentSubpattern->prefixEndpoints, status); + if (U_FAILURE(status)) { return; } + consumePadding(PadPosition::UNUM_PAD_AFTER_PREFIX, status); + if (U_FAILURE(status)) { return; } + consumeFormat(status); + if (U_FAILURE(status)) { return; } + consumeExponent(status); + if (U_FAILURE(status)) { return; } + consumePadding(PadPosition::UNUM_PAD_BEFORE_SUFFIX, status); + if (U_FAILURE(status)) { return; } + consumeAffix(currentSubpattern->suffixEndpoints, status); + if (U_FAILURE(status)) { return; } + consumePadding(PadPosition::UNUM_PAD_AFTER_SUFFIX, status); + if (U_FAILURE(status)) { return; } +} + +void ParsedPatternInfo::consumePadding(PadPosition paddingLocation, UErrorCode& status) { + if (state.peek() != u'*') { + return; + } + if (currentSubpattern->hasPadding) { + state.toParseException(u"Cannot have multiple pad specifiers"); + status = U_MULTIPLE_PAD_SPECIFIERS; + return; + } + currentSubpattern->paddingLocation = paddingLocation; + currentSubpattern->hasPadding = true; + state.next(); // consume the '*' + currentSubpattern->paddingEndpoints.start = state.offset; + consumeLiteral(status); + currentSubpattern->paddingEndpoints.end = state.offset; +} + +void ParsedPatternInfo::consumeAffix(Endpoints& endpoints, UErrorCode& status) { + // literals := { literal } + endpoints.start = state.offset; + while (true) { + switch (state.peek()) { + case u'#': + case u'@': + case u';': + case u'*': + case u'.': + case u',': + case u'0': + case u'1': + case u'2': + case u'3': + case u'4': + case u'5': + case u'6': + case u'7': + case u'8': + case u'9': + case -1: + // Characters that cannot appear unquoted in a literal + // break outer; + goto after_outer; + + case u'%': + currentSubpattern->hasPercentSign = true; + break; + + case u'‰': + currentSubpattern->hasPerMilleSign = true; + break; + + case u'¤': + currentSubpattern->hasCurrencySign = true; + break; + + case u'-': + currentSubpattern->hasMinusSign = true; + break; + + case u'+': + currentSubpattern->hasPlusSign = true; + break; + + default: + break; + } + consumeLiteral(status); + if (U_FAILURE(status)) { return; } + } + after_outer: + endpoints.end = state.offset; +} + +void ParsedPatternInfo::consumeLiteral(UErrorCode& status) { + if (state.peek() == -1) { + state.toParseException(u"Expected unquoted literal but found EOL"); + status = U_PATTERN_SYNTAX_ERROR; + return; + } else if (state.peek() == u'\'') { + state.next(); // consume the starting quote + while (state.peek() != u'\'') { + if (state.peek() == -1) { + state.toParseException(u"Expected quoted literal but found EOL"); + status = U_PATTERN_SYNTAX_ERROR; + return; + } else { + state.next(); // consume a quoted character + } + } + state.next(); // consume the ending quote + } else { + // consume a non-quoted literal character + state.next(); + } +} + +void ParsedPatternInfo::consumeFormat(UErrorCode& status) { + consumeIntegerFormat(status); + if (U_FAILURE(status)) { return; } + if (state.peek() == u'.') { + state.next(); // consume the decimal point + currentSubpattern->hasDecimal = true; + currentSubpattern->widthExceptAffixes += 1; + consumeFractionFormat(status); + if (U_FAILURE(status)) { return; } + } else if (state.peek() == u'¤') { + // Check if currency is a decimal separator + switch (state.peek2()) { + case u'#': + case u'0': + case u'1': + case u'2': + case u'3': + case u'4': + case u'5': + case u'6': + case u'7': + case u'8': + case u'9': + break; + default: + // Currency symbol followed by a non-numeric character; + // treat as a normal affix. + return; + } + // Currency symbol is followed by a numeric character; + // treat as a decimal separator. + currentSubpattern->hasCurrencySign = true; + currentSubpattern->hasCurrencyDecimal = true; + currentSubpattern->hasDecimal = true; + currentSubpattern->widthExceptAffixes += 1; + state.next(); // consume the symbol + consumeFractionFormat(status); + if (U_FAILURE(status)) { return; } + } +} + +void ParsedPatternInfo::consumeIntegerFormat(UErrorCode& status) { + // Convenience reference: + ParsedSubpatternInfo& result = *currentSubpattern; + + while (true) { + switch (state.peek()) { + case u',': + result.widthExceptAffixes += 1; + result.groupingSizes <<= 16; + break; + + case u'#': + if (result.integerNumerals > 0) { + state.toParseException(u"# cannot follow 0 before decimal point"); + status = U_UNEXPECTED_TOKEN; + return; + } + result.widthExceptAffixes += 1; + result.groupingSizes += 1; + if (result.integerAtSigns > 0) { + result.integerTrailingHashSigns += 1; + } else { + result.integerLeadingHashSigns += 1; + } + result.integerTotal += 1; + break; + + case u'@': + if (result.integerNumerals > 0) { + state.toParseException(u"Cannot mix 0 and @"); + status = U_UNEXPECTED_TOKEN; + return; + } + if (result.integerTrailingHashSigns > 0) { + state.toParseException(u"Cannot nest # inside of a run of @"); + status = U_UNEXPECTED_TOKEN; + return; + } + result.widthExceptAffixes += 1; + result.groupingSizes += 1; + result.integerAtSigns += 1; + result.integerTotal += 1; + break; + + case u'0': + case u'1': + case u'2': + case u'3': + case u'4': + case u'5': + case u'6': + case u'7': + case u'8': + case u'9': + if (result.integerAtSigns > 0) { + state.toParseException(u"Cannot mix @ and 0"); + status = U_UNEXPECTED_TOKEN; + return; + } + result.widthExceptAffixes += 1; + result.groupingSizes += 1; + result.integerNumerals += 1; + result.integerTotal += 1; + if (!result.rounding.isZeroish() || state.peek() != u'0') { + result.rounding.appendDigit(static_cast(state.peek() - u'0'), 0, true); + } + break; + + default: + goto after_outer; + } + state.next(); // consume the symbol + } + + after_outer: + // Disallow patterns with a trailing ',' or with two ',' next to each other + auto grouping1 = static_cast (result.groupingSizes & 0xffff); + auto grouping2 = static_cast ((result.groupingSizes >> 16) & 0xffff); + auto grouping3 = static_cast ((result.groupingSizes >> 32) & 0xffff); + if (grouping1 == 0 && grouping2 != -1) { + state.toParseException(u"Trailing grouping separator is invalid"); + status = U_UNEXPECTED_TOKEN; + return; + } + if (grouping2 == 0 && grouping3 != -1) { + state.toParseException(u"Grouping width of zero is invalid"); + status = U_PATTERN_SYNTAX_ERROR; + return; + } +} + +void ParsedPatternInfo::consumeFractionFormat(UErrorCode& status) { + // Convenience reference: + ParsedSubpatternInfo& result = *currentSubpattern; + + int32_t zeroCounter = 0; + while (true) { + switch (state.peek()) { + case u'#': + result.widthExceptAffixes += 1; + result.fractionHashSigns += 1; + result.fractionTotal += 1; + zeroCounter++; + break; + + case u'0': + case u'1': + case u'2': + case u'3': + case u'4': + case u'5': + case u'6': + case u'7': + case u'8': + case u'9': + if (result.fractionHashSigns > 0) { + state.toParseException(u"0 cannot follow # after decimal point"); + status = U_UNEXPECTED_TOKEN; + return; + } + result.widthExceptAffixes += 1; + result.fractionNumerals += 1; + result.fractionTotal += 1; + if (state.peek() == u'0') { + zeroCounter++; + } else { + result.rounding + .appendDigit(static_cast(state.peek() - u'0'), zeroCounter, false); + zeroCounter = 0; + } + break; + + default: + return; + } + state.next(); // consume the symbol + } +} + +void ParsedPatternInfo::consumeExponent(UErrorCode& status) { + // Convenience reference: + ParsedSubpatternInfo& result = *currentSubpattern; + + if (state.peek() != u'E') { + return; + } + if ((result.groupingSizes & 0xffff0000L) != 0xffff0000L) { + state.toParseException(u"Cannot have grouping separator in scientific notation"); + status = U_MALFORMED_EXPONENTIAL_PATTERN; + return; + } + state.next(); // consume the E + result.widthExceptAffixes++; + if (state.peek() == u'+') { + state.next(); // consume the + + result.exponentHasPlusSign = true; + result.widthExceptAffixes++; + } + while (state.peek() == u'0') { + state.next(); // consume the 0 + result.exponentZeros += 1; + result.widthExceptAffixes++; + } +} + +/////////////////////////////////////////////////// +/// END RECURSIVE DESCENT PARSER IMPLEMENTATION /// +/////////////////////////////////////////////////// + +void PatternParser::parseToExistingPropertiesImpl(const UnicodeString& pattern, + DecimalFormatProperties& properties, + IgnoreRounding ignoreRounding, UErrorCode& status) { + if (pattern.length() == 0) { + // Backwards compatibility requires that we reset to the default values. + // TODO: Only overwrite the properties that "saveToProperties" normally touches? + properties.clear(); + return; + } + + ParsedPatternInfo patternInfo; + parseToPatternInfo(pattern, patternInfo, status); + if (U_FAILURE(status)) { return; } + patternInfoToProperties(properties, patternInfo, ignoreRounding, status); +} + +void +PatternParser::patternInfoToProperties(DecimalFormatProperties& properties, ParsedPatternInfo& patternInfo, + IgnoreRounding _ignoreRounding, UErrorCode& status) { + // Translate from PatternParseResult to Properties. + // Note that most data from "negative" is ignored per the specification of DecimalFormat. + + const ParsedSubpatternInfo& positive = patternInfo.positive; + + bool ignoreRounding; + if (_ignoreRounding == IGNORE_ROUNDING_NEVER) { + ignoreRounding = false; + } else if (_ignoreRounding == IGNORE_ROUNDING_IF_CURRENCY) { + ignoreRounding = positive.hasCurrencySign; + } else { + U_ASSERT(_ignoreRounding == IGNORE_ROUNDING_ALWAYS); + ignoreRounding = true; + } + + // Grouping settings + auto grouping1 = static_cast (positive.groupingSizes & 0xffff); + auto grouping2 = static_cast ((positive.groupingSizes >> 16) & 0xffff); + auto grouping3 = static_cast ((positive.groupingSizes >> 32) & 0xffff); + if (grouping2 != -1) { + properties.groupingSize = grouping1; + properties.groupingUsed = true; + } else { + properties.groupingSize = -1; + properties.groupingUsed = false; + } + if (grouping3 != -1) { + properties.secondaryGroupingSize = grouping2; + } else { + properties.secondaryGroupingSize = -1; + } + + // For backwards compatibility, require that the pattern emit at least one min digit. + int minInt, minFrac; + if (positive.integerTotal == 0 && positive.fractionTotal > 0) { + // patterns like ".##" + minInt = 0; + minFrac = uprv_max(1, positive.fractionNumerals); + } else if (positive.integerNumerals == 0 && positive.fractionNumerals == 0) { + // patterns like "#.##" + minInt = 1; + minFrac = 0; + } else { + minInt = positive.integerNumerals; + minFrac = positive.fractionNumerals; + } + + // Rounding settings + // Don't set basic rounding when there is a currency sign; defer to CurrencyUsage + if (positive.integerAtSigns > 0) { + properties.minimumFractionDigits = -1; + properties.maximumFractionDigits = -1; + properties.roundingIncrement = 0.0; + properties.minimumSignificantDigits = positive.integerAtSigns; + properties.maximumSignificantDigits = positive.integerAtSigns + positive.integerTrailingHashSigns; + } else if (!positive.rounding.isZeroish()) { + if (!ignoreRounding) { + properties.minimumFractionDigits = minFrac; + properties.maximumFractionDigits = positive.fractionTotal; + properties.roundingIncrement = positive.rounding.toDouble(); + } else { + properties.minimumFractionDigits = -1; + properties.maximumFractionDigits = -1; + properties.roundingIncrement = 0.0; + } + properties.minimumSignificantDigits = -1; + properties.maximumSignificantDigits = -1; + } else { + if (!ignoreRounding) { + properties.minimumFractionDigits = minFrac; + properties.maximumFractionDigits = positive.fractionTotal; + properties.roundingIncrement = 0.0; + } else { + properties.minimumFractionDigits = -1; + properties.maximumFractionDigits = -1; + properties.roundingIncrement = 0.0; + } + properties.minimumSignificantDigits = -1; + properties.maximumSignificantDigits = -1; + } + + // If the pattern ends with a '.' then force the decimal point. + if (positive.hasDecimal && positive.fractionTotal == 0) { + properties.decimalSeparatorAlwaysShown = true; + } else { + properties.decimalSeparatorAlwaysShown = false; + } + + // Persist the currency as decimal separator + properties.currencyAsDecimal = positive.hasCurrencyDecimal; + + // Scientific notation settings + if (positive.exponentZeros > 0) { + properties.exponentSignAlwaysShown = positive.exponentHasPlusSign; + properties.minimumExponentDigits = positive.exponentZeros; + if (positive.integerAtSigns == 0) { + // patterns without '@' can define max integer digits, used for engineering notation + properties.minimumIntegerDigits = positive.integerNumerals; + properties.maximumIntegerDigits = positive.integerTotal; + } else { + // patterns with '@' cannot define max integer digits + properties.minimumIntegerDigits = 1; + properties.maximumIntegerDigits = -1; + } + } else { + properties.exponentSignAlwaysShown = false; + properties.minimumExponentDigits = -1; + properties.minimumIntegerDigits = minInt; + properties.maximumIntegerDigits = -1; + } + + // Compute the affix patterns (required for both padding and affixes) + UnicodeString posPrefix = patternInfo.getString(AffixPatternProvider::AFFIX_PREFIX); + UnicodeString posSuffix = patternInfo.getString(0); + + // Padding settings + if (positive.hasPadding) { + // The width of the positive prefix and suffix templates are included in the padding + int paddingWidth = positive.widthExceptAffixes + + AffixUtils::estimateLength(posPrefix, status) + + AffixUtils::estimateLength(posSuffix, status); + properties.formatWidth = paddingWidth; + UnicodeString rawPaddingString = patternInfo.getString(AffixPatternProvider::AFFIX_PADDING); + if (rawPaddingString.length() == 1) { + properties.padString = rawPaddingString; + } else if (rawPaddingString.length() == 2) { + if (rawPaddingString.charAt(0) == u'\'') { + properties.padString.setTo(u"'", -1); + } else { + properties.padString = rawPaddingString; + } + } else { + properties.padString = UnicodeString(rawPaddingString, 1, rawPaddingString.length() - 2); + } + properties.padPosition = positive.paddingLocation; + } else { + properties.formatWidth = -1; + properties.padString.setToBogus(); + properties.padPosition.nullify(); + } + + // Set the affixes + // Always call the setter, even if the prefixes are empty, especially in the case of the + // negative prefix pattern, to prevent default values from overriding the pattern. + properties.positivePrefixPattern = posPrefix; + properties.positiveSuffixPattern = posSuffix; + if (patternInfo.fHasNegativeSubpattern) { + properties.negativePrefixPattern = patternInfo.getString( + AffixPatternProvider::AFFIX_NEGATIVE_SUBPATTERN | AffixPatternProvider::AFFIX_PREFIX); + properties.negativeSuffixPattern = patternInfo.getString( + AffixPatternProvider::AFFIX_NEGATIVE_SUBPATTERN); + } else { + properties.negativePrefixPattern.setToBogus(); + properties.negativeSuffixPattern.setToBogus(); + } + + // Set the magnitude multiplier + if (positive.hasPercentSign) { + properties.magnitudeMultiplier = 2; + } else if (positive.hasPerMilleSign) { + properties.magnitudeMultiplier = 3; + } else { + properties.magnitudeMultiplier = 0; + } +} + +/////////////////////////////////////////////////////////////////// +/// End PatternStringParser.java; begin PatternStringUtils.java /// +/////////////////////////////////////////////////////////////////// + +// Determine whether a given roundingIncrement should be ignored for formatting +// based on the current maxFrac value (maximum fraction digits). For example a +// roundingIncrement of 0.01 should be ignored if maxFrac is 1, but not if maxFrac +// is 2 or more. Note that roundingIncrements are rounded in significance, so +// a roundingIncrement of 0.006 is treated like 0.01 for this determination, i.e. +// it should not be ignored if maxFrac is 2 or more (but a roundingIncrement of +// 0.005 is treated like 0.001 for significance). This is the reason for the +// initial doubling below. +// roundIncr must be non-zero. +bool PatternStringUtils::ignoreRoundingIncrement(double roundIncr, int32_t maxFrac) { + if (maxFrac < 0) { + return false; + } + int32_t frac = 0; + roundIncr *= 2.0; + for (frac = 0; frac <= maxFrac && roundIncr <= 1.0; frac++, roundIncr *= 10.0); + return (frac > maxFrac); +} + +UnicodeString PatternStringUtils::propertiesToPatternString(const DecimalFormatProperties& properties, + UErrorCode& status) { + UnicodeString sb; + + // Convenience references + // The uprv_min() calls prevent DoS + int32_t dosMax = 100; + int32_t grouping1 = uprv_max(0, uprv_min(properties.groupingSize, dosMax)); + int32_t grouping2 = uprv_max(0, uprv_min(properties.secondaryGroupingSize, dosMax)); + bool useGrouping = properties.groupingUsed; + int32_t paddingWidth = uprv_min(properties.formatWidth, dosMax); + NullableValue paddingLocation = properties.padPosition; + UnicodeString paddingString = properties.padString; + int32_t minInt = uprv_max(0, uprv_min(properties.minimumIntegerDigits, dosMax)); + int32_t maxInt = uprv_min(properties.maximumIntegerDigits, dosMax); + int32_t minFrac = uprv_max(0, uprv_min(properties.minimumFractionDigits, dosMax)); + int32_t maxFrac = uprv_min(properties.maximumFractionDigits, dosMax); + int32_t minSig = uprv_min(properties.minimumSignificantDigits, dosMax); + int32_t maxSig = uprv_min(properties.maximumSignificantDigits, dosMax); + bool alwaysShowDecimal = properties.decimalSeparatorAlwaysShown; + int32_t exponentDigits = uprv_min(properties.minimumExponentDigits, dosMax); + bool exponentShowPlusSign = properties.exponentSignAlwaysShown; + + AutoAffixPatternProvider affixProvider(properties, status); + + // Prefixes + sb.append(affixProvider.get().getString(AffixPatternProvider::AFFIX_POS_PREFIX)); + int32_t afterPrefixPos = sb.length(); + + // Figure out the grouping sizes. + if (!useGrouping) { + grouping1 = 0; + grouping2 = 0; + } else if (grouping1 == grouping2) { + grouping1 = 0; + } + int32_t groupingLength = grouping1 + grouping2 + 1; + + // Figure out the digits we need to put in the pattern. + double increment = properties.roundingIncrement; + UnicodeString digitsString; + int32_t digitsStringScale = 0; + if (maxSig != uprv_min(dosMax, -1)) { + // Significant Digits. + while (digitsString.length() < minSig) { + digitsString.append(u'@'); + } + while (digitsString.length() < maxSig) { + digitsString.append(u'#'); + } + } else if (increment != 0.0 && !ignoreRoundingIncrement(increment,maxFrac)) { + // Rounding Increment. + DecimalQuantity incrementQuantity; + incrementQuantity.setToDouble(increment); + incrementQuantity.roundToInfinity(); + digitsStringScale = incrementQuantity.getLowerDisplayMagnitude(); + incrementQuantity.adjustMagnitude(-digitsStringScale); + incrementQuantity.setMinInteger(minInt - digitsStringScale); + UnicodeString str = incrementQuantity.toPlainString(); + if (str.charAt(0) == u'-') { + // TODO: Unsupported operation exception or fail silently? + digitsString.append(str, 1, str.length() - 1); + } else { + digitsString.append(str); + } + } + while (digitsString.length() + digitsStringScale < minInt) { + digitsString.insert(0, u'0'); + } + while (-digitsStringScale < minFrac) { + digitsString.append(u'0'); + digitsStringScale--; + } + + // Write the digits to the string builder + int32_t m0 = uprv_max(groupingLength, digitsString.length() + digitsStringScale); + m0 = (maxInt != dosMax) ? uprv_max(maxInt, m0) - 1 : m0 - 1; + int32_t mN = (maxFrac != dosMax) ? uprv_min(-maxFrac, digitsStringScale) : digitsStringScale; + for (int32_t magnitude = m0; magnitude >= mN; magnitude--) { + int32_t di = digitsString.length() + digitsStringScale - magnitude - 1; + if (di < 0 || di >= digitsString.length()) { + sb.append(u'#'); + } else { + sb.append(digitsString.charAt(di)); + } + // Decimal separator + if (magnitude == 0 && (alwaysShowDecimal || mN < 0)) { + if (properties.currencyAsDecimal) { + sb.append(u'¤'); + } else { + sb.append(u'.'); + } + } + if (!useGrouping) { + continue; + } + // Least-significant grouping separator + if (magnitude > 0 && magnitude == grouping1) { + sb.append(u','); + } + // All other grouping separators + if (magnitude > grouping1 && grouping2 > 0 && (magnitude - grouping1) % grouping2 == 0) { + sb.append(u','); + } + } + + // Exponential notation + if (exponentDigits != uprv_min(dosMax, -1)) { + sb.append(u'E'); + if (exponentShowPlusSign) { + sb.append(u'+'); + } + for (int32_t i = 0; i < exponentDigits; i++) { + sb.append(u'0'); + } + } + + // Suffixes + int32_t beforeSuffixPos = sb.length(); + sb.append(affixProvider.get().getString(AffixPatternProvider::AFFIX_POS_SUFFIX)); + + // Resolve Padding + if (paddingWidth > 0 && !paddingLocation.isNull()) { + while (paddingWidth - sb.length() > 0) { + sb.insert(afterPrefixPos, u'#'); + beforeSuffixPos++; + } + int32_t addedLength; + switch (paddingLocation.get(status)) { + case PadPosition::UNUM_PAD_BEFORE_PREFIX: + addedLength = escapePaddingString(paddingString, sb, 0, status); + sb.insert(0, u'*'); + afterPrefixPos += addedLength + 1; + beforeSuffixPos += addedLength + 1; + break; + case PadPosition::UNUM_PAD_AFTER_PREFIX: + addedLength = escapePaddingString(paddingString, sb, afterPrefixPos, status); + sb.insert(afterPrefixPos, u'*'); + afterPrefixPos += addedLength + 1; + beforeSuffixPos += addedLength + 1; + break; + case PadPosition::UNUM_PAD_BEFORE_SUFFIX: + escapePaddingString(paddingString, sb, beforeSuffixPos, status); + sb.insert(beforeSuffixPos, u'*'); + break; + case PadPosition::UNUM_PAD_AFTER_SUFFIX: + sb.append(u'*'); + escapePaddingString(paddingString, sb, sb.length(), status); + break; + } + if (U_FAILURE(status)) { return sb; } + } + + // Negative affixes + // Ignore if the negative prefix pattern is "-" and the negative suffix is empty + if (affixProvider.get().hasNegativeSubpattern()) { + sb.append(u';'); + sb.append(affixProvider.get().getString(AffixPatternProvider::AFFIX_NEG_PREFIX)); + // Copy the positive digit format into the negative. + // This is optional; the pattern is the same as if '#' were appended here instead. + // NOTE: It is not safe to append the UnicodeString to itself, so we need to copy. + // See https://unicode-org.atlassian.net/browse/ICU-13707 + UnicodeString copy(sb); + sb.append(copy, afterPrefixPos, beforeSuffixPos - afterPrefixPos); + sb.append(affixProvider.get().getString(AffixPatternProvider::AFFIX_NEG_SUFFIX)); + } + + return sb; +} + +int PatternStringUtils::escapePaddingString(UnicodeString input, UnicodeString& output, int startIndex, + UErrorCode& status) { + (void) status; + if (input.length() == 0) { + input.setTo(kFallbackPaddingString, -1); + } + int startLength = output.length(); + if (input.length() == 1) { + if (input.compare(u"'", -1) == 0) { + output.insert(startIndex, u"''", -1); + } else { + output.insert(startIndex, input); + } + } else { + output.insert(startIndex, u'\''); + int offset = 1; + for (int i = 0; i < input.length(); i++) { + // it's okay to deal in chars here because the quote mark is the only interesting thing. + char16_t ch = input.charAt(i); + if (ch == u'\'') { + output.insert(startIndex + offset, u"''", -1); + offset += 2; + } else { + output.insert(startIndex + offset, ch); + offset += 1; + } + } + output.insert(startIndex + offset, u'\''); + } + return output.length() - startLength; +} + +UnicodeString +PatternStringUtils::convertLocalized(const UnicodeString& input, const DecimalFormatSymbols& symbols, + bool toLocalized, UErrorCode& status) { + // Construct a table of strings to be converted between localized and standard. + static constexpr int32_t LEN = 21; + UnicodeString table[LEN][2]; + int standIdx = toLocalized ? 0 : 1; + int localIdx = toLocalized ? 1 : 0; + // TODO: Add approximately sign here? + table[0][standIdx] = u"%"; + table[0][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kPercentSymbol); + table[1][standIdx] = u"‰"; + table[1][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kPerMillSymbol); + table[2][standIdx] = u"."; + table[2][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kDecimalSeparatorSymbol); + table[3][standIdx] = u","; + table[3][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kGroupingSeparatorSymbol); + table[4][standIdx] = u"-"; + table[4][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kMinusSignSymbol); + table[5][standIdx] = u"+"; + table[5][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kPlusSignSymbol); + table[6][standIdx] = u";"; + table[6][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kPatternSeparatorSymbol); + table[7][standIdx] = u"@"; + table[7][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kSignificantDigitSymbol); + table[8][standIdx] = u"E"; + table[8][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kExponentialSymbol); + table[9][standIdx] = u"*"; + table[9][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kPadEscapeSymbol); + table[10][standIdx] = u"#"; + table[10][localIdx] = symbols.getConstSymbol(DecimalFormatSymbols::kDigitSymbol); + for (int i = 0; i < 10; i++) { + table[11 + i][standIdx] = u'0' + i; + table[11 + i][localIdx] = symbols.getConstDigitSymbol(i); + } + + // Special case: quotes are NOT allowed to be in any localIdx strings. + // Substitute them with '’' instead. + for (int32_t i = 0; i < LEN; i++) { + table[i][localIdx].findAndReplace(u'\'', u'’'); + } + + // Iterate through the string and convert. + // State table: + // 0 => base state + // 1 => first char inside a quoted sequence in input and output string + // 2 => inside a quoted sequence in input and output string + // 3 => first char after a close quote in input string; + // close quote still needs to be written to output string + // 4 => base state in input string; inside quoted sequence in output string + // 5 => first char inside a quoted sequence in input string; + // inside quoted sequence in output string + UnicodeString result; + int state = 0; + for (int offset = 0; offset < input.length(); offset++) { + char16_t ch = input.charAt(offset); + + // Handle a quote character (state shift) + if (ch == u'\'') { + if (state == 0) { + result.append(u'\''); + state = 1; + continue; + } else if (state == 1) { + result.append(u'\''); + state = 0; + continue; + } else if (state == 2) { + state = 3; + continue; + } else if (state == 3) { + result.append(u'\''); + result.append(u'\''); + state = 1; + continue; + } else if (state == 4) { + state = 5; + continue; + } else { + U_ASSERT(state == 5); + result.append(u'\''); + result.append(u'\''); + state = 4; + continue; + } + } + + if (state == 0 || state == 3 || state == 4) { + for (auto& pair : table) { + // Perform a greedy match on this symbol string + UnicodeString temp = input.tempSubString(offset, pair[0].length()); + if (temp == pair[0]) { + // Skip ahead past this region for the next iteration + offset += pair[0].length() - 1; + if (state == 3 || state == 4) { + result.append(u'\''); + state = 0; + } + result.append(pair[1]); + goto continue_outer; + } + } + // No replacement found. Check if a special quote is necessary + for (auto& pair : table) { + UnicodeString temp = input.tempSubString(offset, pair[1].length()); + if (temp == pair[1]) { + if (state == 0) { + result.append(u'\''); + state = 4; + } + result.append(ch); + goto continue_outer; + } + } + // Still nothing. Copy the char verbatim. (Add a close quote if necessary) + if (state == 3 || state == 4) { + result.append(u'\''); + state = 0; + } + result.append(ch); + } else { + U_ASSERT(state == 1 || state == 2 || state == 5); + result.append(ch); + state = 2; + } + continue_outer:; + } + // Resolve final quotes + if (state == 3 || state == 4) { + result.append(u'\''); + state = 0; + } + if (state != 0) { + // Malformed localized pattern: unterminated quote + status = U_PATTERN_SYNTAX_ERROR; + } + return result; +} + +void PatternStringUtils::patternInfoToStringBuilder(const AffixPatternProvider& patternInfo, bool isPrefix, + PatternSignType patternSignType, + bool approximately, + StandardPlural::Form plural, + bool perMilleReplacesPercent, + bool dropCurrencySymbols, + UnicodeString& output) { + + // Should the output render '+' where '-' would normally appear in the pattern? + bool plusReplacesMinusSign = (patternSignType == PATTERN_SIGN_TYPE_POS_SIGN) + && !patternInfo.positiveHasPlusSign(); + + // Should we use the affix from the negative subpattern? + // (If not, we will use the positive subpattern.) + bool useNegativeAffixPattern = patternInfo.hasNegativeSubpattern() + && (patternSignType == PATTERN_SIGN_TYPE_NEG + || (patternInfo.negativeHasMinusSign() && (plusReplacesMinusSign || approximately))); + + // Resolve the flags for the affix pattern. + int flags = 0; + if (useNegativeAffixPattern) { + flags |= AffixPatternProvider::AFFIX_NEGATIVE_SUBPATTERN; + } + if (isPrefix) { + flags |= AffixPatternProvider::AFFIX_PREFIX; + } + if (plural != StandardPlural::Form::COUNT) { + U_ASSERT(plural == (AffixPatternProvider::AFFIX_PLURAL_MASK & plural)); + flags |= plural; + } + + // Should we prepend a sign to the pattern? + bool prependSign; + if (!isPrefix || useNegativeAffixPattern) { + prependSign = false; + } else if (patternSignType == PATTERN_SIGN_TYPE_NEG) { + prependSign = true; + } else { + prependSign = plusReplacesMinusSign || approximately; + } + + // What symbols should take the place of the sign placeholder? + const char16_t* signSymbols = u"-"; + if (approximately) { + if (plusReplacesMinusSign) { + signSymbols = u"~+"; + } else if (patternSignType == PATTERN_SIGN_TYPE_NEG) { + signSymbols = u"~-"; + } else { + signSymbols = u"~"; + } + } else if (plusReplacesMinusSign) { + signSymbols = u"+"; + } + + // Compute the number of tokens in the affix pattern (signSymbols is considered one token). + int length = patternInfo.length(flags) + (prependSign ? 1 : 0); + + // Finally, set the result into the StringBuilder. + output.remove(); + for (int index = 0; index < length; index++) { + char16_t candidate; + if (prependSign && index == 0) { + candidate = u'-'; + } else if (prependSign) { + candidate = patternInfo.charAt(flags, index - 1); + } else { + candidate = patternInfo.charAt(flags, index); + } + if (candidate == u'-') { + if (u_strlen(signSymbols) == 1) { + candidate = signSymbols[0]; + } else { + output.append(signSymbols[0]); + candidate = signSymbols[1]; + } + } + if (perMilleReplacesPercent && candidate == u'%') { + candidate = u'‰'; + } + if (dropCurrencySymbols && candidate == u'\u00A4') { + continue; + } + output.append(candidate); + } +} + +PatternSignType PatternStringUtils::resolveSignDisplay(UNumberSignDisplay signDisplay, Signum signum) { + switch (signDisplay) { + case UNUM_SIGN_AUTO: + case UNUM_SIGN_ACCOUNTING: + switch (signum) { + case SIGNUM_NEG: + case SIGNUM_NEG_ZERO: + return PATTERN_SIGN_TYPE_NEG; + case SIGNUM_POS_ZERO: + case SIGNUM_POS: + return PATTERN_SIGN_TYPE_POS; + default: + break; + } + break; + + case UNUM_SIGN_ALWAYS: + case UNUM_SIGN_ACCOUNTING_ALWAYS: + switch (signum) { + case SIGNUM_NEG: + case SIGNUM_NEG_ZERO: + return PATTERN_SIGN_TYPE_NEG; + case SIGNUM_POS_ZERO: + case SIGNUM_POS: + return PATTERN_SIGN_TYPE_POS_SIGN; + default: + break; + } + break; + + case UNUM_SIGN_EXCEPT_ZERO: + case UNUM_SIGN_ACCOUNTING_EXCEPT_ZERO: + switch (signum) { + case SIGNUM_NEG: + return PATTERN_SIGN_TYPE_NEG; + case SIGNUM_NEG_ZERO: + case SIGNUM_POS_ZERO: + return PATTERN_SIGN_TYPE_POS; + case SIGNUM_POS: + return PATTERN_SIGN_TYPE_POS_SIGN; + default: + break; + } + break; + + case UNUM_SIGN_NEGATIVE: + case UNUM_SIGN_ACCOUNTING_NEGATIVE: + switch (signum) { + case SIGNUM_NEG: + return PATTERN_SIGN_TYPE_NEG; + case SIGNUM_NEG_ZERO: + case SIGNUM_POS_ZERO: + case SIGNUM_POS: + return PATTERN_SIGN_TYPE_POS; + default: + break; + } + break; + + case UNUM_SIGN_NEVER: + return PATTERN_SIGN_TYPE_POS; + + default: + break; + } + + UPRV_UNREACHABLE_EXIT; + return PATTERN_SIGN_TYPE_POS; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_patternstring.h b/intl/icu/source/i18n/number_patternstring.h new file mode 100644 index 0000000000..989d0d3464 --- /dev/null +++ b/intl/icu/source/i18n/number_patternstring.h @@ -0,0 +1,340 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_PATTERNSTRING_H__ +#define __NUMBER_PATTERNSTRING_H__ + + +#include +#include "unicode/unum.h" +#include "unicode/unistr.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "number_decimfmtprops.h" +#include "number_affixutils.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +// Forward declaration +class PatternParser; + +// Note: the order of fields in this enum matters for parsing. +enum PatternSignType { + /** Render using normal positive subpattern rules */ + PATTERN_SIGN_TYPE_POS, + /** Render using rules to force the display of a plus sign */ + PATTERN_SIGN_TYPE_POS_SIGN, + /** Render using negative subpattern rules */ + PATTERN_SIGN_TYPE_NEG, + /** Count for looping over the possibilities */ + PATTERN_SIGN_TYPE_COUNT +}; + +// Exported as U_I18N_API because it is a public member field of exported ParsedSubpatternInfo +struct U_I18N_API Endpoints { + int32_t start = 0; + int32_t end = 0; +}; + +// Exported as U_I18N_API because it is a public member field of exported ParsedPatternInfo +struct U_I18N_API ParsedSubpatternInfo { + uint64_t groupingSizes = 0x0000ffffffff0000L; + int32_t integerLeadingHashSigns = 0; + int32_t integerTrailingHashSigns = 0; + int32_t integerNumerals = 0; + int32_t integerAtSigns = 0; + int32_t integerTotal = 0; // for convenience + int32_t fractionNumerals = 0; + int32_t fractionHashSigns = 0; + int32_t fractionTotal = 0; // for convenience + bool hasDecimal = false; + int32_t widthExceptAffixes = 0; + // Note: NullableValue causes issues here with std::move. + bool hasPadding = false; + UNumberFormatPadPosition paddingLocation = UNUM_PAD_BEFORE_PREFIX; + DecimalQuantity rounding; + bool exponentHasPlusSign = false; + int32_t exponentZeros = 0; + bool hasPercentSign = false; + bool hasPerMilleSign = false; + bool hasCurrencySign = false; + bool hasCurrencyDecimal = false; + bool hasMinusSign = false; + bool hasPlusSign = false; + + Endpoints prefixEndpoints; + Endpoints suffixEndpoints; + Endpoints paddingEndpoints; +}; + +// Exported as U_I18N_API because it is needed for the unit test PatternStringTest +struct U_I18N_API ParsedPatternInfo : public AffixPatternProvider, public UMemory { + UnicodeString pattern; + ParsedSubpatternInfo positive; + ParsedSubpatternInfo negative; + + ParsedPatternInfo() + : state(this->pattern), currentSubpattern(nullptr) {} + + ~ParsedPatternInfo() override = default; + + // Need to declare this explicitly because of the destructor + ParsedPatternInfo& operator=(ParsedPatternInfo&& src) noexcept = default; + + static int32_t getLengthFromEndpoints(const Endpoints& endpoints); + + char16_t charAt(int32_t flags, int32_t index) const override; + + int32_t length(int32_t flags) const override; + + UnicodeString getString(int32_t flags) const override; + + bool positiveHasPlusSign() const override; + + bool hasNegativeSubpattern() const override; + + bool negativeHasMinusSign() const override; + + bool hasCurrencySign() const override; + + bool containsSymbolType(AffixPatternType type, UErrorCode& status) const override; + + bool hasBody() const override; + + bool currencyAsDecimal() const override; + + private: + struct U_I18N_API ParserState { + const UnicodeString& pattern; // reference to the parent + int32_t offset = 0; + + explicit ParserState(const UnicodeString& _pattern) + : pattern(_pattern) {} + + ParserState& operator=(ParserState&& src) noexcept { + // Leave pattern reference alone; it will continue to point to the same place in memory, + // which gets overwritten by ParsedPatternInfo's implicit move assignment. + offset = src.offset; + return *this; + } + + /** Returns the next code point, or -1 if string is too short. */ + UChar32 peek(); + + /** Returns the code point after the next code point, or -1 if string is too short. */ + UChar32 peek2(); + + /** Returns the next code point and then steps forward. */ + UChar32 next(); + + // TODO: We don't currently do anything with the message string. + // This method is here as a shell for Java compatibility. + inline void toParseException(const char16_t* message) { (void) message; } + } state; + + // NOTE: In Java, these are written as pure functions. + // In C++, they're written as methods. + // The behavior is the same. + + // Mutable transient pointer: + ParsedSubpatternInfo* currentSubpattern; + + // In Java, "negative == null" tells us whether or not we had a negative subpattern. + // In C++, we need to remember in another boolean. + bool fHasNegativeSubpattern = false; + + const Endpoints& getEndpoints(int32_t flags) const; + + /** Run the recursive descent parser. */ + void consumePattern(const UnicodeString& patternString, UErrorCode& status); + + void consumeSubpattern(UErrorCode& status); + + void consumePadding(PadPosition paddingLocation, UErrorCode& status); + + void consumeAffix(Endpoints& endpoints, UErrorCode& status); + + void consumeLiteral(UErrorCode& status); + + void consumeFormat(UErrorCode& status); + + void consumeIntegerFormat(UErrorCode& status); + + void consumeFractionFormat(UErrorCode& status); + + void consumeExponent(UErrorCode& status); + + friend class PatternParser; +}; + +enum IgnoreRounding { + IGNORE_ROUNDING_NEVER = 0, IGNORE_ROUNDING_IF_CURRENCY = 1, IGNORE_ROUNDING_ALWAYS = 2 +}; + +class U_I18N_API PatternParser { + public: + /** + * Runs the recursive descent parser on the given pattern string, returning a data structure with raw information + * about the pattern string. + * + *

    + * To obtain a more useful form of the data, consider using {@link #parseToProperties} instead. + * + * TODO: Change argument type to const char16_t* instead of UnicodeString? + * + * @param patternString + * The LDML decimal format pattern (Excel-style pattern) to parse. + * @return The results of the parse. + */ + static void parseToPatternInfo(const UnicodeString& patternString, ParsedPatternInfo& patternInfo, + UErrorCode& status); + + /** + * Parses a pattern string into a new property bag. + * + * @param pattern + * The pattern string, like "#,##0.00" + * @param ignoreRounding + * Whether to leave out rounding information (minFrac, maxFrac, and rounding increment) when parsing the + * pattern. This may be desirable if a custom rounding mode, such as CurrencyUsage, is to be used + * instead. + * @return A property bag object. + * @throws IllegalArgumentException + * If there is a syntax error in the pattern string. + */ + static DecimalFormatProperties parseToProperties(const UnicodeString& pattern, + IgnoreRounding ignoreRounding, UErrorCode& status); + + static DecimalFormatProperties parseToProperties(const UnicodeString& pattern, UErrorCode& status); + + /** + * Parses a pattern string into an existing property bag. All properties that can be encoded into a pattern string + * will be overwritten with either their default value or with the value coming from the pattern string. Properties + * that cannot be encoded into a pattern string, such as rounding mode, are not modified. + * + * @param pattern + * The pattern string, like "#,##0.00" + * @param properties + * The property bag object to overwrite. + * @param ignoreRounding + * See {@link #parseToProperties(String pattern, int ignoreRounding)}. + * @throws IllegalArgumentException + * If there was a syntax error in the pattern string. + */ + static void parseToExistingProperties(const UnicodeString& pattern, + DecimalFormatProperties& properties, + IgnoreRounding ignoreRounding, UErrorCode& status); + + private: + static void parseToExistingPropertiesImpl(const UnicodeString& pattern, + DecimalFormatProperties& properties, + IgnoreRounding ignoreRounding, UErrorCode& status); + + /** Finalizes the temporary data stored in the ParsedPatternInfo to the Properties. */ + static void patternInfoToProperties(DecimalFormatProperties& properties, + ParsedPatternInfo& patternInfo, IgnoreRounding _ignoreRounding, + UErrorCode& status); +}; + +class U_I18N_API PatternStringUtils { + public: + /** + * Determine whether a given roundingIncrement should be ignored for formatting + * based on the current maxFrac value (maximum fraction digits). For example a + * roundingIncrement of 0.01 should be ignored if maxFrac is 1, but not if maxFrac + * is 2 or more. Note that roundingIncrements are rounded up in significance, so + * a roundingIncrement of 0.006 is treated like 0.01 for this determination, i.e. + * it should not be ignored if maxFrac is 2 or more (but a roundingIncrement of + * 0.005 is treated like 0.001 for significance). + * + * This test is needed for both NumberPropertyMapper::oldToNew and + * PatternStringUtils::propertiesToPatternString. In Java it cannot be + * exported by NumberPropertyMapper (package private) so it is in + * PatternStringUtils, do the same in C. + * + * @param roundIncr + * The roundingIncrement to be checked. Must be non-zero. + * @param maxFrac + * The current maximum fraction digits value. + * @return true if roundIncr should be ignored for formatting. + */ + static bool ignoreRoundingIncrement(double roundIncr, int32_t maxFrac); + + /** + * Creates a pattern string from a property bag. + * + *

    + * Since pattern strings support only a subset of the functionality available in a property bag, a new property bag + * created from the string returned by this function may not be the same as the original property bag. + * + * @param properties + * The property bag to serialize. + * @return A pattern string approximately serializing the property bag. + */ + static UnicodeString propertiesToPatternString(const DecimalFormatProperties& properties, + UErrorCode& status); + + + /** + * Converts a pattern between standard notation and localized notation. Localized notation means that instead of + * using generic placeholders in the pattern, you use the corresponding locale-specific characters instead. For + * example, in locale fr-FR, the period in the pattern "0.000" means "decimal" in standard notation (as it + * does in every other locale), but it means "grouping" in localized notation. + * + *

    + * A greedy string-substitution strategy is used to substitute locale symbols. If two symbols are ambiguous or have + * the same prefix, the result is not well-defined. + * + *

    + * Locale symbols are not allowed to contain the ASCII quote character. + * + *

    + * This method is provided for backwards compatibility and should not be used in any new code. + * + * TODO(C++): This method is not yet implemented. + * + * @param input + * The pattern to convert. + * @param symbols + * The symbols corresponding to the localized pattern. + * @param toLocalized + * true to convert from standard to localized notation; false to convert from localized to standard + * notation. + * @return The pattern expressed in the other notation. + */ + static UnicodeString convertLocalized(const UnicodeString& input, const DecimalFormatSymbols& symbols, + bool toLocalized, UErrorCode& status); + + /** + * This method contains the heart of the logic for rendering LDML affix strings. It handles + * sign-always-shown resolution, whether to use the positive or negative subpattern, permille + * substitution, and plural forms for CurrencyPluralInfo. + */ + static void patternInfoToStringBuilder(const AffixPatternProvider& patternInfo, bool isPrefix, + PatternSignType patternSignType, + bool approximately, + StandardPlural::Form plural, + bool perMilleReplacesPercent, + bool dropCurrencySymbols, + UnicodeString& output); + + static PatternSignType resolveSignDisplay(UNumberSignDisplay signDisplay, Signum signum); + + private: + /** @return The number of chars inserted. */ + static int escapePaddingString(UnicodeString input, UnicodeString& output, int startIndex, + UErrorCode& status); +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +#endif //__NUMBER_PATTERNSTRING_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_rounding.cpp b/intl/icu/source/i18n/number_rounding.cpp new file mode 100644 index 0000000000..e6bb509ffd --- /dev/null +++ b/intl/icu/source/i18n/number_rounding.cpp @@ -0,0 +1,552 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "charstr.h" +#include "uassert.h" +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "double-conversion.h" +#include "number_roundingutils.h" +#include "number_skeletons.h" +#include "number_decnum.h" +#include "putilimp.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +using double_conversion::DoubleToStringConverter; +using icu::StringSegment; + +void number::impl::parseIncrementOption(const StringSegment &segment, + Precision &outPrecision, + UErrorCode &status) { + // Need to do char <-> char16_t conversion... + U_ASSERT(U_SUCCESS(status)); + CharString buffer; + SKELETON_UCHAR_TO_CHAR(buffer, segment.toTempUnicodeString(), 0, segment.length(), status); + + // Utilize DecimalQuantity/decNumber to parse this for us. + DecimalQuantity dq; + UErrorCode localStatus = U_ZERO_ERROR; + dq.setToDecNumber({buffer.data(), buffer.length()}, localStatus); + if (U_FAILURE(localStatus) || dq.isNaN() || dq.isInfinite()) { + // throw new SkeletonSyntaxException("Invalid rounding increment", segment, e); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + // Now we break apart the number into a mantissa and exponent (magnitude). + int32_t magnitude = dq.adjustToZeroScale(); + // setToDecNumber drops trailing zeros, so we search for the '.' manually. + for (int32_t i=0; i= 0 && minMaxFractionPlaces <= kMaxIntFracSig) { + return constructFraction(minMaxFractionPlaces, minMaxFractionPlaces); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +FractionPrecision Precision::minFraction(int32_t minFractionPlaces) { + if (minFractionPlaces >= 0 && minFractionPlaces <= kMaxIntFracSig) { + return constructFraction(minFractionPlaces, -1); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +FractionPrecision Precision::maxFraction(int32_t maxFractionPlaces) { + if (maxFractionPlaces >= 0 && maxFractionPlaces <= kMaxIntFracSig) { + return constructFraction(0, maxFractionPlaces); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +FractionPrecision Precision::minMaxFraction(int32_t minFractionPlaces, int32_t maxFractionPlaces) { + if (minFractionPlaces >= 0 && maxFractionPlaces <= kMaxIntFracSig && + minFractionPlaces <= maxFractionPlaces) { + return constructFraction(minFractionPlaces, maxFractionPlaces); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision Precision::fixedSignificantDigits(int32_t minMaxSignificantDigits) { + if (minMaxSignificantDigits >= 1 && minMaxSignificantDigits <= kMaxIntFracSig) { + return constructSignificant(minMaxSignificantDigits, minMaxSignificantDigits); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision Precision::minSignificantDigits(int32_t minSignificantDigits) { + if (minSignificantDigits >= 1 && minSignificantDigits <= kMaxIntFracSig) { + return constructSignificant(minSignificantDigits, -1); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision Precision::maxSignificantDigits(int32_t maxSignificantDigits) { + if (maxSignificantDigits >= 1 && maxSignificantDigits <= kMaxIntFracSig) { + return constructSignificant(1, maxSignificantDigits); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision Precision::minMaxSignificantDigits(int32_t minSignificantDigits, int32_t maxSignificantDigits) { + if (minSignificantDigits >= 1 && maxSignificantDigits <= kMaxIntFracSig && + minSignificantDigits <= maxSignificantDigits) { + return constructSignificant(minSignificantDigits, maxSignificantDigits); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision Precision::trailingZeroDisplay(UNumberTrailingZeroDisplay trailingZeroDisplay) const { + Precision result(*this); // copy constructor + result.fTrailingZeroDisplay = trailingZeroDisplay; + return result; +} + +IncrementPrecision Precision::increment(double roundingIncrement) { + if (roundingIncrement > 0.0) { + DecimalQuantity dq; + dq.setToDouble(roundingIncrement); + dq.roundToInfinity(); + int32_t magnitude = dq.adjustToZeroScale(); + return constructIncrement(dq.toLong(), magnitude); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +IncrementPrecision Precision::incrementExact(uint64_t mantissa, int16_t magnitude) { + if (mantissa > 0.0) { + return constructIncrement(mantissa, magnitude); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +CurrencyPrecision Precision::currency(UCurrencyUsage currencyUsage) { + return constructCurrency(currencyUsage); +} + +Precision FractionPrecision::withSignificantDigits( + int32_t minSignificantDigits, + int32_t maxSignificantDigits, + UNumberRoundingPriority priority) const { + if (fType == RND_ERROR) { return *this; } // no-op in error state + if (minSignificantDigits >= 1 && + maxSignificantDigits >= minSignificantDigits && + maxSignificantDigits <= kMaxIntFracSig) { + return constructFractionSignificant( + *this, + minSignificantDigits, + maxSignificantDigits, + priority, + false); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision FractionPrecision::withMinDigits(int32_t minSignificantDigits) const { + if (fType == RND_ERROR) { return *this; } // no-op in error state + if (minSignificantDigits >= 1 && minSignificantDigits <= kMaxIntFracSig) { + return constructFractionSignificant( + *this, + 1, + minSignificantDigits, + UNUM_ROUNDING_PRIORITY_RELAXED, + true); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +Precision FractionPrecision::withMaxDigits(int32_t maxSignificantDigits) const { + if (fType == RND_ERROR) { return *this; } // no-op in error state + if (maxSignificantDigits >= 1 && maxSignificantDigits <= kMaxIntFracSig) { + return constructFractionSignificant(*this, + 1, + maxSignificantDigits, + UNUM_ROUNDING_PRIORITY_STRICT, + true); + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +// Private method on base class +Precision Precision::withCurrency(const CurrencyUnit ¤cy, UErrorCode &status) const { + if (fType == RND_ERROR) { return *this; } // no-op in error state + U_ASSERT(fType == RND_CURRENCY); + const char16_t *isoCode = currency.getISOCurrency(); + double increment = ucurr_getRoundingIncrementForUsage(isoCode, fUnion.currencyUsage, &status); + int32_t minMaxFrac = ucurr_getDefaultFractionDigitsForUsage( + isoCode, fUnion.currencyUsage, &status); + Precision retval = (increment != 0.0) + ? Precision::increment(increment) + : static_cast(Precision::fixedFraction(minMaxFrac)); + retval.fTrailingZeroDisplay = fTrailingZeroDisplay; + return retval; +} + +// Public method on CurrencyPrecision subclass +Precision CurrencyPrecision::withCurrency(const CurrencyUnit ¤cy) const { + UErrorCode localStatus = U_ZERO_ERROR; + Precision result = Precision::withCurrency(currency, localStatus); + if (U_FAILURE(localStatus)) { + return {localStatus}; + } + return result; +} + +Precision IncrementPrecision::withMinFraction(int32_t minFrac) const { + if (fType == RND_ERROR) { return *this; } // no-op in error state + if (minFrac >= 0 && minFrac <= kMaxIntFracSig) { + IncrementPrecision copy = *this; + copy.fUnion.increment.fMinFrac = minFrac; + return copy; + } else { + return {U_NUMBER_ARG_OUTOFBOUNDS_ERROR}; + } +} + +FractionPrecision Precision::constructFraction(int32_t minFrac, int32_t maxFrac) { + FractionSignificantSettings settings; + settings.fMinFrac = static_cast(minFrac); + settings.fMaxFrac = static_cast(maxFrac); + settings.fMinSig = -1; + settings.fMaxSig = -1; + PrecisionUnion union_; + union_.fracSig = settings; + return {RND_FRACTION, union_}; +} + +Precision Precision::constructSignificant(int32_t minSig, int32_t maxSig) { + FractionSignificantSettings settings; + settings.fMinFrac = -1; + settings.fMaxFrac = -1; + settings.fMinSig = static_cast(minSig); + settings.fMaxSig = static_cast(maxSig); + PrecisionUnion union_; + union_.fracSig = settings; + return {RND_SIGNIFICANT, union_}; +} + +Precision +Precision::constructFractionSignificant( + const FractionPrecision &base, + int32_t minSig, + int32_t maxSig, + UNumberRoundingPriority priority, + bool retain) { + FractionSignificantSettings settings = base.fUnion.fracSig; + settings.fMinSig = static_cast(minSig); + settings.fMaxSig = static_cast(maxSig); + settings.fPriority = priority; + settings.fRetain = retain; + PrecisionUnion union_; + union_.fracSig = settings; + return {RND_FRACTION_SIGNIFICANT, union_}; +} + +IncrementPrecision Precision::constructIncrement(uint64_t increment, digits_t magnitude) { + IncrementSettings settings; + // Note: For number formatting, fIncrement is used for RND_INCREMENT but not + // RND_INCREMENT_ONE or RND_INCREMENT_FIVE. However, fIncrement is used in all + // three when constructing a skeleton. + settings.fIncrement = increment; + settings.fIncrementMagnitude = magnitude; + settings.fMinFrac = magnitude > 0 ? 0 : -magnitude; + PrecisionUnion union_; + union_.increment = settings; + if (increment == 1) { + // NOTE: In C++, we must return the correct value type with the correct union. + // It would be invalid to return a RND_FRACTION here because the methods on the + // IncrementPrecision type assume that the union is backed by increment data. + return {RND_INCREMENT_ONE, union_}; + } else if (increment == 5) { + return {RND_INCREMENT_FIVE, union_}; + } else { + return {RND_INCREMENT, union_}; + } +} + +CurrencyPrecision Precision::constructCurrency(UCurrencyUsage usage) { + PrecisionUnion union_; + union_.currencyUsage = usage; + return {RND_CURRENCY, union_}; +} + + +RoundingImpl::RoundingImpl(const Precision& precision, UNumberFormatRoundingMode roundingMode, + const CurrencyUnit& currency, UErrorCode& status) + : fPrecision(precision), fRoundingMode(roundingMode), fPassThrough(false) { + if (precision.fType == Precision::RND_CURRENCY) { + fPrecision = precision.withCurrency(currency, status); + } +} + +RoundingImpl RoundingImpl::passThrough() { + return {}; +} + +bool RoundingImpl::isSignificantDigits() const { + return fPrecision.fType == Precision::RND_SIGNIFICANT; +} + +int32_t +RoundingImpl::chooseMultiplierAndApply(impl::DecimalQuantity &input, const impl::MultiplierProducer &producer, + UErrorCode &status) { + // Do not call this method with zero, NaN, or infinity. + U_ASSERT(!input.isZeroish()); + + // Perform the first attempt at rounding. + int magnitude = input.getMagnitude(); + int multiplier = producer.getMultiplier(magnitude); + input.adjustMagnitude(multiplier); + apply(input, status); + + // If the number rounded to zero, exit. + if (input.isZeroish() || U_FAILURE(status)) { + return multiplier; + } + + // If the new magnitude after rounding is the same as it was before rounding, then we are done. + // This case applies to most numbers. + if (input.getMagnitude() == magnitude + multiplier) { + return multiplier; + } + + // If the above case DIDN'T apply, then we have a case like 99.9 -> 100 or 999.9 -> 1000: + // The number rounded up to the next magnitude. Check if the multiplier changes; if it doesn't, + // we do not need to make any more adjustments. + int _multiplier = producer.getMultiplier(magnitude + 1); + if (multiplier == _multiplier) { + return multiplier; + } + + // We have a case like 999.9 -> 1000, where the correct output is "1K", not "1000". + // Fix the magnitude and re-apply the rounding strategy. + input.adjustMagnitude(_multiplier - multiplier); + apply(input, status); + return _multiplier; +} + +/** This is the method that contains the actual rounding logic. */ +void RoundingImpl::apply(impl::DecimalQuantity &value, UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + if (fPassThrough) { + return; + } + int32_t resolvedMinFraction = 0; + switch (fPrecision.fType) { + case Precision::RND_BOGUS: + case Precision::RND_ERROR: + // Errors should be caught before the apply() method is called + status = U_INTERNAL_PROGRAM_ERROR; + break; + + case Precision::RND_NONE: + value.roundToInfinity(); + break; + + case Precision::RND_FRACTION: + value.roundToMagnitude( + getRoundingMagnitudeFraction(fPrecision.fUnion.fracSig.fMaxFrac), + fRoundingMode, + status); + resolvedMinFraction = + uprv_max(0, -getDisplayMagnitudeFraction(fPrecision.fUnion.fracSig.fMinFrac)); + break; + + case Precision::RND_SIGNIFICANT: + value.roundToMagnitude( + getRoundingMagnitudeSignificant(value, fPrecision.fUnion.fracSig.fMaxSig), + fRoundingMode, + status); + resolvedMinFraction = + uprv_max(0, -getDisplayMagnitudeSignificant(value, fPrecision.fUnion.fracSig.fMinSig)); + // Make sure that digits are displayed on zero. + if (value.isZeroish() && fPrecision.fUnion.fracSig.fMinSig > 0) { + value.setMinInteger(1); + } + break; + + case Precision::RND_FRACTION_SIGNIFICANT: { + // From ECMA-402: + /* + Let sResult be ToRawPrecision(...). + Let fResult be ToRawFixed(...). + If intlObj.[[RoundingType]] is morePrecision, then + If sResult.[[RoundingMagnitude]] ≤ fResult.[[RoundingMagnitude]], then + Let result be sResult. + Else, + Let result be fResult. + Else, + Assert: intlObj.[[RoundingType]] is lessPrecision. + If sResult.[[RoundingMagnitude]] ≤ fResult.[[RoundingMagnitude]], then + Let result be fResult. + Else, + Let result be sResult. + */ + + int32_t roundingMag1 = getRoundingMagnitudeFraction(fPrecision.fUnion.fracSig.fMaxFrac); + int32_t roundingMag2 = getRoundingMagnitudeSignificant(value, fPrecision.fUnion.fracSig.fMaxSig); + int32_t roundingMag; + if (fPrecision.fUnion.fracSig.fPriority == UNUM_ROUNDING_PRIORITY_RELAXED) { + roundingMag = uprv_min(roundingMag1, roundingMag2); + } else { + roundingMag = uprv_max(roundingMag1, roundingMag2); + } + if (!value.isZeroish()) { + int32_t upperMag = value.getMagnitude(); + value.roundToMagnitude(roundingMag, fRoundingMode, status); + if (!value.isZeroish() && value.getMagnitude() != upperMag && roundingMag1 == roundingMag2) { + // roundingMag2 needs to be the magnitude after rounding + roundingMag2 += 1; + } + } + + int32_t displayMag1 = getDisplayMagnitudeFraction(fPrecision.fUnion.fracSig.fMinFrac); + int32_t displayMag2 = getDisplayMagnitudeSignificant(value, fPrecision.fUnion.fracSig.fMinSig); + int32_t displayMag; + if (fPrecision.fUnion.fracSig.fRetain) { + // withMinDigits + withMaxDigits + displayMag = uprv_min(displayMag1, displayMag2); + } else if (fPrecision.fUnion.fracSig.fPriority == UNUM_ROUNDING_PRIORITY_RELAXED) { + if (roundingMag2 <= roundingMag1) { + displayMag = displayMag2; + } else { + displayMag = displayMag1; + } + } else { + U_ASSERT(fPrecision.fUnion.fracSig.fPriority == UNUM_ROUNDING_PRIORITY_STRICT); + if (roundingMag2 <= roundingMag1) { + displayMag = displayMag1; + } else { + displayMag = displayMag2; + } + } + resolvedMinFraction = uprv_max(0, -displayMag); + + break; + } + + case Precision::RND_INCREMENT: + value.roundToIncrement( + fPrecision.fUnion.increment.fIncrement, + fPrecision.fUnion.increment.fIncrementMagnitude, + fRoundingMode, + status); + resolvedMinFraction = fPrecision.fUnion.increment.fMinFrac; + break; + + case Precision::RND_INCREMENT_ONE: + value.roundToMagnitude( + fPrecision.fUnion.increment.fIncrementMagnitude, + fRoundingMode, + status); + resolvedMinFraction = fPrecision.fUnion.increment.fMinFrac; + break; + + case Precision::RND_INCREMENT_FIVE: + value.roundToNickel( + fPrecision.fUnion.increment.fIncrementMagnitude, + fRoundingMode, + status); + resolvedMinFraction = fPrecision.fUnion.increment.fMinFrac; + break; + + case Precision::RND_CURRENCY: + // Call .withCurrency() before .apply()! + UPRV_UNREACHABLE_EXIT; + + default: + UPRV_UNREACHABLE_EXIT; + } + + if (fPrecision.fTrailingZeroDisplay == UNUM_TRAILING_ZERO_AUTO || + // PLURAL_OPERAND_T returns fraction digits as an integer + value.getPluralOperand(PLURAL_OPERAND_T) != 0) { + value.setMinFraction(resolvedMinFraction); + } +} + +void RoundingImpl::apply(impl::DecimalQuantity &value, int32_t minInt, UErrorCode /*status*/) { + // This method is intended for the one specific purpose of helping print "00.000E0". + // Question: Is it useful to look at trailingZeroDisplay here? + U_ASSERT(isSignificantDigits()); + U_ASSERT(value.isZeroish()); + value.setMinFraction(fPrecision.fUnion.fracSig.fMinSig - minInt); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_roundingutils.h b/intl/icu/source/i18n/number_roundingutils.h new file mode 100644 index 0000000000..6657127254 --- /dev/null +++ b/intl/icu/source/i18n/number_roundingutils.h @@ -0,0 +1,247 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_ROUNDINGUTILS_H__ +#define __NUMBER_ROUNDINGUTILS_H__ + +#include "number_types.h" +#include "string_segment.h" + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { +namespace roundingutils { + +enum Section { + SECTION_LOWER_EDGE = -1, + SECTION_UPPER_EDGE = -2, + SECTION_LOWER = 1, + SECTION_MIDPOINT = 2, + SECTION_UPPER = 3 +}; + +/** + * Converts a rounding mode and metadata about the quantity being rounded to a boolean determining + * whether the value should be rounded toward infinity or toward zero. + * + *

    The parameters are of type int because benchmarks on an x86-64 processor against OpenJDK + * showed that ints were demonstrably faster than enums in switch statements. + * + * @param isEven Whether the digit immediately before the rounding magnitude is even. + * @param isNegative Whether the quantity is negative. + * @param section Whether the part of the quantity to the right of the rounding magnitude is + * exactly halfway between two digits, whether it is in the lower part (closer to zero), or + * whether it is in the upper part (closer to infinity). See {@link #SECTION_LOWER}, {@link + * #SECTION_MIDPOINT}, and {@link #SECTION_UPPER}. + * @param roundingMode The integer version of the {@link RoundingMode}, which you can get via + * {@link RoundingMode#ordinal}. + * @param status Error code, set to U_FORMAT_INEXACT_ERROR if the rounding mode is kRoundUnnecessary. + * @return true if the number should be rounded toward zero; false if it should be rounded toward + * infinity. + */ +inline bool +getRoundingDirection(bool isEven, bool isNegative, Section section, RoundingMode roundingMode, + UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + switch (roundingMode) { + case RoundingMode::UNUM_ROUND_UP: + // round away from zero + return false; + + case RoundingMode::UNUM_ROUND_DOWN: + // round toward zero + return true; + + case RoundingMode::UNUM_ROUND_CEILING: + // round toward positive infinity + return isNegative; + + case RoundingMode::UNUM_ROUND_FLOOR: + // round toward negative infinity + return !isNegative; + + case RoundingMode::UNUM_ROUND_HALFUP: + switch (section) { + case SECTION_MIDPOINT: + return false; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + case RoundingMode::UNUM_ROUND_HALFDOWN: + switch (section) { + case SECTION_MIDPOINT: + return true; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + case RoundingMode::UNUM_ROUND_HALFEVEN: + switch (section) { + case SECTION_MIDPOINT: + return isEven; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + case RoundingMode::UNUM_ROUND_HALF_ODD: + switch (section) { + case SECTION_MIDPOINT: + return !isEven; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + case RoundingMode::UNUM_ROUND_HALF_CEILING: + switch (section) { + case SECTION_MIDPOINT: + return isNegative; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + case RoundingMode::UNUM_ROUND_HALF_FLOOR: + switch (section) { + case SECTION_MIDPOINT: + return !isNegative; + case SECTION_LOWER: + return true; + case SECTION_UPPER: + return false; + default: + break; + } + break; + + default: + break; + } + + status = U_FORMAT_INEXACT_ERROR; + return false; +} + +/** + * Gets whether the given rounding mode's rounding boundary is at the midpoint. The rounding + * boundary is the point at which a number switches from being rounded down to being rounded up. + * For example, with rounding mode HALF_EVEN, HALF_UP, or HALF_DOWN, the rounding boundary is at + * the midpoint, and this function would return true. However, for UP, DOWN, CEILING, and FLOOR, + * the rounding boundary is at the "edge", and this function would return false. + * + * @param roundingMode The integer version of the {@link RoundingMode}. + * @return true if rounding mode is HALF_EVEN, HALF_UP, or HALF_DOWN; false otherwise. + */ +inline bool roundsAtMidpoint(int roundingMode) { + switch (roundingMode) { + case RoundingMode::UNUM_ROUND_UP: + case RoundingMode::UNUM_ROUND_DOWN: + case RoundingMode::UNUM_ROUND_CEILING: + case RoundingMode::UNUM_ROUND_FLOOR: + return false; + + default: + return true; + } +} + +} // namespace roundingutils + + +/** + * Encapsulates a Precision and a RoundingMode and performs rounding on a DecimalQuantity. + * + * This class does not exist in Java: instead, the base Precision class is used. + */ +class RoundingImpl { + public: + RoundingImpl() = default; // defaults to pass-through rounder + + RoundingImpl(const Precision& precision, UNumberFormatRoundingMode roundingMode, + const CurrencyUnit& currency, UErrorCode& status); + + static RoundingImpl passThrough(); + + /** Required for ScientificFormatter */ + bool isSignificantDigits() const; + + /** + * Rounding endpoint used by Engineering and Compact notation. Chooses the most appropriate multiplier (magnitude + * adjustment), applies the adjustment, rounds, and returns the chosen multiplier. + * + *

    + * In most cases, this is simple. However, when rounding the number causes it to cross a multiplier boundary, we + * need to re-do the rounding. For example, to display 999,999 in Engineering notation with 2 sigfigs, first you + * guess the multiplier to be -3. However, then you end up getting 1000E3, which is not the correct output. You then + * change your multiplier to be -6, and you get 1.0E6, which is correct. + * + * @param input The quantity to process. + * @param producer Function to call to return a multiplier based on a magnitude. + * @return The number of orders of magnitude the input was adjusted by this method. + */ + int32_t + chooseMultiplierAndApply(impl::DecimalQuantity &input, const impl::MultiplierProducer &producer, + UErrorCode &status); + + void apply(impl::DecimalQuantity &value, UErrorCode &status) const; + + /** Version of {@link #apply} that obeys minInt constraints. Used for scientific notation compatibility mode. */ + void apply(impl::DecimalQuantity &value, int32_t minInt, UErrorCode status); + + private: + Precision fPrecision; + UNumberFormatRoundingMode fRoundingMode; + bool fPassThrough = true; // default value + + // Permits access to fPrecision. + friend class units::UnitsRouter; + + // Permits access to fPrecision. + friend class UnitConversionHandler; +}; + +/** + * Parses Precision-related skeleton strings without knowledge of MacroProps + * - see blueprint_helpers::parseIncrementOption(). + * + * Referencing MacroProps means needing to pull in the .o files that have the + * destructors for the SymbolsWrapper, StringProp, and Scale classes. + */ +void parseIncrementOption(const StringSegment &segment, Precision &outPrecision, UErrorCode &status); + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_ROUNDINGUTILS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_scientific.cpp b/intl/icu/source/i18n/number_scientific.cpp new file mode 100644 index 0000000000..d365d982d4 --- /dev/null +++ b/intl/icu/source/i18n/number_scientific.cpp @@ -0,0 +1,177 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include +#include "number_scientific.h" +#include "number_utils.h" +#include "formatted_string_builder.h" +#include "unicode/unum.h" +#include "number_microprops.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +// NOTE: The object lifecycle of ScientificModifier and ScientificHandler differ greatly in Java and C++. +// +// During formatting, we need to provide an object with state (the exponent) as the inner modifier. +// +// In Java, where the priority is put on reducing object creations, the unsafe code path re-uses the +// ScientificHandler as a ScientificModifier, and the safe code path pre-computes 25 ScientificModifier +// instances. This scheme reduces the number of object creations by 1 in both safe and unsafe. +// +// In C++, MicroProps provides a pre-allocated ScientificModifier, and ScientificHandler simply populates +// the state (the exponent) into that ScientificModifier. There is no difference between safe and unsafe. + +ScientificModifier::ScientificModifier() : fExponent(0), fHandler(nullptr) {} + +void ScientificModifier::set(int32_t exponent, const ScientificHandler *handler) { + // ScientificModifier should be set only once. + U_ASSERT(fHandler == nullptr); + fExponent = exponent; + fHandler = handler; +} + +int32_t ScientificModifier::apply(FormattedStringBuilder &output, int32_t /*leftIndex*/, int32_t rightIndex, + UErrorCode &status) const { + // FIXME: Localized exponent separator location. + int i = rightIndex; + // Append the exponent separator and sign + i += output.insert( + i, + fHandler->fSymbols->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kExponentialSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_EXPONENT_SYMBOL_FIELD}, + status); + if (fExponent < 0 && fHandler->fSettings.fExponentSignDisplay != UNUM_SIGN_NEVER) { + i += output.insert( + i, + fHandler->fSymbols + ->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kMinusSignSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_EXPONENT_SIGN_FIELD}, + status); + } else if (fExponent >= 0 && fHandler->fSettings.fExponentSignDisplay == UNUM_SIGN_ALWAYS) { + i += output.insert( + i, + fHandler->fSymbols + ->getSymbol(DecimalFormatSymbols::ENumberFormatSymbol::kPlusSignSymbol), + {UFIELD_CATEGORY_NUMBER, UNUM_EXPONENT_SIGN_FIELD}, + status); + } + // Append the exponent digits (using a simple inline algorithm) + int32_t disp = std::abs(fExponent); + for (int j = 0; j < fHandler->fSettings.fMinExponentDigits || disp > 0; j++, disp /= 10) { + auto d = static_cast(disp % 10); + i += utils::insertDigitFromSymbols( + output, + i - j, + d, + *fHandler->fSymbols, + {UFIELD_CATEGORY_NUMBER, UNUM_EXPONENT_FIELD}, + status); + } + return i - rightIndex; +} + +int32_t ScientificModifier::getPrefixLength() const { + // TODO: Localized exponent separator location. + return 0; +} + +int32_t ScientificModifier::getCodePointCount() const { + // NOTE: This method is only called one place, NumberRangeFormatterImpl. + // The call site only cares about != 0 and != 1. + // Return a very large value so that if this method is used elsewhere, we should notice. + return 999; +} + +bool ScientificModifier::isStrong() const { + // Scientific is always strong + return true; +} + +bool ScientificModifier::containsField(Field field) const { + (void)field; + // This method is not used for inner modifiers. + UPRV_UNREACHABLE_EXIT; +} + +void ScientificModifier::getParameters(Parameters& output) const { + // Not part of any plural sets + output.obj = nullptr; +} + +bool ScientificModifier::semanticallyEquivalent(const Modifier& other) const { + auto* _other = dynamic_cast(&other); + if (_other == nullptr) { + return false; + } + // TODO: Check for locale symbols and settings as well? Could be less efficient. + return fExponent == _other->fExponent; +} + +// Note: Visual Studio does not compile this function without full name space. Why? +icu::number::impl::ScientificHandler::ScientificHandler(const Notation *notation, const DecimalFormatSymbols *symbols, + const MicroPropsGenerator *parent) : + fSettings(notation->fUnion.scientific), fSymbols(symbols), fParent(parent) {} + +void ScientificHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + fParent->processQuantity(quantity, micros, status); + if (U_FAILURE(status)) { return; } + + // Do not apply scientific notation to special doubles + if (quantity.isInfinite() || quantity.isNaN()) { + micros.modInner = µs.helpers.emptyStrongModifier; + return; + } + + // Treat zero as if it had magnitude 0 + int32_t exponent; + if (quantity.isZeroish()) { + if (fSettings.fRequireMinInt && micros.rounder.isSignificantDigits()) { + // Show "00.000E0" on pattern "00.000E0" + micros.rounder.apply(quantity, fSettings.fEngineeringInterval, status); + exponent = 0; + } else { + micros.rounder.apply(quantity, status); + exponent = 0; + } + } else { + exponent = -micros.rounder.chooseMultiplierAndApply(quantity, *this, status); + } + + // Use MicroProps's helper ScientificModifier and save it as the modInner. + ScientificModifier &mod = micros.helpers.scientificModifier; + mod.set(exponent, this); + micros.modInner = &mod; + + // Change the exponent only after we select appropriate plural form + // for formatting purposes so that we preserve expected formatted + // string behavior. + quantity.adjustExponent(exponent); + + // We already performed rounding. Do not perform it again. + micros.rounder = RoundingImpl::passThrough(); +} + +int32_t ScientificHandler::getMultiplier(int32_t magnitude) const { + int32_t interval = fSettings.fEngineeringInterval; + int32_t digitsShown; + if (fSettings.fRequireMinInt) { + // For patterns like "000.00E0" and ".00E0" + digitsShown = interval; + } else if (interval <= 1) { + // For patterns like "0.00E0" and "@@@E0" + digitsShown = 1; + } else { + // For patterns like "##0.00" + digitsShown = ((magnitude % interval + interval) % interval) + 1; + } + return digitsShown - magnitude - 1; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_scientific.h b/intl/icu/source/i18n/number_scientific.h new file mode 100644 index 0000000000..22140a09af --- /dev/null +++ b/intl/icu/source/i18n/number_scientific.h @@ -0,0 +1,68 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_SCIENTIFIC_H__ +#define __NUMBER_SCIENTIFIC_H__ + +#include "number_types.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + +// Forward-declare +class ScientificHandler; + +class U_I18N_API ScientificModifier : public UMemory, public Modifier { + public: + ScientificModifier(); + + void set(int32_t exponent, const ScientificHandler *handler); + + int32_t apply(FormattedStringBuilder &output, int32_t leftIndex, int32_t rightIndex, + UErrorCode &status) const override; + + int32_t getPrefixLength() const override; + + int32_t getCodePointCount() const override; + + bool isStrong() const override; + + bool containsField(Field field) const override; + + void getParameters(Parameters& output) const override; + + bool semanticallyEquivalent(const Modifier& other) const override; + + private: + int32_t fExponent; + const ScientificHandler *fHandler; +}; + +class ScientificHandler : public UMemory, public MicroPropsGenerator, public MultiplierProducer { + public: + ScientificHandler(const Notation *notation, const DecimalFormatSymbols *symbols, + const MicroPropsGenerator *parent); + + void + processQuantity(DecimalQuantity &quantity, MicroProps µs, UErrorCode &status) const override; + + int32_t getMultiplier(int32_t magnitude) const override; + + private: + const Notation::ScientificSettings fSettings; + const DecimalFormatSymbols *fSymbols; + const MicroPropsGenerator *fParent; + + friend class ScientificModifier; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_SCIENTIFIC_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_simple.cpp b/intl/icu/source/i18n/number_simple.cpp new file mode 100644 index 0000000000..a2af6be42d --- /dev/null +++ b/intl/icu/source/i18n/number_simple.cpp @@ -0,0 +1,255 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numberformatter.h" +#include "unicode/simplenumberformatter.h" +#include "number_formatimpl.h" +#include "number_utils.h" +#include "number_patternmodifier.h" +#include "number_utypes.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +SimpleNumber +SimpleNumber::forInt64(int64_t value, UErrorCode& status) { + if (U_FAILURE(status)) { + return SimpleNumber(); + } + auto results = new UFormattedNumberData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return SimpleNumber(); + } + results->quantity.setToLong(value); + return SimpleNumber(results, status); +} + +SimpleNumber::SimpleNumber(UFormattedNumberData* data, UErrorCode& status) : fData(data) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (fData->quantity.isNegative()) { + fSign = UNUM_SIMPLE_NUMBER_MINUS_SIGN; + } else { + fSign = UNUM_SIMPLE_NUMBER_NO_SIGN; + } +} + +void SimpleNumber::cleanup() { + delete fData; + fData = nullptr; +} + +void SimpleNumber::multiplyByPowerOfTen(int32_t power, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fData->quantity.adjustMagnitude(power); +} + +void SimpleNumber::roundTo(int32_t position, UNumberFormatRoundingMode roundingMode, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fData->quantity.roundToMagnitude(position, roundingMode, status); +} + +void SimpleNumber::setMinimumIntegerDigits(uint32_t position, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fData->quantity.setMinInteger(position); +} + +void SimpleNumber::setMinimumFractionDigits(uint32_t position, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fData->quantity.setMinFraction(position); +} + +void SimpleNumber::truncateStart(uint32_t position, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fData->quantity.applyMaxInteger(position); +} + +void SimpleNumber::setSign(USimpleNumberSign sign, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fData == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + fSign = sign; +} + + +void SimpleNumberFormatter::cleanup() { + delete fOwnedSymbols; + delete fMicros; + delete fPatternModifier; + fOwnedSymbols = nullptr; + fMicros = nullptr; + fPatternModifier = nullptr; +} + +SimpleNumberFormatter SimpleNumberFormatter::forLocale(const icu::Locale &locale, UErrorCode &status) { + return SimpleNumberFormatter::forLocaleAndGroupingStrategy(locale, UNUM_GROUPING_AUTO, status); +} + +SimpleNumberFormatter SimpleNumberFormatter::forLocaleAndGroupingStrategy( + const icu::Locale &locale, + UNumberGroupingStrategy groupingStrategy, + UErrorCode &status) { + SimpleNumberFormatter retval; + retval.fOwnedSymbols = new DecimalFormatSymbols(locale, status); + if (U_FAILURE(status)) { + return retval; + } + if (retval.fOwnedSymbols == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return retval; + } + retval.initialize(locale, *retval.fOwnedSymbols, groupingStrategy, status); + return retval; +} + + +SimpleNumberFormatter SimpleNumberFormatter::forLocaleAndSymbolsAndGroupingStrategy( + const icu::Locale &locale, + const DecimalFormatSymbols &symbols, + UNumberGroupingStrategy groupingStrategy, + UErrorCode &status) { + SimpleNumberFormatter retval; + retval.initialize(locale, symbols, groupingStrategy, status); + return retval; +} + + +void SimpleNumberFormatter::initialize( + const icu::Locale &locale, + const DecimalFormatSymbols &symbols, + UNumberGroupingStrategy groupingStrategy, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + + fMicros = new SimpleMicroProps(); + if (fMicros == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + fMicros->symbols = &symbols; + + auto pattern = utils::getPatternForStyle( + locale, + symbols.getNumberingSystemName(), + CLDR_PATTERN_STYLE_DECIMAL, + status); + if (U_FAILURE(status)) { + return; + } + + ParsedPatternInfo patternInfo; + PatternParser::parseToPatternInfo(UnicodeString(pattern), patternInfo, status); + if (U_FAILURE(status)) { + return; + } + + auto grouper = Grouper::forStrategy(groupingStrategy); + grouper.setLocaleData(patternInfo, locale); + fMicros->grouping = grouper; + + MutablePatternModifier patternModifier(false); + patternModifier.setPatternInfo(&patternInfo, kUndefinedField); + patternModifier.setPatternAttributes(UNUM_SIGN_EXCEPT_ZERO, false, false); + patternModifier.setSymbols(fMicros->symbols, {}, UNUM_UNIT_WIDTH_SHORT, nullptr, status); + + fPatternModifier = new AdoptingSignumModifierStore(patternModifier.createImmutableForPlural(StandardPlural::COUNT, status)); + + fGroupingStrategy = groupingStrategy; + return; +} + +FormattedNumber SimpleNumberFormatter::format(SimpleNumber value, UErrorCode &status) const { + formatImpl(value.fData, value.fSign, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + auto temp = value.fData; + value.fData = nullptr; + return FormattedNumber(temp); + } else { + return FormattedNumber(status); + } +} + +void SimpleNumberFormatter::formatImpl(UFormattedNumberData* data, USimpleNumberSign sign, UErrorCode &status) const { + if (U_FAILURE(status)) { + return; + } + if (data == nullptr) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (fPatternModifier == nullptr || fMicros == nullptr) { + status = U_INVALID_STATE_ERROR; + return; + } + + Signum signum; + if (sign == UNUM_SIMPLE_NUMBER_MINUS_SIGN) { + signum = SIGNUM_NEG; + } else if (sign == UNUM_SIMPLE_NUMBER_PLUS_SIGN) { + signum = SIGNUM_POS; + } else { + signum = SIGNUM_POS_ZERO; + } + + const Modifier* modifier = (*fPatternModifier)[signum]; + auto length = NumberFormatterImpl::writeNumber( + *fMicros, + data->quantity, + data->getStringRef(), + 0, + status); + length += modifier->apply(data->getStringRef(), 0, length, status); + data->getStringRef().writeTerminator(status); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_skeletons.cpp b/intl/icu/source/i18n/number_skeletons.cpp new file mode 100644 index 0000000000..ef3befbffa --- /dev/null +++ b/intl/icu/source/i18n/number_skeletons.cpp @@ -0,0 +1,1813 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "number_decnum.h" +#include "number_roundingutils.h" +#include "number_skeletons.h" +#include "umutex.h" +#include "ucln_in.h" +#include "patternprops.h" +#include "unicode/ucharstriebuilder.h" +#include "number_utils.h" +#include "number_decimalquantity.h" +#include "unicode/numberformatter.h" +#include "uinvchar.h" +#include "charstr.h" +#include "string_segment.h" +#include "unicode/errorcode.h" +#include "util.h" +#include "measunit_impl.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using namespace icu::number::impl::skeleton; + +namespace { + +icu::UInitOnce gNumberSkeletonsInitOnce {}; + +char16_t* kSerializedStemTrie = nullptr; + +UBool U_CALLCONV cleanupNumberSkeletons() { + uprv_free(kSerializedStemTrie); + kSerializedStemTrie = nullptr; + gNumberSkeletonsInitOnce.reset(); + return true; +} + +void U_CALLCONV initNumberSkeletons(UErrorCode& status) { + ucln_i18n_registerCleanup(UCLN_I18N_NUMBER_SKELETONS, cleanupNumberSkeletons); + + UCharsTrieBuilder b(status); + if (U_FAILURE(status)) { return; } + + // Section 1: + b.add(u"compact-short", STEM_COMPACT_SHORT, status); + b.add(u"compact-long", STEM_COMPACT_LONG, status); + b.add(u"scientific", STEM_SCIENTIFIC, status); + b.add(u"engineering", STEM_ENGINEERING, status); + b.add(u"notation-simple", STEM_NOTATION_SIMPLE, status); + b.add(u"base-unit", STEM_BASE_UNIT, status); + b.add(u"percent", STEM_PERCENT, status); + b.add(u"permille", STEM_PERMILLE, status); + b.add(u"precision-integer", STEM_PRECISION_INTEGER, status); + b.add(u"precision-unlimited", STEM_PRECISION_UNLIMITED, status); + b.add(u"precision-currency-standard", STEM_PRECISION_CURRENCY_STANDARD, status); + b.add(u"precision-currency-cash", STEM_PRECISION_CURRENCY_CASH, status); + b.add(u"rounding-mode-ceiling", STEM_ROUNDING_MODE_CEILING, status); + b.add(u"rounding-mode-floor", STEM_ROUNDING_MODE_FLOOR, status); + b.add(u"rounding-mode-down", STEM_ROUNDING_MODE_DOWN, status); + b.add(u"rounding-mode-up", STEM_ROUNDING_MODE_UP, status); + b.add(u"rounding-mode-half-even", STEM_ROUNDING_MODE_HALF_EVEN, status); + b.add(u"rounding-mode-half-odd", STEM_ROUNDING_MODE_HALF_ODD, status); + b.add(u"rounding-mode-half-ceiling", STEM_ROUNDING_MODE_HALF_CEILING, status); + b.add(u"rounding-mode-half-floor", STEM_ROUNDING_MODE_HALF_FLOOR, status); + b.add(u"rounding-mode-half-down", STEM_ROUNDING_MODE_HALF_DOWN, status); + b.add(u"rounding-mode-half-up", STEM_ROUNDING_MODE_HALF_UP, status); + b.add(u"rounding-mode-unnecessary", STEM_ROUNDING_MODE_UNNECESSARY, status); + b.add(u"integer-width-trunc", STEM_INTEGER_WIDTH_TRUNC, status); + b.add(u"group-off", STEM_GROUP_OFF, status); + b.add(u"group-min2", STEM_GROUP_MIN2, status); + b.add(u"group-auto", STEM_GROUP_AUTO, status); + b.add(u"group-on-aligned", STEM_GROUP_ON_ALIGNED, status); + b.add(u"group-thousands", STEM_GROUP_THOUSANDS, status); + b.add(u"latin", STEM_LATIN, status); + b.add(u"unit-width-narrow", STEM_UNIT_WIDTH_NARROW, status); + b.add(u"unit-width-short", STEM_UNIT_WIDTH_SHORT, status); + b.add(u"unit-width-full-name", STEM_UNIT_WIDTH_FULL_NAME, status); + b.add(u"unit-width-iso-code", STEM_UNIT_WIDTH_ISO_CODE, status); + b.add(u"unit-width-formal", STEM_UNIT_WIDTH_FORMAL, status); + b.add(u"unit-width-variant", STEM_UNIT_WIDTH_VARIANT, status); + b.add(u"unit-width-hidden", STEM_UNIT_WIDTH_HIDDEN, status); + b.add(u"sign-auto", STEM_SIGN_AUTO, status); + b.add(u"sign-always", STEM_SIGN_ALWAYS, status); + b.add(u"sign-never", STEM_SIGN_NEVER, status); + b.add(u"sign-accounting", STEM_SIGN_ACCOUNTING, status); + b.add(u"sign-accounting-always", STEM_SIGN_ACCOUNTING_ALWAYS, status); + b.add(u"sign-except-zero", STEM_SIGN_EXCEPT_ZERO, status); + b.add(u"sign-accounting-except-zero", STEM_SIGN_ACCOUNTING_EXCEPT_ZERO, status); + b.add(u"sign-negative", STEM_SIGN_NEGATIVE, status); + b.add(u"sign-accounting-negative", STEM_SIGN_ACCOUNTING_NEGATIVE, status); + b.add(u"decimal-auto", STEM_DECIMAL_AUTO, status); + b.add(u"decimal-always", STEM_DECIMAL_ALWAYS, status); + if (U_FAILURE(status)) { return; } + + // Section 2: + b.add(u"precision-increment", STEM_PRECISION_INCREMENT, status); + b.add(u"measure-unit", STEM_MEASURE_UNIT, status); + b.add(u"per-measure-unit", STEM_PER_MEASURE_UNIT, status); + b.add(u"unit", STEM_UNIT, status); + b.add(u"usage", STEM_UNIT_USAGE, status); + b.add(u"currency", STEM_CURRENCY, status); + b.add(u"integer-width", STEM_INTEGER_WIDTH, status); + b.add(u"numbering-system", STEM_NUMBERING_SYSTEM, status); + b.add(u"scale", STEM_SCALE, status); + if (U_FAILURE(status)) { return; } + + // Section 3 (concise tokens): + b.add(u"K", STEM_COMPACT_SHORT, status); + b.add(u"KK", STEM_COMPACT_LONG, status); + b.add(u"%", STEM_PERCENT, status); + b.add(u"%x100", STEM_PERCENT_100, status); + b.add(u",_", STEM_GROUP_OFF, status); + b.add(u",?", STEM_GROUP_MIN2, status); + b.add(u",!", STEM_GROUP_ON_ALIGNED, status); + b.add(u"+!", STEM_SIGN_ALWAYS, status); + b.add(u"+_", STEM_SIGN_NEVER, status); + b.add(u"()", STEM_SIGN_ACCOUNTING, status); + b.add(u"()!", STEM_SIGN_ACCOUNTING_ALWAYS, status); + b.add(u"+?", STEM_SIGN_EXCEPT_ZERO, status); + b.add(u"()?", STEM_SIGN_ACCOUNTING_EXCEPT_ZERO, status); + b.add(u"+-", STEM_SIGN_NEGATIVE, status); + b.add(u"()-", STEM_SIGN_ACCOUNTING_NEGATIVE, status); + if (U_FAILURE(status)) { return; } + + // Build the CharsTrie + // TODO: Use SLOW or FAST here? + UnicodeString result; + b.buildUnicodeString(USTRINGTRIE_BUILD_FAST, result, status); + if (U_FAILURE(status)) { return; } + + // Copy the result into the global constant pointer + size_t numBytes = result.length() * sizeof(char16_t); + kSerializedStemTrie = static_cast(uprv_malloc(numBytes)); + uprv_memcpy(kSerializedStemTrie, result.getBuffer(), numBytes); +} + + +inline void appendMultiple(UnicodeString& sb, UChar32 cp, int32_t count) { + for (int i = 0; i < count; i++) { + sb.append(cp); + } +} + + +#define CHECK_NULL(seen, field, status) (void)(seen); /* for auto-format line wrapping */ \ +UPRV_BLOCK_MACRO_BEGIN { \ + if ((seen).field) { \ + (status) = U_NUMBER_SKELETON_SYNTAX_ERROR; \ + return STATE_NULL; \ + } \ + (seen).field = true; \ +} UPRV_BLOCK_MACRO_END + + +} // anonymous namespace + + +Notation stem_to_object::notation(skeleton::StemEnum stem) { + switch (stem) { + case STEM_COMPACT_SHORT: + return Notation::compactShort(); + case STEM_COMPACT_LONG: + return Notation::compactLong(); + case STEM_SCIENTIFIC: + return Notation::scientific(); + case STEM_ENGINEERING: + return Notation::engineering(); + case STEM_NOTATION_SIMPLE: + return Notation::simple(); + default: + UPRV_UNREACHABLE_EXIT; + } +} + +MeasureUnit stem_to_object::unit(skeleton::StemEnum stem) { + switch (stem) { + case STEM_BASE_UNIT: + return MeasureUnit(); + case STEM_PERCENT: + return MeasureUnit::getPercent(); + case STEM_PERMILLE: + return MeasureUnit::getPermille(); + default: + UPRV_UNREACHABLE_EXIT; + } +} + +Precision stem_to_object::precision(skeleton::StemEnum stem) { + switch (stem) { + case STEM_PRECISION_INTEGER: + return Precision::integer(); + case STEM_PRECISION_UNLIMITED: + return Precision::unlimited(); + case STEM_PRECISION_CURRENCY_STANDARD: + return Precision::currency(UCURR_USAGE_STANDARD); + case STEM_PRECISION_CURRENCY_CASH: + return Precision::currency(UCURR_USAGE_CASH); + default: + UPRV_UNREACHABLE_EXIT; + } +} + +UNumberFormatRoundingMode stem_to_object::roundingMode(skeleton::StemEnum stem) { + switch (stem) { + case STEM_ROUNDING_MODE_CEILING: + return UNUM_ROUND_CEILING; + case STEM_ROUNDING_MODE_FLOOR: + return UNUM_ROUND_FLOOR; + case STEM_ROUNDING_MODE_DOWN: + return UNUM_ROUND_DOWN; + case STEM_ROUNDING_MODE_UP: + return UNUM_ROUND_UP; + case STEM_ROUNDING_MODE_HALF_EVEN: + return UNUM_ROUND_HALFEVEN; + case STEM_ROUNDING_MODE_HALF_ODD: + return UNUM_ROUND_HALF_ODD; + case STEM_ROUNDING_MODE_HALF_CEILING: + return UNUM_ROUND_HALF_CEILING; + case STEM_ROUNDING_MODE_HALF_FLOOR: + return UNUM_ROUND_HALF_FLOOR; + case STEM_ROUNDING_MODE_HALF_DOWN: + return UNUM_ROUND_HALFDOWN; + case STEM_ROUNDING_MODE_HALF_UP: + return UNUM_ROUND_HALFUP; + case STEM_ROUNDING_MODE_UNNECESSARY: + return UNUM_ROUND_UNNECESSARY; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +UNumberGroupingStrategy stem_to_object::groupingStrategy(skeleton::StemEnum stem) { + switch (stem) { + case STEM_GROUP_OFF: + return UNUM_GROUPING_OFF; + case STEM_GROUP_MIN2: + return UNUM_GROUPING_MIN2; + case STEM_GROUP_AUTO: + return UNUM_GROUPING_AUTO; + case STEM_GROUP_ON_ALIGNED: + return UNUM_GROUPING_ON_ALIGNED; + case STEM_GROUP_THOUSANDS: + return UNUM_GROUPING_THOUSANDS; + default: + return UNUM_GROUPING_COUNT; // for objects, throw; for enums, return COUNT + } +} + +UNumberUnitWidth stem_to_object::unitWidth(skeleton::StemEnum stem) { + switch (stem) { + case STEM_UNIT_WIDTH_NARROW: + return UNUM_UNIT_WIDTH_NARROW; + case STEM_UNIT_WIDTH_SHORT: + return UNUM_UNIT_WIDTH_SHORT; + case STEM_UNIT_WIDTH_FULL_NAME: + return UNUM_UNIT_WIDTH_FULL_NAME; + case STEM_UNIT_WIDTH_ISO_CODE: + return UNUM_UNIT_WIDTH_ISO_CODE; + case STEM_UNIT_WIDTH_FORMAL: + return UNUM_UNIT_WIDTH_FORMAL; + case STEM_UNIT_WIDTH_VARIANT: + return UNUM_UNIT_WIDTH_VARIANT; + case STEM_UNIT_WIDTH_HIDDEN: + return UNUM_UNIT_WIDTH_HIDDEN; + default: + return UNUM_UNIT_WIDTH_COUNT; // for objects, throw; for enums, return COUNT + } +} + +UNumberSignDisplay stem_to_object::signDisplay(skeleton::StemEnum stem) { + switch (stem) { + case STEM_SIGN_AUTO: + return UNUM_SIGN_AUTO; + case STEM_SIGN_ALWAYS: + return UNUM_SIGN_ALWAYS; + case STEM_SIGN_NEVER: + return UNUM_SIGN_NEVER; + case STEM_SIGN_ACCOUNTING: + return UNUM_SIGN_ACCOUNTING; + case STEM_SIGN_ACCOUNTING_ALWAYS: + return UNUM_SIGN_ACCOUNTING_ALWAYS; + case STEM_SIGN_EXCEPT_ZERO: + return UNUM_SIGN_EXCEPT_ZERO; + case STEM_SIGN_ACCOUNTING_EXCEPT_ZERO: + return UNUM_SIGN_ACCOUNTING_EXCEPT_ZERO; + case STEM_SIGN_NEGATIVE: + return UNUM_SIGN_NEGATIVE; + case STEM_SIGN_ACCOUNTING_NEGATIVE: + return UNUM_SIGN_ACCOUNTING_NEGATIVE; + default: + return UNUM_SIGN_COUNT; // for objects, throw; for enums, return COUNT + } +} + +UNumberDecimalSeparatorDisplay stem_to_object::decimalSeparatorDisplay(skeleton::StemEnum stem) { + switch (stem) { + case STEM_DECIMAL_AUTO: + return UNUM_DECIMAL_SEPARATOR_AUTO; + case STEM_DECIMAL_ALWAYS: + return UNUM_DECIMAL_SEPARATOR_ALWAYS; + default: + return UNUM_DECIMAL_SEPARATOR_COUNT; // for objects, throw; for enums, return COUNT + } +} + + +void enum_to_stem_string::roundingMode(UNumberFormatRoundingMode value, UnicodeString& sb) { + switch (value) { + case UNUM_ROUND_CEILING: + sb.append(u"rounding-mode-ceiling", -1); + break; + case UNUM_ROUND_FLOOR: + sb.append(u"rounding-mode-floor", -1); + break; + case UNUM_ROUND_DOWN: + sb.append(u"rounding-mode-down", -1); + break; + case UNUM_ROUND_UP: + sb.append(u"rounding-mode-up", -1); + break; + case UNUM_ROUND_HALFEVEN: + sb.append(u"rounding-mode-half-even", -1); + break; + case UNUM_ROUND_HALF_ODD: + sb.append(u"rounding-mode-half-odd", -1); + break; + case UNUM_ROUND_HALF_CEILING: + sb.append(u"rounding-mode-half-ceiling", -1); + break; + case UNUM_ROUND_HALF_FLOOR: + sb.append(u"rounding-mode-half-floor", -1); + break; + case UNUM_ROUND_HALFDOWN: + sb.append(u"rounding-mode-half-down", -1); + break; + case UNUM_ROUND_HALFUP: + sb.append(u"rounding-mode-half-up", -1); + break; + case UNUM_ROUND_UNNECESSARY: + sb.append(u"rounding-mode-unnecessary", -1); + break; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +void enum_to_stem_string::groupingStrategy(UNumberGroupingStrategy value, UnicodeString& sb) { + switch (value) { + case UNUM_GROUPING_OFF: + sb.append(u"group-off", -1); + break; + case UNUM_GROUPING_MIN2: + sb.append(u"group-min2", -1); + break; + case UNUM_GROUPING_AUTO: + sb.append(u"group-auto", -1); + break; + case UNUM_GROUPING_ON_ALIGNED: + sb.append(u"group-on-aligned", -1); + break; + case UNUM_GROUPING_THOUSANDS: + sb.append(u"group-thousands", -1); + break; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +void enum_to_stem_string::unitWidth(UNumberUnitWidth value, UnicodeString& sb) { + switch (value) { + case UNUM_UNIT_WIDTH_NARROW: + sb.append(u"unit-width-narrow", -1); + break; + case UNUM_UNIT_WIDTH_SHORT: + sb.append(u"unit-width-short", -1); + break; + case UNUM_UNIT_WIDTH_FULL_NAME: + sb.append(u"unit-width-full-name", -1); + break; + case UNUM_UNIT_WIDTH_ISO_CODE: + sb.append(u"unit-width-iso-code", -1); + break; + case UNUM_UNIT_WIDTH_FORMAL: + sb.append(u"unit-width-formal", -1); + break; + case UNUM_UNIT_WIDTH_VARIANT: + sb.append(u"unit-width-variant", -1); + break; + case UNUM_UNIT_WIDTH_HIDDEN: + sb.append(u"unit-width-hidden", -1); + break; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +void enum_to_stem_string::signDisplay(UNumberSignDisplay value, UnicodeString& sb) { + switch (value) { + case UNUM_SIGN_AUTO: + sb.append(u"sign-auto", -1); + break; + case UNUM_SIGN_ALWAYS: + sb.append(u"sign-always", -1); + break; + case UNUM_SIGN_NEVER: + sb.append(u"sign-never", -1); + break; + case UNUM_SIGN_ACCOUNTING: + sb.append(u"sign-accounting", -1); + break; + case UNUM_SIGN_ACCOUNTING_ALWAYS: + sb.append(u"sign-accounting-always", -1); + break; + case UNUM_SIGN_EXCEPT_ZERO: + sb.append(u"sign-except-zero", -1); + break; + case UNUM_SIGN_ACCOUNTING_EXCEPT_ZERO: + sb.append(u"sign-accounting-except-zero", -1); + break; + case UNUM_SIGN_NEGATIVE: + sb.append(u"sign-negative", -1); + break; + case UNUM_SIGN_ACCOUNTING_NEGATIVE: + sb.append(u"sign-accounting-negative", -1); + break; + default: + UPRV_UNREACHABLE_EXIT; + } +} + +void +enum_to_stem_string::decimalSeparatorDisplay(UNumberDecimalSeparatorDisplay value, UnicodeString& sb) { + switch (value) { + case UNUM_DECIMAL_SEPARATOR_AUTO: + sb.append(u"decimal-auto", -1); + break; + case UNUM_DECIMAL_SEPARATOR_ALWAYS: + sb.append(u"decimal-always", -1); + break; + default: + UPRV_UNREACHABLE_EXIT; + } +} + + +UnlocalizedNumberFormatter skeleton::create( + const UnicodeString& skeletonString, UParseError* perror, UErrorCode& status) { + + // Initialize perror + if (perror != nullptr) { + perror->line = 0; + perror->offset = -1; + perror->preContext[0] = 0; + perror->postContext[0] = 0; + } + + umtx_initOnce(gNumberSkeletonsInitOnce, &initNumberSkeletons, status); + if (U_FAILURE(status)) { + return {}; + } + + int32_t errOffset; + MacroProps macros = parseSkeleton(skeletonString, errOffset, status); + if (U_SUCCESS(status)) { + return NumberFormatter::with().macros(macros); + } + + if (perror == nullptr) { + return {}; + } + + // Populate the UParseError with the error location + perror->offset = errOffset; + int32_t contextStart = uprv_max(0, errOffset - U_PARSE_CONTEXT_LEN + 1); + int32_t contextEnd = uprv_min(skeletonString.length(), errOffset + U_PARSE_CONTEXT_LEN - 1); + skeletonString.extract(contextStart, errOffset - contextStart, perror->preContext, 0); + perror->preContext[errOffset - contextStart] = 0; + skeletonString.extract(errOffset, contextEnd - errOffset, perror->postContext, 0); + perror->postContext[contextEnd - errOffset] = 0; + return {}; +} + +UnicodeString skeleton::generate(const MacroProps& macros, UErrorCode& status) { + umtx_initOnce(gNumberSkeletonsInitOnce, &initNumberSkeletons, status); + UnicodeString sb; + GeneratorHelpers::generateSkeleton(macros, sb, status); + return sb; +} + +MacroProps skeleton::parseSkeleton( + const UnicodeString& skeletonString, int32_t& errOffset, UErrorCode& status) { + U_ASSERT(U_SUCCESS(status)); + U_ASSERT(kSerializedStemTrie != nullptr); + + // Add a trailing whitespace to the end of the skeleton string to make code cleaner. + UnicodeString tempSkeletonString(skeletonString); + tempSkeletonString.append(u' '); + + SeenMacroProps seen; + MacroProps macros; + StringSegment segment(tempSkeletonString, false); + UCharsTrie stemTrie(kSerializedStemTrie); + ParseState stem = STATE_NULL; + int32_t offset = 0; + + // Primary skeleton parse loop: + while (offset < segment.length()) { + UChar32 cp = segment.codePointAt(offset); + bool isTokenSeparator = PatternProps::isWhiteSpace(cp); + bool isOptionSeparator = (cp == u'/'); + + if (!isTokenSeparator && !isOptionSeparator) { + // Non-separator token; consume it. + offset += U16_LENGTH(cp); + if (stem == STATE_NULL) { + // We are currently consuming a stem. + // Go to the next state in the stem trie. + stemTrie.nextForCodePoint(cp); + } + continue; + } + + // We are looking at a token or option separator. + // If the segment is nonempty, parse it and reset the segment. + // Otherwise, make sure it is a valid repeating separator. + if (offset != 0) { + segment.setLength(offset); + if (stem == STATE_NULL) { + // The first separator after the start of a token. Parse it as a stem. + stem = parseStem(segment, stemTrie, seen, macros, status); + stemTrie.reset(); + } else { + // A separator after the first separator of a token. Parse it as an option. + stem = parseOption(stem, segment, macros, status); + } + segment.resetLength(); + if (U_FAILURE(status)) { + errOffset = segment.getOffset(); + return macros; + } + + // Consume the segment: + segment.adjustOffset(offset); + offset = 0; + + } else if (stem != STATE_NULL) { + // A separator ('/' or whitespace) following an option separator ('/') + // segment.setLength(U16_LENGTH(cp)); // for error message + // throw new SkeletonSyntaxException("Unexpected separator character", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + errOffset = segment.getOffset(); + return macros; + + } else { + // Two spaces in a row; this is OK. + } + + // Does the current stem forbid options? + if (isOptionSeparator && stem == STATE_NULL) { + // segment.setLength(U16_LENGTH(cp)); // for error message + // throw new SkeletonSyntaxException("Unexpected option separator", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + errOffset = segment.getOffset(); + return macros; + } + + // Does the current stem require an option? + if (isTokenSeparator && stem != STATE_NULL) { + switch (stem) { + case STATE_INCREMENT_PRECISION: + case STATE_MEASURE_UNIT: + case STATE_PER_MEASURE_UNIT: + case STATE_IDENTIFIER_UNIT: + case STATE_UNIT_USAGE: + case STATE_CURRENCY_UNIT: + case STATE_INTEGER_WIDTH: + case STATE_NUMBERING_SYSTEM: + case STATE_SCALE: + // segment.setLength(U16_LENGTH(cp)); // for error message + // throw new SkeletonSyntaxException("Stem requires an option", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + errOffset = segment.getOffset(); + return macros; + default: + break; + } + stem = STATE_NULL; + } + + // Consume the separator: + segment.adjustOffset(U16_LENGTH(cp)); + } + U_ASSERT(stem == STATE_NULL); + return macros; +} + +ParseState +skeleton::parseStem(const StringSegment& segment, const UCharsTrie& stemTrie, SeenMacroProps& seen, + MacroProps& macros, UErrorCode& status) { + U_ASSERT(U_SUCCESS(status)); + + // First check for "blueprint" stems, which start with a "signal char" + switch (segment.charAt(0)) { + case u'.': + CHECK_NULL(seen, precision, status); + blueprint_helpers::parseFractionStem(segment, macros, status); + return STATE_FRACTION_PRECISION; + case u'@': + CHECK_NULL(seen, precision, status); + blueprint_helpers::parseDigitsStem(segment, macros, status); + return STATE_PRECISION; + case u'E': + CHECK_NULL(seen, notation, status); + blueprint_helpers::parseScientificStem(segment, macros, status); + return STATE_NULL; + case u'0': + CHECK_NULL(seen, integerWidth, status); + blueprint_helpers::parseIntegerStem(segment, macros, status); + return STATE_NULL; + default: + break; + } + + // Now look at the stemsTrie, which is already be pointing at our stem. + UStringTrieResult stemResult = stemTrie.current(); + + if (stemResult != USTRINGTRIE_INTERMEDIATE_VALUE && stemResult != USTRINGTRIE_FINAL_VALUE) { + // throw new SkeletonSyntaxException("Unknown stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return STATE_NULL; + } + + auto stem = static_cast(stemTrie.getValue()); + switch (stem) { + + // Stems with meaning on their own, not requiring an option: + + case STEM_COMPACT_SHORT: + case STEM_COMPACT_LONG: + case STEM_SCIENTIFIC: + case STEM_ENGINEERING: + case STEM_NOTATION_SIMPLE: + CHECK_NULL(seen, notation, status); + macros.notation = stem_to_object::notation(stem); + switch (stem) { + case STEM_SCIENTIFIC: + case STEM_ENGINEERING: + return STATE_SCIENTIFIC; // allows for scientific options + default: + return STATE_NULL; + } + + case STEM_BASE_UNIT: + case STEM_PERCENT: + case STEM_PERMILLE: + CHECK_NULL(seen, unit, status); + macros.unit = stem_to_object::unit(stem); + return STATE_NULL; + + case STEM_PERCENT_100: + CHECK_NULL(seen, scale, status); + CHECK_NULL(seen, unit, status); + macros.scale = Scale::powerOfTen(2); + macros.unit = NoUnit::percent(); + return STATE_NULL; + + case STEM_PRECISION_INTEGER: + case STEM_PRECISION_UNLIMITED: + case STEM_PRECISION_CURRENCY_STANDARD: + case STEM_PRECISION_CURRENCY_CASH: + CHECK_NULL(seen, precision, status); + macros.precision = stem_to_object::precision(stem); + switch (stem) { + case STEM_PRECISION_INTEGER: + return STATE_FRACTION_PRECISION; // allows for "precision-integer/@##" + default: + return STATE_PRECISION; + } + + case STEM_ROUNDING_MODE_CEILING: + case STEM_ROUNDING_MODE_FLOOR: + case STEM_ROUNDING_MODE_DOWN: + case STEM_ROUNDING_MODE_UP: + case STEM_ROUNDING_MODE_HALF_EVEN: + case STEM_ROUNDING_MODE_HALF_ODD: + case STEM_ROUNDING_MODE_HALF_CEILING: + case STEM_ROUNDING_MODE_HALF_FLOOR: + case STEM_ROUNDING_MODE_HALF_DOWN: + case STEM_ROUNDING_MODE_HALF_UP: + case STEM_ROUNDING_MODE_UNNECESSARY: + CHECK_NULL(seen, roundingMode, status); + macros.roundingMode = stem_to_object::roundingMode(stem); + return STATE_NULL; + + case STEM_INTEGER_WIDTH_TRUNC: + CHECK_NULL(seen, integerWidth, status); + macros.integerWidth = IntegerWidth::zeroFillTo(0).truncateAt(0); + return STATE_NULL; + + case STEM_GROUP_OFF: + case STEM_GROUP_MIN2: + case STEM_GROUP_AUTO: + case STEM_GROUP_ON_ALIGNED: + case STEM_GROUP_THOUSANDS: + CHECK_NULL(seen, grouper, status); + macros.grouper = Grouper::forStrategy(stem_to_object::groupingStrategy(stem)); + return STATE_NULL; + + case STEM_LATIN: + CHECK_NULL(seen, symbols, status); + macros.symbols.setTo(NumberingSystem::createInstanceByName("latn", status)); + return STATE_NULL; + + case STEM_UNIT_WIDTH_NARROW: + case STEM_UNIT_WIDTH_SHORT: + case STEM_UNIT_WIDTH_FULL_NAME: + case STEM_UNIT_WIDTH_ISO_CODE: + case STEM_UNIT_WIDTH_FORMAL: + case STEM_UNIT_WIDTH_VARIANT: + case STEM_UNIT_WIDTH_HIDDEN: + CHECK_NULL(seen, unitWidth, status); + macros.unitWidth = stem_to_object::unitWidth(stem); + return STATE_NULL; + + case STEM_SIGN_AUTO: + case STEM_SIGN_ALWAYS: + case STEM_SIGN_NEVER: + case STEM_SIGN_ACCOUNTING: + case STEM_SIGN_ACCOUNTING_ALWAYS: + case STEM_SIGN_EXCEPT_ZERO: + case STEM_SIGN_ACCOUNTING_EXCEPT_ZERO: + case STEM_SIGN_NEGATIVE: + case STEM_SIGN_ACCOUNTING_NEGATIVE: + CHECK_NULL(seen, sign, status); + macros.sign = stem_to_object::signDisplay(stem); + return STATE_NULL; + + case STEM_DECIMAL_AUTO: + case STEM_DECIMAL_ALWAYS: + CHECK_NULL(seen, decimal, status); + macros.decimal = stem_to_object::decimalSeparatorDisplay(stem); + return STATE_NULL; + + // Stems requiring an option: + + case STEM_PRECISION_INCREMENT: + CHECK_NULL(seen, precision, status); + return STATE_INCREMENT_PRECISION; + + case STEM_MEASURE_UNIT: + CHECK_NULL(seen, unit, status); + return STATE_MEASURE_UNIT; + + case STEM_PER_MEASURE_UNIT: + CHECK_NULL(seen, perUnit, status); + return STATE_PER_MEASURE_UNIT; + + case STEM_UNIT: + CHECK_NULL(seen, unit, status); + CHECK_NULL(seen, perUnit, status); + return STATE_IDENTIFIER_UNIT; + + case STEM_UNIT_USAGE: + CHECK_NULL(seen, usage, status); + return STATE_UNIT_USAGE; + + case STEM_CURRENCY: + CHECK_NULL(seen, unit, status); + CHECK_NULL(seen, perUnit, status); + return STATE_CURRENCY_UNIT; + + case STEM_INTEGER_WIDTH: + CHECK_NULL(seen, integerWidth, status); + return STATE_INTEGER_WIDTH; + + case STEM_NUMBERING_SYSTEM: + CHECK_NULL(seen, symbols, status); + return STATE_NUMBERING_SYSTEM; + + case STEM_SCALE: + CHECK_NULL(seen, scale, status); + return STATE_SCALE; + + default: + UPRV_UNREACHABLE_EXIT; + } +} + +ParseState skeleton::parseOption(ParseState stem, const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + U_ASSERT(U_SUCCESS(status)); + + ///// Required options: ///// + + switch (stem) { + case STATE_CURRENCY_UNIT: + blueprint_helpers::parseCurrencyOption(segment, macros, status); + return STATE_NULL; + case STATE_MEASURE_UNIT: + blueprint_helpers::parseMeasureUnitOption(segment, macros, status); + return STATE_NULL; + case STATE_PER_MEASURE_UNIT: + blueprint_helpers::parseMeasurePerUnitOption(segment, macros, status); + return STATE_NULL; + case STATE_IDENTIFIER_UNIT: + blueprint_helpers::parseIdentifierUnitOption(segment, macros, status); + return STATE_NULL; + case STATE_UNIT_USAGE: + blueprint_helpers::parseUnitUsageOption(segment, macros, status); + return STATE_NULL; + case STATE_INCREMENT_PRECISION: + blueprint_helpers::parseIncrementOption(segment, macros, status); + return STATE_PRECISION; + case STATE_INTEGER_WIDTH: + blueprint_helpers::parseIntegerWidthOption(segment, macros, status); + return STATE_NULL; + case STATE_NUMBERING_SYSTEM: + blueprint_helpers::parseNumberingSystemOption(segment, macros, status); + return STATE_NULL; + case STATE_SCALE: + blueprint_helpers::parseScaleOption(segment, macros, status); + return STATE_NULL; + default: + break; + } + + ///// Non-required options: ///// + + // Scientific options + switch (stem) { + case STATE_SCIENTIFIC: + if (blueprint_helpers::parseExponentWidthOption(segment, macros, status)) { + return STATE_SCIENTIFIC; + } + if (U_FAILURE(status)) { + return {}; + } + if (blueprint_helpers::parseExponentSignOption(segment, macros, status)) { + return STATE_SCIENTIFIC; + } + if (U_FAILURE(status)) { + return {}; + } + break; + default: + break; + } + + // Frac-sig option + switch (stem) { + case STATE_FRACTION_PRECISION: + if (blueprint_helpers::parseFracSigOption(segment, macros, status)) { + return STATE_PRECISION; + } + if (U_FAILURE(status)) { + return {}; + } + // If the fracSig option was not found, try normal precision options. + stem = STATE_PRECISION; + break; + default: + break; + } + + // Trailing zeros option + switch (stem) { + case STATE_PRECISION: + if (blueprint_helpers::parseTrailingZeroOption(segment, macros, status)) { + return STATE_NULL; + } + if (U_FAILURE(status)) { + return {}; + } + break; + default: + break; + } + + // Unknown option + // throw new SkeletonSyntaxException("Invalid option", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return STATE_NULL; +} + +void GeneratorHelpers::generateSkeleton(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + + // Supported options + if (GeneratorHelpers::notation(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::unit(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::usage(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::precision(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::roundingMode(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::grouping(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::integerWidth(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::symbols(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::unitWidth(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::sign(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::decimal(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + if (GeneratorHelpers::scale(macros, sb, status)) { + sb.append(u' '); + } + if (U_FAILURE(status)) { return; } + + // Unsupported options + if (!macros.padder.isBogus()) { + status = U_UNSUPPORTED_ERROR; + return; + } + if (macros.unitDisplayCase.isSet()) { + status = U_UNSUPPORTED_ERROR; + return; + } + if (macros.affixProvider != nullptr) { + status = U_UNSUPPORTED_ERROR; + return; + } + if (macros.rules != nullptr) { + status = U_UNSUPPORTED_ERROR; + return; + } + + // Remove the trailing space + if (sb.length() > 0) { + sb.truncate(sb.length() - 1); + } +} + + +bool blueprint_helpers::parseExponentWidthOption(const StringSegment& segment, MacroProps& macros, + UErrorCode&) { + if (!isWildcardChar(segment.charAt(0))) { + return false; + } + int32_t offset = 1; + int32_t minExp = 0; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'e') { + minExp++; + } else { + break; + } + } + if (offset < segment.length()) { + return false; + } + // Use the public APIs to enforce bounds checking + macros.notation = static_cast(macros.notation).withMinExponentDigits(minExp); + return true; +} + +void +blueprint_helpers::generateExponentWidthOption(int32_t minExponentDigits, UnicodeString& sb, UErrorCode&) { + sb.append(kWildcardChar); + appendMultiple(sb, u'e', minExponentDigits); +} + +bool +blueprint_helpers::parseExponentSignOption(const StringSegment& segment, MacroProps& macros, UErrorCode&) { + // Get the sign display type out of the CharsTrie data structure. + UCharsTrie tempStemTrie(kSerializedStemTrie); + UStringTrieResult result = tempStemTrie.next( + segment.toTempUnicodeString().getBuffer(), + segment.length()); + if (result != USTRINGTRIE_INTERMEDIATE_VALUE && result != USTRINGTRIE_FINAL_VALUE) { + return false; + } + auto sign = stem_to_object::signDisplay(static_cast(tempStemTrie.getValue())); + if (sign == UNUM_SIGN_COUNT) { + return false; + } + macros.notation = static_cast(macros.notation).withExponentSignDisplay(sign); + return true; +} + +void blueprint_helpers::parseCurrencyOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + // Unlike ICU4J, have to check length manually because ICU4C CurrencyUnit does not check it for us + if (segment.length() != 3) { + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + const char16_t* currencyCode = segment.toTempUnicodeString().getBuffer(); + UErrorCode localStatus = U_ZERO_ERROR; + CurrencyUnit currency(currencyCode, localStatus); + if (U_FAILURE(localStatus)) { + // Not 3 ascii chars + // throw new SkeletonSyntaxException("Invalid currency", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + // Slicing is OK + macros.unit = currency; // NOLINT +} + +void +blueprint_helpers::generateCurrencyOption(const CurrencyUnit& currency, UnicodeString& sb, UErrorCode&) { + sb.append(currency.getISOCurrency(), -1); +} + +void blueprint_helpers::parseMeasureUnitOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + U_ASSERT(U_SUCCESS(status)); + const UnicodeString stemString = segment.toTempUnicodeString(); + + // NOTE: The category (type) of the unit is guaranteed to be a valid subtag (alphanumeric) + // http://unicode.org/reports/tr35/#Validity_Data + int firstHyphen = 0; + while (firstHyphen < stemString.length() && stemString.charAt(firstHyphen) != '-') { + firstHyphen++; + } + if (firstHyphen == stemString.length()) { + // throw new SkeletonSyntaxException("Invalid measure unit option", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + + // Need to do char <-> char16_t conversion... + CharString type; + SKELETON_UCHAR_TO_CHAR(type, stemString, 0, firstHyphen, status); + CharString subType; + SKELETON_UCHAR_TO_CHAR(subType, stemString, firstHyphen + 1, stemString.length(), status); + + // Note: the largest type as of this writing (Aug 2020) is "volume", which has 33 units. + static constexpr int32_t CAPACITY = 40; + MeasureUnit units[CAPACITY]; + UErrorCode localStatus = U_ZERO_ERROR; + int32_t numUnits = MeasureUnit::getAvailable(type.data(), units, CAPACITY, localStatus); + if (U_FAILURE(localStatus)) { + // More than 30 units in this type? + status = U_INTERNAL_PROGRAM_ERROR; + return; + } + for (int32_t i = 0; i < numUnits; i++) { + auto& unit = units[i]; + if (uprv_strcmp(subType.data(), unit.getSubtype()) == 0) { + macros.unit = unit; + return; + } + } + + // throw new SkeletonSyntaxException("Unknown measure unit", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; +} + +void blueprint_helpers::parseMeasurePerUnitOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + // A little bit of a hack: save the current unit (numerator), call the main measure unit + // parsing code, put back the numerator unit, and put the new unit into per-unit. + MeasureUnit numerator = macros.unit; + parseMeasureUnitOption(segment, macros, status); + if (U_FAILURE(status)) { return; } + macros.perUnit = macros.unit; + macros.unit = numerator; +} + +void blueprint_helpers::parseIdentifierUnitOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + // Need to do char <-> char16_t conversion... + U_ASSERT(U_SUCCESS(status)); + CharString buffer; + SKELETON_UCHAR_TO_CHAR(buffer, segment.toTempUnicodeString(), 0, segment.length(), status); + + ErrorCode internalStatus; + macros.unit = MeasureUnit::forIdentifier(buffer.toStringPiece(), internalStatus); + if (internalStatus.isFailure()) { + // throw new SkeletonSyntaxException("Invalid core unit identifier", segment, e); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } +} + +void blueprint_helpers::parseUnitUsageOption(const StringSegment &segment, MacroProps ¯os, + UErrorCode &status) { + // Need to do char <-> char16_t conversion... + U_ASSERT(U_SUCCESS(status)); + CharString buffer; + SKELETON_UCHAR_TO_CHAR(buffer, segment.toTempUnicodeString(), 0, segment.length(), status); + macros.usage.set(buffer.toStringPiece()); + // We do not do any validation of the usage string: it depends on the + // unitPreferenceData in the units resources. +} + +void blueprint_helpers::parseFractionStem(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + U_ASSERT(segment.charAt(0) == u'.'); + int32_t offset = 1; + int32_t minFrac = 0; + int32_t maxFrac; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'0') { + minFrac++; + } else { + break; + } + } + if (offset < segment.length()) { + if (isWildcardChar(segment.charAt(offset))) { + maxFrac = -1; + offset++; + } else { + maxFrac = minFrac; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'#') { + maxFrac++; + } else { + break; + } + } + } + } else { + maxFrac = minFrac; + } + if (offset < segment.length()) { + // throw new SkeletonSyntaxException("Invalid fraction stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + // Use the public APIs to enforce bounds checking + if (maxFrac == -1) { + if (minFrac == 0) { + macros.precision = Precision::unlimited(); + } else { + macros.precision = Precision::minFraction(minFrac); + } + } else { + macros.precision = Precision::minMaxFraction(minFrac, maxFrac); + } +} + +void +blueprint_helpers::generateFractionStem(int32_t minFrac, int32_t maxFrac, UnicodeString& sb, UErrorCode&) { + if (minFrac == 0 && maxFrac == 0) { + sb.append(u"precision-integer", -1); + return; + } + sb.append(u'.'); + appendMultiple(sb, u'0', minFrac); + if (maxFrac == -1) { + sb.append(kWildcardChar); + } else { + appendMultiple(sb, u'#', maxFrac - minFrac); + } +} + +void +blueprint_helpers::parseDigitsStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status) { + U_ASSERT(segment.charAt(0) == u'@'); + int32_t offset = 0; + int32_t minSig = 0; + int32_t maxSig; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'@') { + minSig++; + } else { + break; + } + } + if (offset < segment.length()) { + if (isWildcardChar(segment.charAt(offset))) { + maxSig = -1; + offset++; + } else { + maxSig = minSig; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'#') { + maxSig++; + } else { + break; + } + } + } + } else { + maxSig = minSig; + } + if (offset < segment.length()) { + // throw new SkeletonSyntaxException("Invalid significant digits stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + // Use the public APIs to enforce bounds checking + if (maxSig == -1) { + macros.precision = Precision::minSignificantDigits(minSig); + } else { + macros.precision = Precision::minMaxSignificantDigits(minSig, maxSig); + } +} + +void +blueprint_helpers::generateDigitsStem(int32_t minSig, int32_t maxSig, UnicodeString& sb, UErrorCode&) { + appendMultiple(sb, u'@', minSig); + if (maxSig == -1) { + sb.append(kWildcardChar); + } else { + appendMultiple(sb, u'#', maxSig - minSig); + } +} + +void blueprint_helpers::parseScientificStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status) { + U_ASSERT(segment.charAt(0) == u'E'); + { + int32_t offset = 1; + if (segment.length() == offset) { + goto fail; + } + bool isEngineering = false; + if (segment.charAt(offset) == u'E') { + isEngineering = true; + offset++; + if (segment.length() == offset) { + goto fail; + } + } + UNumberSignDisplay signDisplay = UNUM_SIGN_AUTO; + if (segment.charAt(offset) == u'+') { + offset++; + if (segment.length() == offset) { + goto fail; + } + if (segment.charAt(offset) == u'!') { + signDisplay = UNUM_SIGN_ALWAYS; + } else if (segment.charAt(offset) == u'?') { + signDisplay = UNUM_SIGN_EXCEPT_ZERO; + } else { + // NOTE: Other sign displays are not included because they aren't useful in this context + goto fail; + } + offset++; + if (segment.length() == offset) { + goto fail; + } + } + int32_t minDigits = 0; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) != u'0') { + goto fail; + } + minDigits++; + } + macros.notation = (isEngineering ? Notation::engineering() : Notation::scientific()) + .withExponentSignDisplay(signDisplay) + .withMinExponentDigits(minDigits); + return; + } + fail: void(); + // throw new SkeletonSyntaxException("Invalid scientific stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; +} + +void blueprint_helpers::parseIntegerStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status) { + U_ASSERT(segment.charAt(0) == u'0'); + int32_t offset = 1; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) != u'0') { + offset--; + break; + } + } + if (offset < segment.length()) { + // throw new SkeletonSyntaxException("Invalid integer stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + macros.integerWidth = IntegerWidth::zeroFillTo(offset); + return; +} + +bool blueprint_helpers::parseFracSigOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + if (segment.charAt(0) != u'@') { + return false; + } + int offset = 0; + int minSig = 0; + int maxSig; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'@') { + minSig++; + } else { + break; + } + } + if (offset < segment.length()) { + if (isWildcardChar(segment.charAt(offset))) { + // @+, @@+, @@@+ + maxSig = -1; + offset++; + } else { + // @#, @##, @### + // @@#, @@##, @@@# + maxSig = minSig; + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'#') { + maxSig++; + } else { + break; + } + } + } + } else { + // @, @@, @@@ + maxSig = minSig; + } + auto& oldPrecision = static_cast(macros.precision); + if (offset < segment.length()) { + UNumberRoundingPriority priority; + if (maxSig == -1) { + // The wildcard character is not allowed with the priority annotation + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return false; + } + if (segment.codePointAt(offset) == u'r') { + priority = UNUM_ROUNDING_PRIORITY_RELAXED; + offset++; + } else if (segment.codePointAt(offset) == u's') { + priority = UNUM_ROUNDING_PRIORITY_STRICT; + offset++; + } else { + // Invalid digits option for fraction rounder + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return false; + } + if (offset < segment.length()) { + // Invalid digits option for fraction rounder + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return false; + } + macros.precision = oldPrecision.withSignificantDigits(minSig, maxSig, priority); + } else if (maxSig == -1) { + // withMinDigits + macros.precision = oldPrecision.withMinDigits(minSig); + } else if (minSig == 1) { + // withMaxDigits + macros.precision = oldPrecision.withMaxDigits(maxSig); + } else { + // Digits options with both min and max sig require the priority option + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return false; + } + + return true; +} + +bool blueprint_helpers::parseTrailingZeroOption(const StringSegment& segment, MacroProps& macros, UErrorCode&) { + if (segment == u"w") { + macros.precision = macros.precision.trailingZeroDisplay(UNUM_TRAILING_ZERO_HIDE_IF_WHOLE); + return true; + } + return false; +} + +void blueprint_helpers::parseIncrementOption(const StringSegment &segment, MacroProps ¯os, + UErrorCode &status) { + number::impl::parseIncrementOption(segment, macros.precision, status); +} + +void blueprint_helpers::generateIncrementOption( + uint32_t increment, + digits_t incrementMagnitude, + int32_t minFrac, + UnicodeString& sb, + UErrorCode&) { + // Utilize DecimalQuantity/double_conversion to format this for us. + DecimalQuantity dq; + dq.setToLong(increment); + dq.adjustMagnitude(incrementMagnitude); + dq.setMinFraction(minFrac); + sb.append(dq.toPlainString()); +} + +void blueprint_helpers::parseIntegerWidthOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + int32_t offset = 0; + int32_t minInt = 0; + int32_t maxInt; + if (isWildcardChar(segment.charAt(0))) { + maxInt = -1; + offset++; + } else { + maxInt = 0; + } + for (; offset < segment.length(); offset++) { + if (maxInt != -1 && segment.charAt(offset) == u'#') { + maxInt++; + } else { + break; + } + } + if (offset < segment.length()) { + for (; offset < segment.length(); offset++) { + if (segment.charAt(offset) == u'0') { + minInt++; + } else { + break; + } + } + } + if (maxInt != -1) { + maxInt += minInt; + } + if (offset < segment.length()) { + // throw new SkeletonSyntaxException("Invalid integer width stem", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + // Use the public APIs to enforce bounds checking + if (maxInt == -1) { + macros.integerWidth = IntegerWidth::zeroFillTo(minInt); + } else { + macros.integerWidth = IntegerWidth::zeroFillTo(minInt).truncateAt(maxInt); + } +} + +void blueprint_helpers::generateIntegerWidthOption(int32_t minInt, int32_t maxInt, UnicodeString& sb, + UErrorCode&) { + if (maxInt == -1) { + sb.append(kWildcardChar); + } else { + appendMultiple(sb, u'#', maxInt - minInt); + } + appendMultiple(sb, u'0', minInt); +} + +void blueprint_helpers::parseNumberingSystemOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + // Need to do char <-> char16_t conversion... + U_ASSERT(U_SUCCESS(status)); + CharString buffer; + SKELETON_UCHAR_TO_CHAR(buffer, segment.toTempUnicodeString(), 0, segment.length(), status); + + NumberingSystem* ns = NumberingSystem::createInstanceByName(buffer.data(), status); + if (ns == nullptr || U_FAILURE(status)) { + // This is a skeleton syntax error; don't bubble up the low-level NumberingSystem error + // throw new SkeletonSyntaxException("Unknown numbering system", segment); + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + macros.symbols.setTo(ns); +} + +void blueprint_helpers::generateNumberingSystemOption(const NumberingSystem& ns, UnicodeString& sb, + UErrorCode&) { + // Need to do char <-> char16_t conversion... + sb.append(UnicodeString(ns.getName(), -1, US_INV)); +} + +void blueprint_helpers::parseScaleOption(const StringSegment& segment, MacroProps& macros, + UErrorCode& status) { + // Need to do char <-> char16_t conversion... + U_ASSERT(U_SUCCESS(status)); + CharString buffer; + SKELETON_UCHAR_TO_CHAR(buffer, segment.toTempUnicodeString(), 0, segment.length(), status); + + LocalPointer decnum(new DecNum(), status); + if (U_FAILURE(status)) { return; } + decnum->setTo({buffer.data(), buffer.length()}, status); + if (U_FAILURE(status) || decnum->isSpecial()) { + // This is a skeleton syntax error; don't let the low-level decnum error bubble up + status = U_NUMBER_SKELETON_SYNTAX_ERROR; + return; + } + + // NOTE: The constructor will optimize the decnum for us if possible. + macros.scale = {0, decnum.orphan()}; +} + +void blueprint_helpers::generateScaleOption(int32_t magnitude, const DecNum* arbitrary, UnicodeString& sb, + UErrorCode& status) { + // Utilize DecimalQuantity/double_conversion to format this for us. + DecimalQuantity dq; + if (arbitrary != nullptr) { + dq.setToDecNum(*arbitrary, status); + if (U_FAILURE(status)) { return; } + } else { + dq.setToInt(1); + } + dq.adjustMagnitude(magnitude); + dq.roundToInfinity(); + sb.append(dq.toPlainString()); +} + + +bool GeneratorHelpers::notation(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (macros.notation.fType == Notation::NTN_COMPACT) { + UNumberCompactStyle style = macros.notation.fUnion.compactStyle; + if (style == UNumberCompactStyle::UNUM_LONG) { + sb.append(u"compact-long", -1); + return true; + } else if (style == UNumberCompactStyle::UNUM_SHORT) { + sb.append(u"compact-short", -1); + return true; + } else { + // Compact notation generated from custom data (not supported in skeleton) + // The other compact notations are literals + status = U_UNSUPPORTED_ERROR; + return false; + } + } else if (macros.notation.fType == Notation::NTN_SCIENTIFIC) { + const Notation::ScientificSettings& impl = macros.notation.fUnion.scientific; + if (impl.fEngineeringInterval == 3) { + sb.append(u"engineering", -1); + } else { + sb.append(u"scientific", -1); + } + if (impl.fMinExponentDigits > 1) { + sb.append(u'/'); + blueprint_helpers::generateExponentWidthOption(impl.fMinExponentDigits, sb, status); + if (U_FAILURE(status)) { + return false; + } + } + if (impl.fExponentSignDisplay != UNUM_SIGN_AUTO) { + sb.append(u'/'); + enum_to_stem_string::signDisplay(impl.fExponentSignDisplay, sb); + } + return true; + } else { + // Default value is not shown in normalized form + return false; + } +} + +bool GeneratorHelpers::unit(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + MeasureUnit unit = macros.unit; + if (!utils::unitIsBaseUnit(macros.perUnit)) { + if (utils::unitIsCurrency(macros.unit) || utils::unitIsCurrency(macros.perUnit)) { + status = U_UNSUPPORTED_ERROR; + return false; + } + unit = unit.product(macros.perUnit.reciprocal(status), status); + } + + if (utils::unitIsCurrency(unit)) { + sb.append(u"currency/", -1); + CurrencyUnit currency(unit, status); + if (U_FAILURE(status)) { + return false; + } + blueprint_helpers::generateCurrencyOption(currency, sb, status); + return true; + } else if (utils::unitIsBaseUnit(unit)) { + // Default value is not shown in normalized form + return false; + } else if (utils::unitIsPercent(unit)) { + sb.append(u"percent", -1); + return true; + } else if (utils::unitIsPermille(unit)) { + sb.append(u"permille", -1); + return true; + } else { + sb.append(u"unit/", -1); + sb.append(unit.getIdentifier()); + return true; + } +} + +bool GeneratorHelpers::usage(const MacroProps& macros, UnicodeString& sb, UErrorCode& /* status */) { + if (macros.usage.isSet()) { + sb.append(u"usage/", -1); + sb.append(UnicodeString(macros.usage.fValue, -1, US_INV)); + return true; + } + return false; +} + +bool GeneratorHelpers::precision(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (macros.precision.fType == Precision::RND_NONE) { + sb.append(u"precision-unlimited", -1); + } else if (macros.precision.fType == Precision::RND_FRACTION) { + const Precision::FractionSignificantSettings& impl = macros.precision.fUnion.fracSig; + blueprint_helpers::generateFractionStem(impl.fMinFrac, impl.fMaxFrac, sb, status); + } else if (macros.precision.fType == Precision::RND_SIGNIFICANT) { + const Precision::FractionSignificantSettings& impl = macros.precision.fUnion.fracSig; + blueprint_helpers::generateDigitsStem(impl.fMinSig, impl.fMaxSig, sb, status); + } else if (macros.precision.fType == Precision::RND_FRACTION_SIGNIFICANT) { + const Precision::FractionSignificantSettings& impl = macros.precision.fUnion.fracSig; + blueprint_helpers::generateFractionStem(impl.fMinFrac, impl.fMaxFrac, sb, status); + sb.append(u'/'); + if (impl.fRetain) { + if (impl.fPriority == UNUM_ROUNDING_PRIORITY_RELAXED) { + // withMinDigits + blueprint_helpers::generateDigitsStem(impl.fMaxSig, -1, sb, status); + } else { + // withMaxDigits + blueprint_helpers::generateDigitsStem(1, impl.fMaxSig, sb, status); + } + } else { + blueprint_helpers::generateDigitsStem(impl.fMinSig, impl.fMaxSig, sb, status); + if (impl.fPriority == UNUM_ROUNDING_PRIORITY_RELAXED) { + sb.append(u'r'); + } else { + sb.append(u's'); + } + } + } else if (macros.precision.fType == Precision::RND_INCREMENT + || macros.precision.fType == Precision::RND_INCREMENT_ONE + || macros.precision.fType == Precision::RND_INCREMENT_FIVE) { + const Precision::IncrementSettings& impl = macros.precision.fUnion.increment; + sb.append(u"precision-increment/", -1); + blueprint_helpers::generateIncrementOption( + impl.fIncrement, + impl.fIncrementMagnitude, + impl.fMinFrac, + sb, + status); + } else if (macros.precision.fType == Precision::RND_CURRENCY) { + UCurrencyUsage usage = macros.precision.fUnion.currencyUsage; + if (usage == UCURR_USAGE_STANDARD) { + sb.append(u"precision-currency-standard", -1); + } else { + sb.append(u"precision-currency-cash", -1); + } + } else { + // Bogus or Error + return false; + } + + if (macros.precision.fTrailingZeroDisplay == UNUM_TRAILING_ZERO_HIDE_IF_WHOLE) { + sb.append(u"/w", -1); + } + + // NOTE: Always return true for rounding because the default value depends on other options. + return true; +} + +bool GeneratorHelpers::roundingMode(const MacroProps& macros, UnicodeString& sb, UErrorCode&) { + if (macros.roundingMode == kDefaultMode) { + return false; // Default + } + enum_to_stem_string::roundingMode(macros.roundingMode, sb); + return true; +} + +bool GeneratorHelpers::grouping(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (macros.grouper.isBogus()) { + return false; // No value + } else if (macros.grouper.fStrategy == UNUM_GROUPING_COUNT) { + status = U_UNSUPPORTED_ERROR; + return false; + } else if (macros.grouper.fStrategy == UNUM_GROUPING_AUTO) { + return false; // Default value + } else { + enum_to_stem_string::groupingStrategy(macros.grouper.fStrategy, sb); + return true; + } +} + +bool GeneratorHelpers::integerWidth(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (macros.integerWidth.fHasError || macros.integerWidth.isBogus() || + macros.integerWidth == IntegerWidth::standard()) { + // Error or Default + return false; + } + const auto& minMaxInt = macros.integerWidth.fUnion.minMaxInt; + if (minMaxInt.fMinInt == 0 && minMaxInt.fMaxInt == 0) { + sb.append(u"integer-width-trunc", -1); + return true; + } + sb.append(u"integer-width/", -1); + blueprint_helpers::generateIntegerWidthOption( + minMaxInt.fMinInt, + minMaxInt.fMaxInt, + sb, + status); + return true; +} + +bool GeneratorHelpers::symbols(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (macros.symbols.isNumberingSystem()) { + const NumberingSystem& ns = *macros.symbols.getNumberingSystem(); + if (uprv_strcmp(ns.getName(), "latn") == 0) { + sb.append(u"latin", -1); + } else { + sb.append(u"numbering-system/", -1); + blueprint_helpers::generateNumberingSystemOption(ns, sb, status); + } + return true; + } else if (macros.symbols.isDecimalFormatSymbols()) { + status = U_UNSUPPORTED_ERROR; + return false; + } else { + // No custom symbols + return false; + } +} + +bool GeneratorHelpers::unitWidth(const MacroProps& macros, UnicodeString& sb, UErrorCode&) { + if (macros.unitWidth == UNUM_UNIT_WIDTH_SHORT || macros.unitWidth == UNUM_UNIT_WIDTH_COUNT) { + return false; // Default or Bogus + } + enum_to_stem_string::unitWidth(macros.unitWidth, sb); + return true; +} + +bool GeneratorHelpers::sign(const MacroProps& macros, UnicodeString& sb, UErrorCode&) { + if (macros.sign == UNUM_SIGN_AUTO || macros.sign == UNUM_SIGN_COUNT) { + return false; // Default or Bogus + } + enum_to_stem_string::signDisplay(macros.sign, sb); + return true; +} + +bool GeneratorHelpers::decimal(const MacroProps& macros, UnicodeString& sb, UErrorCode&) { + if (macros.decimal == UNUM_DECIMAL_SEPARATOR_AUTO || macros.decimal == UNUM_DECIMAL_SEPARATOR_COUNT) { + return false; // Default or Bogus + } + enum_to_stem_string::decimalSeparatorDisplay(macros.decimal, sb); + return true; +} + +bool GeneratorHelpers::scale(const MacroProps& macros, UnicodeString& sb, UErrorCode& status) { + if (!macros.scale.isValid()) { + return false; // Default or Bogus + } + sb.append(u"scale/", -1); + blueprint_helpers::generateScaleOption( + macros.scale.fMagnitude, + macros.scale.fArbitrary, + sb, + status); + return true; +} + + +// Definitions of public API methods (put here for dependency disentanglement) + +#if (U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN) && defined(_MSC_VER) +// Ignore MSVC warning 4661. This is generated for NumberFormatterSettings<>::toSkeleton() as this method +// is defined elsewhere (in number_skeletons.cpp). The compiler is warning that the explicit template instantiation +// inside this single translation unit (CPP file) is incomplete, and thus it isn't sure if the template class is +// fully defined. However, since each translation unit explicitly instantiates all the necessary template classes, +// they will all be passed to the linker, and the linker will still find and export all the class members. +#pragma warning(push) +#pragma warning(disable: 4661) +#endif + +template +UnicodeString NumberFormatterSettings::toSkeleton(UErrorCode& status) const { + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + if (fMacros.copyErrorTo(status)) { + return ICU_Utility::makeBogusString(); + } + return skeleton::generate(fMacros, status); +} + +// Declare all classes that implement NumberFormatterSettings +// See https://stackoverflow.com/a/495056/1407170 +template +class icu::number::NumberFormatterSettings; +template +class icu::number::NumberFormatterSettings; + +UnlocalizedNumberFormatter +NumberFormatter::forSkeleton(const UnicodeString& skeleton, UErrorCode& status) { + return skeleton::create(skeleton, nullptr, status); +} + +UnlocalizedNumberFormatter +NumberFormatter::forSkeleton(const UnicodeString& skeleton, UParseError& perror, UErrorCode& status) { + return skeleton::create(skeleton, &perror, status); +} + +#if (U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN) && defined(_MSC_VER) +// Warning 4661. +#pragma warning(pop) +#endif + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_skeletons.h b/intl/icu/source/i18n/number_skeletons.h new file mode 100644 index 0000000000..27f69cd48c --- /dev/null +++ b/intl/icu/source/i18n/number_skeletons.h @@ -0,0 +1,393 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMBER_SKELETONS_H__ +#define __SOURCE_NUMBER_SKELETONS_H__ + +#include "number_types.h" +#include "numparse_types.h" +#include "unicode/ucharstrie.h" +#include "string_segment.h" + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { + +// Forward-declaration +struct SeenMacroProps; + +// namespace for enums and entrypoint functions +namespace skeleton { + +//////////////////////////////////////////////////////////////////////////////////////// +// NOTE: For examples of how to add a new stem to the number skeleton parser, see: // +// https://github.com/unicode-org/icu/commit/a2a7982216b2348070dc71093775ac7195793d73 // +// and // +// https://github.com/unicode-org/icu/commit/6fe86f3934a8a5701034f648a8f7c5087e84aa28 // +//////////////////////////////////////////////////////////////////////////////////////// + +/** + * While parsing a skeleton, this enum records what type of option we expect to find next. + */ +enum ParseState { + + // Section 0: We expect whitespace or a stem, but not an option: + + STATE_NULL, + + // Section 1: We might accept an option, but it is not required: + + STATE_SCIENTIFIC, + STATE_FRACTION_PRECISION, + STATE_PRECISION, + + // Section 2: An option is required: + + STATE_INCREMENT_PRECISION, + STATE_MEASURE_UNIT, + STATE_PER_MEASURE_UNIT, + STATE_IDENTIFIER_UNIT, + STATE_UNIT_USAGE, + STATE_CURRENCY_UNIT, + STATE_INTEGER_WIDTH, + STATE_NUMBERING_SYSTEM, + STATE_SCALE, +}; + +/** + * All possible stem literals have an entry in the StemEnum. The enum name is the kebab case stem + * string literal written in upper snake case. + * + * @see StemToObject + * @see #SERIALIZED_STEM_TRIE + */ +enum StemEnum { + + // Section 1: Stems that do not require an option: + + STEM_COMPACT_SHORT, + STEM_COMPACT_LONG, + STEM_SCIENTIFIC, + STEM_ENGINEERING, + STEM_NOTATION_SIMPLE, + STEM_BASE_UNIT, + STEM_PERCENT, + STEM_PERMILLE, + STEM_PERCENT_100, // concise-only + STEM_PRECISION_INTEGER, + STEM_PRECISION_UNLIMITED, + STEM_PRECISION_CURRENCY_STANDARD, + STEM_PRECISION_CURRENCY_CASH, + STEM_ROUNDING_MODE_CEILING, + STEM_ROUNDING_MODE_FLOOR, + STEM_ROUNDING_MODE_DOWN, + STEM_ROUNDING_MODE_UP, + STEM_ROUNDING_MODE_HALF_EVEN, + STEM_ROUNDING_MODE_HALF_ODD, + STEM_ROUNDING_MODE_HALF_CEILING, + STEM_ROUNDING_MODE_HALF_FLOOR, + STEM_ROUNDING_MODE_HALF_DOWN, + STEM_ROUNDING_MODE_HALF_UP, + STEM_ROUNDING_MODE_UNNECESSARY, + STEM_INTEGER_WIDTH_TRUNC, + STEM_GROUP_OFF, + STEM_GROUP_MIN2, + STEM_GROUP_AUTO, + STEM_GROUP_ON_ALIGNED, + STEM_GROUP_THOUSANDS, + STEM_LATIN, + STEM_UNIT_WIDTH_NARROW, + STEM_UNIT_WIDTH_SHORT, + STEM_UNIT_WIDTH_FULL_NAME, + STEM_UNIT_WIDTH_ISO_CODE, + STEM_UNIT_WIDTH_FORMAL, + STEM_UNIT_WIDTH_VARIANT, + STEM_UNIT_WIDTH_HIDDEN, + STEM_SIGN_AUTO, + STEM_SIGN_ALWAYS, + STEM_SIGN_NEVER, + STEM_SIGN_ACCOUNTING, + STEM_SIGN_ACCOUNTING_ALWAYS, + STEM_SIGN_EXCEPT_ZERO, + STEM_SIGN_ACCOUNTING_EXCEPT_ZERO, + STEM_SIGN_NEGATIVE, + STEM_SIGN_ACCOUNTING_NEGATIVE, + STEM_DECIMAL_AUTO, + STEM_DECIMAL_ALWAYS, + + // Section 2: Stems that DO require an option: + + STEM_PRECISION_INCREMENT, + STEM_MEASURE_UNIT, + STEM_PER_MEASURE_UNIT, + STEM_UNIT, + STEM_UNIT_USAGE, + STEM_CURRENCY, + STEM_INTEGER_WIDTH, + STEM_NUMBERING_SYSTEM, + STEM_SCALE, +}; + +/** Default wildcard char, accepted on input and printed in output */ +constexpr char16_t kWildcardChar = u'*'; + +/** Alternative wildcard char, accept on input but not printed in output */ +constexpr char16_t kAltWildcardChar = u'+'; + +/** Checks whether the char is a wildcard on input */ +inline bool isWildcardChar(char16_t c) { + return c == kWildcardChar || c == kAltWildcardChar; +} + +/** + * Creates a NumberFormatter corresponding to the given skeleton string. + * + * @param skeletonString + * A number skeleton string, possibly not in its shortest form. + * @return An UnlocalizedNumberFormatter with behavior defined by the given skeleton string. + */ +UnlocalizedNumberFormatter create( + const UnicodeString& skeletonString, UParseError* perror, UErrorCode& status); + +/** + * Create a skeleton string corresponding to the given NumberFormatter. + * + * @param macros + * The NumberFormatter options object. + * @return A skeleton string in normalized form. + */ +UnicodeString generate(const MacroProps& macros, UErrorCode& status); + +/** + * Converts from a skeleton string to a MacroProps. This method contains the primary parse loop. + * + * Internal: use the create() endpoint instead of this function. + */ +MacroProps parseSkeleton(const UnicodeString& skeletonString, int32_t& errOffset, UErrorCode& status); + +/** + * Given that the current segment represents a stem, parse it and save the result. + * + * @return The next state after parsing this stem, corresponding to what subset of options to expect. + */ +ParseState parseStem(const StringSegment& segment, const UCharsTrie& stemTrie, SeenMacroProps& seen, + MacroProps& macros, UErrorCode& status); + +/** + * Given that the current segment represents an option, parse it and save the result. + * + * @return The next state after parsing this option, corresponding to what subset of options to + * expect next. + */ +ParseState +parseOption(ParseState stem, const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +} // namespace skeleton + + +/** + * Namespace for utility methods that convert from StemEnum to corresponding objects or enums. This + * applies to only the "Section 1" stems, those that are well-defined without an option. + */ +namespace stem_to_object { + +Notation notation(skeleton::StemEnum stem); + +MeasureUnit unit(skeleton::StemEnum stem); + +Precision precision(skeleton::StemEnum stem); + +UNumberFormatRoundingMode roundingMode(skeleton::StemEnum stem); + +UNumberGroupingStrategy groupingStrategy(skeleton::StemEnum stem); + +UNumberUnitWidth unitWidth(skeleton::StemEnum stem); + +UNumberSignDisplay signDisplay(skeleton::StemEnum stem); + +UNumberDecimalSeparatorDisplay decimalSeparatorDisplay(skeleton::StemEnum stem); + +} // namespace stem_to_object + +/** + * Namespace for utility methods that convert from enums to stem strings. More complex object conversions + * take place in the object_to_stem_string namespace. + */ +namespace enum_to_stem_string { + +void roundingMode(UNumberFormatRoundingMode value, UnicodeString& sb); + +void groupingStrategy(UNumberGroupingStrategy value, UnicodeString& sb); + +void unitWidth(UNumberUnitWidth value, UnicodeString& sb); + +void signDisplay(UNumberSignDisplay value, UnicodeString& sb); + +void decimalSeparatorDisplay(UNumberDecimalSeparatorDisplay value, UnicodeString& sb); + +} // namespace enum_to_stem_string + +/** + * Namespace for utility methods for processing stems and options that cannot be interpreted literally. + */ +namespace blueprint_helpers { + +/** @return Whether we successfully found and parsed an exponent width option. */ +bool parseExponentWidthOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateExponentWidthOption(int32_t minExponentDigits, UnicodeString& sb, UErrorCode& status); + +/** @return Whether we successfully found and parsed an exponent sign option. */ +bool parseExponentSignOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void parseCurrencyOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateCurrencyOption(const CurrencyUnit& currency, UnicodeString& sb, UErrorCode& status); + +// "measure-unit/" is deprecated in favour of "unit/". +void parseMeasureUnitOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +// "per-measure-unit/" is deprecated in favour of "unit/". +void parseMeasurePerUnitOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +/** + * Parses unit identifiers like "meter-per-second" and "foot-and-inch", as + * specified via a "unit/" concise skeleton. + */ +void parseIdentifierUnitOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void parseUnitUsageOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void parseFractionStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateFractionStem(int32_t minFrac, int32_t maxFrac, UnicodeString& sb, UErrorCode& status); + +void parseDigitsStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateDigitsStem(int32_t minSig, int32_t maxSig, UnicodeString& sb, UErrorCode& status); + +void parseScientificStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +// Note: no generateScientificStem since this syntax was added later in ICU 67 + +void parseIntegerStem(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +// Note: no generateIntegerStem since this syntax was added later in ICU 67 + +/** @return Whether we successfully found and parsed a frac-sig option. */ +bool parseFracSigOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +/** @return Whether we successfully found and parsed a trailing zero option. */ +bool parseTrailingZeroOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void parseIncrementOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void +generateIncrementOption(uint32_t increment, digits_t incrementMagnitude, int32_t minFrac, UnicodeString& sb, UErrorCode& status); + +void parseIntegerWidthOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateIntegerWidthOption(int32_t minInt, int32_t maxInt, UnicodeString& sb, UErrorCode& status); + +void parseNumberingSystemOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateNumberingSystemOption(const NumberingSystem& ns, UnicodeString& sb, UErrorCode& status); + +void parseScaleOption(const StringSegment& segment, MacroProps& macros, UErrorCode& status); + +void generateScaleOption(int32_t magnitude, const DecNum* arbitrary, UnicodeString& sb, + UErrorCode& status); + +} // namespace blueprint_helpers + +/** + * Class for utility methods for generating a token corresponding to each macro-prop. Each method + * returns whether or not a token was written to the string builder. + * + * This needs to be a class, not a namespace, so it can be friended. + */ +class GeneratorHelpers { + public: + /** + * Main skeleton generator function. Appends the normalized skeleton for the MacroProps to the given + * StringBuilder. + * + * Internal: use the create() endpoint instead of this function. + */ + static void generateSkeleton(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + private: + static bool notation(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool unit(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool usage(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool precision(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool roundingMode(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool grouping(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool integerWidth(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool symbols(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool unitWidth(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool sign(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool decimal(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + + static bool scale(const MacroProps& macros, UnicodeString& sb, UErrorCode& status); + +}; + +/** + * Struct for null-checking. + * In Java, we can just check the object reference. In C++, we need a different method. + */ +struct SeenMacroProps { + bool notation = false; + bool unit = false; + bool perUnit = false; + bool usage = false; + bool precision = false; + bool roundingMode = false; + bool grouper = false; + bool padder = false; + bool integerWidth = false; + bool symbols = false; + bool unitWidth = false; + bool sign = false; + bool decimal = false; + bool scale = false; +}; + +namespace { + +#define SKELETON_UCHAR_TO_CHAR(dest, src, start, end, status) (void)(dest); \ +UPRV_BLOCK_MACRO_BEGIN { \ + UErrorCode conversionStatus = U_ZERO_ERROR; \ + (dest).appendInvariantChars({false, (src).getBuffer() + (start), (end) - (start)}, conversionStatus); \ + if (conversionStatus == U_INVARIANT_CONVERSION_ERROR) { \ + /* Don't propagate the invariant conversion error; it is a skeleton syntax error */ \ + (status) = U_NUMBER_SKELETON_SYNTAX_ERROR; \ + return; \ + } else if (U_FAILURE(conversionStatus)) { \ + (status) = conversionStatus; \ + return; \ + } \ +} UPRV_BLOCK_MACRO_END + +} // namespace + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__SOURCE_NUMBER_SKELETONS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_symbolswrapper.cpp b/intl/icu/source/i18n/number_symbolswrapper.cpp new file mode 100644 index 0000000000..4742a69c1f --- /dev/null +++ b/intl/icu/source/i18n/number_symbolswrapper.cpp @@ -0,0 +1,131 @@ +// © 2020 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "number_microprops.h" +#include "unicode/numberformatter.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +SymbolsWrapper::SymbolsWrapper(const SymbolsWrapper &other) { + doCopyFrom(other); +} + +SymbolsWrapper::SymbolsWrapper(SymbolsWrapper &&src) noexcept { + doMoveFrom(std::move(src)); +} + +SymbolsWrapper &SymbolsWrapper::operator=(const SymbolsWrapper &other) { + if (this == &other) { + return *this; + } + doCleanup(); + doCopyFrom(other); + return *this; +} + +SymbolsWrapper &SymbolsWrapper::operator=(SymbolsWrapper &&src) noexcept { + if (this == &src) { + return *this; + } + doCleanup(); + doMoveFrom(std::move(src)); + return *this; +} + +SymbolsWrapper::~SymbolsWrapper() { + doCleanup(); +} + +void SymbolsWrapper::setTo(const DecimalFormatSymbols &dfs) { + doCleanup(); + fType = SYMPTR_DFS; + fPtr.dfs = new DecimalFormatSymbols(dfs); +} + +void SymbolsWrapper::setTo(const NumberingSystem *ns) { + doCleanup(); + fType = SYMPTR_NS; + fPtr.ns = ns; +} + +void SymbolsWrapper::doCopyFrom(const SymbolsWrapper &other) { + fType = other.fType; + switch (fType) { + case SYMPTR_NONE: + // No action necessary + break; + case SYMPTR_DFS: + // Memory allocation failures are exposed in copyErrorTo() + if (other.fPtr.dfs != nullptr) { + fPtr.dfs = new DecimalFormatSymbols(*other.fPtr.dfs); + } else { + fPtr.dfs = nullptr; + } + break; + case SYMPTR_NS: + // Memory allocation failures are exposed in copyErrorTo() + if (other.fPtr.ns != nullptr) { + fPtr.ns = new NumberingSystem(*other.fPtr.ns); + } else { + fPtr.ns = nullptr; + } + break; + } +} + +void SymbolsWrapper::doMoveFrom(SymbolsWrapper &&src) { + fType = src.fType; + switch (fType) { + case SYMPTR_NONE: + // No action necessary + break; + case SYMPTR_DFS: + fPtr.dfs = src.fPtr.dfs; + src.fPtr.dfs = nullptr; + break; + case SYMPTR_NS: + fPtr.ns = src.fPtr.ns; + src.fPtr.ns = nullptr; + break; + } +} + +void SymbolsWrapper::doCleanup() { + switch (fType) { + case SYMPTR_NONE: + // No action necessary + break; + case SYMPTR_DFS: + delete fPtr.dfs; + break; + case SYMPTR_NS: + delete fPtr.ns; + break; + } +} + +bool SymbolsWrapper::isDecimalFormatSymbols() const { + return fType == SYMPTR_DFS; +} + +bool SymbolsWrapper::isNumberingSystem() const { + return fType == SYMPTR_NS; +} + +const DecimalFormatSymbols *SymbolsWrapper::getDecimalFormatSymbols() const { + U_ASSERT(fType == SYMPTR_DFS); + return fPtr.dfs; +} + +const NumberingSystem *SymbolsWrapper::getNumberingSystem() const { + U_ASSERT(fType == SYMPTR_NS); + return fPtr.ns; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_types.h b/intl/icu/source/i18n/number_types.h new file mode 100644 index 0000000000..84846efb92 --- /dev/null +++ b/intl/icu/source/i18n/number_types.h @@ -0,0 +1,374 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_TYPES_H__ +#define __NUMBER_TYPES_H__ + +#include +#include "unicode/decimfmt.h" +#include "unicode/unum.h" +#include "unicode/numsys.h" +#include "unicode/numberformatter.h" +#include "unicode/utf16.h" +#include "uassert.h" +#include "unicode/platform.h" +#include "unicode/uniset.h" +#include "standardplural.h" +#include "formatted_string_builder.h" + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { + +// For convenience and historical reasons, import the Field typedef to the namespace. +typedef FormattedStringBuilder::Field Field; + +// Typedef several enums for brevity and for easier comparison to Java. + +typedef UNumberFormatRoundingMode RoundingMode; + +typedef UNumberFormatPadPosition PadPosition; + +typedef UNumberCompactStyle CompactStyle; + +// ICU4J Equivalent: RoundingUtils.MAX_INT_FRAC_SIG +static constexpr int32_t kMaxIntFracSig = 999; + +// ICU4J Equivalent: RoundingUtils.DEFAULT_ROUNDING_MODE +static constexpr RoundingMode kDefaultMode = RoundingMode::UNUM_FOUND_HALFEVEN; + +// ICU4J Equivalent: Padder.FALLBACK_PADDING_STRING +static constexpr char16_t kFallbackPaddingString[] = u" "; + +// Forward declarations: + +class Modifier; +class MutablePatternModifier; +class DecimalQuantity; +class ModifierStore; +struct MicroProps; + + +enum AffixPatternType { + // Represents a literal character; the value is stored in the code point field. + TYPE_CODEPOINT = 0, + + // Represents a minus sign symbol '-'. + TYPE_MINUS_SIGN = -1, + + // Represents a plus sign symbol '+'. + TYPE_PLUS_SIGN = -2, + + // Represents an approximately sign symbol '~'. + TYPE_APPROXIMATELY_SIGN = -3, + + // Represents a percent sign symbol '%'. + TYPE_PERCENT = -4, + + // Represents a permille sign symbol '‰'. + TYPE_PERMILLE = -5, + + // Represents a single currency symbol '¤'. + TYPE_CURRENCY_SINGLE = -6, + + // Represents a double currency symbol '¤¤'. + TYPE_CURRENCY_DOUBLE = -7, + + // Represents a triple currency symbol '¤¤¤'. + TYPE_CURRENCY_TRIPLE = -8, + + // Represents a quadruple currency symbol '¤¤¤¤'. + TYPE_CURRENCY_QUAD = -9, + + // Represents a quintuple currency symbol '¤¤¤¤¤'. + TYPE_CURRENCY_QUINT = -10, + + // Represents a sequence of six or more currency symbols. + TYPE_CURRENCY_OVERFLOW = -15 +}; + +enum CompactType { + TYPE_DECIMAL, TYPE_CURRENCY +}; + +enum Signum { + SIGNUM_NEG = 0, + SIGNUM_NEG_ZERO = 1, + SIGNUM_POS_ZERO = 2, + SIGNUM_POS = 3, + SIGNUM_COUNT = 4, +}; + + +class U_I18N_API AffixPatternProvider { + public: + static const int32_t AFFIX_PLURAL_MASK = 0xff; + static const int32_t AFFIX_PREFIX = 0x100; + static const int32_t AFFIX_NEGATIVE_SUBPATTERN = 0x200; + static const int32_t AFFIX_PADDING = 0x400; + + // Convenience compound flags + static const int32_t AFFIX_POS_PREFIX = AFFIX_PREFIX; + static const int32_t AFFIX_POS_SUFFIX = 0; + static const int32_t AFFIX_NEG_PREFIX = AFFIX_PREFIX | AFFIX_NEGATIVE_SUBPATTERN; + static const int32_t AFFIX_NEG_SUFFIX = AFFIX_NEGATIVE_SUBPATTERN; + + virtual ~AffixPatternProvider(); + + virtual char16_t charAt(int flags, int i) const = 0; + + virtual int length(int flags) const = 0; + + virtual UnicodeString getString(int flags) const = 0; + + virtual bool hasCurrencySign() const = 0; + + virtual bool positiveHasPlusSign() const = 0; + + virtual bool hasNegativeSubpattern() const = 0; + + virtual bool negativeHasMinusSign() const = 0; + + virtual bool containsSymbolType(AffixPatternType, UErrorCode&) const = 0; + + /** + * True if the pattern has a number placeholder like "0" or "#,##0.00"; false if the pattern does not + * have one. This is used in cases like compact notation, where the pattern replaces the entire + * number instead of rendering the number. + */ + virtual bool hasBody() const = 0; + + /** + * True if the currency symbol should replace the decimal separator. + */ + virtual bool currencyAsDecimal() const = 0; +}; + + +/** + * A Modifier is an object that can be passed through the formatting pipeline until it is finally applied to the string + * builder. A Modifier usually contains a prefix and a suffix that are applied, but it could contain something else, + * like a {@link com.ibm.icu.text.SimpleFormatter} pattern. + * + * A Modifier is usually immutable, except in cases such as {@link MutablePatternModifier}, which are mutable for performance + * reasons. + * + * Exported as U_I18N_API because it is a base class for other exported types + */ +class U_I18N_API Modifier { + public: + virtual ~Modifier(); + + /** + * Apply this Modifier to the string builder. + * + * @param output + * The string builder to which to apply this modifier. + * @param leftIndex + * The left index of the string within the builder. Equal to 0 when only one number is being formatted. + * @param rightIndex + * The right index of the string within the string builder. Equal to length when only one number is being + * formatted. + * @return The number of characters (UTF-16 code units) that were added to the string builder. + */ + virtual int32_t apply(FormattedStringBuilder& output, int leftIndex, int rightIndex, + UErrorCode& status) const = 0; + + /** + * Gets the length of the prefix. This information can be used in combination with {@link #apply} to extract the + * prefix and suffix strings. + * + * @return The number of characters (UTF-16 code units) in the prefix. + */ + virtual int32_t getPrefixLength() const = 0; + + /** + * Returns the number of code points in the modifier, prefix plus suffix. + */ + virtual int32_t getCodePointCount() const = 0; + + /** + * Whether this modifier is strong. If a modifier is strong, it should always be applied immediately and not allowed + * to bubble up. With regard to padding, strong modifiers are considered to be on the inside of the prefix and + * suffix. + * + * @return Whether the modifier is strong. + */ + virtual bool isStrong() const = 0; + + /** + * Whether the modifier contains at least one occurrence of the given field. + */ + virtual bool containsField(Field field) const = 0; + + /** + * A fill-in for getParameters(). obj will always be set; if non-null, the other + * two fields are also safe to read. + */ + struct U_I18N_API Parameters { + const ModifierStore* obj = nullptr; + Signum signum; + StandardPlural::Form plural; + + Parameters(); + Parameters(const ModifierStore* _obj, Signum _signum, StandardPlural::Form _plural); + }; + + /** + * Gets a set of "parameters" for this Modifier. + * + * TODO: Make this return a `const Parameters*` more like Java? + */ + virtual void getParameters(Parameters& output) const = 0; + + /** + * Returns whether this Modifier is *semantically equivalent* to the other Modifier; + * in many cases, this is the same as equal, but parameters should be ignored. + */ + virtual bool semanticallyEquivalent(const Modifier& other) const = 0; +}; + + +/** + * This is *not* a modifier; rather, it is an object that can return modifiers + * based on given parameters. + * + * Exported as U_I18N_API because it is a base class for other exported types. + */ +class U_I18N_API ModifierStore { + public: + virtual ~ModifierStore(); + + /** + * Returns a Modifier with the given parameters (best-effort). + */ + virtual const Modifier* getModifier(Signum signum, StandardPlural::Form plural) const = 0; +}; + + +/** + * This interface is used when all number formatting settings, including the locale, are known, except for the quantity + * itself. The {@link #processQuantity} method performs the final step in the number processing pipeline: it uses the + * quantity to generate a finalized {@link MicroProps}, which can be used to render the number to output. + * + * In other words, this interface is used for the parts of number processing that are quantity-dependent. + * + * In order to allow for multiple different objects to all mutate the same MicroProps, a "chain" of MicroPropsGenerators + * are linked together, and each one is responsible for manipulating a certain quantity-dependent part of the + * MicroProps. At the tail of the linked list is a base instance of {@link MicroProps} with properties that are not + * quantity-dependent. Each element in the linked list calls {@link #processQuantity} on its "parent", then does its + * work, and then returns the result. + * + * This chain of MicroPropsGenerators is typically constructed by NumberFormatterImpl::macrosToMicroGenerator() when + * constructing a NumberFormatter. + * + * Exported as U_I18N_API because it is a base class for other exported types + * + */ +class U_I18N_API MicroPropsGenerator { + public: + virtual ~MicroPropsGenerator() = default; + + /** + * Considers the given {@link DecimalQuantity}, optionally mutates it, and + * populates a {@link MicroProps} instance. + * + * @param quantity The quantity for consideration and optional mutation. + * @param micros The MicroProps instance to populate. It will be modified as + * needed for the given quantity. + */ + virtual void processQuantity(DecimalQuantity& quantity, MicroProps& micros, + UErrorCode& status) const = 0; +}; + +/** + * An interface used by compact notation and scientific notation to choose a multiplier while rounding. + */ +class MultiplierProducer { + public: + virtual ~MultiplierProducer(); + + /** + * Maps a magnitude to a multiplier in powers of ten. For example, in compact notation in English, a magnitude of 5 + * (e.g., 100,000) should return a multiplier of -3, since the number is displayed in thousands. + * + * @param magnitude + * The power of ten of the input number. + * @return The shift in powers of ten. + */ + virtual int32_t getMultiplier(int32_t magnitude) const = 0; +}; + +// Exported as U_I18N_API because it is a public member field of exported DecimalFormatProperties +template +class U_I18N_API NullableValue { + public: + NullableValue() + : fNull(true) {} + + NullableValue(const NullableValue& other) = default; + + explicit NullableValue(const T& other) { + fValue = other; + fNull = false; + } + + NullableValue& operator=(const NullableValue& other) { + fNull = other.fNull; + if (!fNull) { + fValue = other.fValue; + } + return *this; + } + + NullableValue& operator=(const T& other) { + fValue = other; + fNull = false; + return *this; + } + + bool operator==(const NullableValue& other) const { + // "fValue == other.fValue" returns UBool, not bool (causes compiler warnings) + return fNull ? other.fNull : (other.fNull ? false : static_cast(fValue == other.fValue)); + } + + void nullify() { + // TODO: It might be nice to call the destructor here. + fNull = true; + } + + bool isNull() const { + return fNull; + } + + T get(UErrorCode& status) const { + if (fNull) { + status = U_UNDEFINED_VARIABLE; + } + return fValue; + } + + T getNoError() const { + return fValue; + } + + T getOrDefault(T defaultValue) const { + return fNull ? defaultValue : fValue; + } + + private: + bool fNull; + T fValue; +}; + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__NUMBER_TYPES_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_usageprefs.cpp b/intl/icu/source/i18n/number_usageprefs.cpp new file mode 100644 index 0000000000..6f7fdaa9dc --- /dev/null +++ b/intl/icu/source/i18n/number_usageprefs.cpp @@ -0,0 +1,216 @@ +// © 2020 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "number_usageprefs.h" +#include "cstring.h" +#include "number_decimalquantity.h" +#include "number_microprops.h" +#include "number_roundingutils.h" +#include "number_skeletons.h" +#include "unicode/char16ptr.h" +#include "unicode/currunit.h" +#include "unicode/fmtable.h" +#include "unicode/measure.h" +#include "unicode/numberformatter.h" +#include "unicode/platform.h" +#include "unicode/unum.h" +#include "unicode/urename.h" +#include "units_data.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using icu::StringSegment; +using icu::units::ConversionRates; + +// Copy constructor +StringProp::StringProp(const StringProp &other) : StringProp() { + this->operator=(other); +} + +// Copy assignment operator +StringProp &StringProp::operator=(const StringProp &other) { + if (this == &other) { return *this; } // self-assignment: no-op + fLength = 0; + fError = other.fError; + if (fValue != nullptr) { + uprv_free(fValue); + fValue = nullptr; + } + if (other.fValue == nullptr) { + return *this; + } + if (U_FAILURE(other.fError)) { + // We don't bother trying to allocating memory if we're in any case busy + // copying an errored StringProp. + return *this; + } + fValue = (char *)uprv_malloc(other.fLength + 1); + if (fValue == nullptr) { + fError = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + fLength = other.fLength; + uprv_strncpy(fValue, other.fValue, fLength + 1); + return *this; +} + +// Move constructor +StringProp::StringProp(StringProp &&src) noexcept : fValue(src.fValue), + fLength(src.fLength), + fError(src.fError) { + // Take ownership away from src if necessary + src.fValue = nullptr; +} + +// Move assignment operator +StringProp &StringProp::operator=(StringProp &&src) noexcept { + if (this == &src) { + return *this; + } + if (fValue != nullptr) { + uprv_free(fValue); + } + fValue = src.fValue; + fLength = src.fLength; + fError = src.fError; + // Take ownership away from src if necessary + src.fValue = nullptr; + return *this; +} + +StringProp::~StringProp() { + if (fValue != nullptr) { + uprv_free(fValue); + fValue = nullptr; + } +} + +void StringProp::set(StringPiece value) { + if (fValue != nullptr) { + uprv_free(fValue); + fValue = nullptr; + } + fLength = value.length(); + fValue = (char *)uprv_malloc(fLength + 1); + if (fValue == nullptr) { + fLength = 0; + fError = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (fLength > 0) { + uprv_strncpy(fValue, value.data(), fLength); + } + fValue[fLength] = 0; +} + +// Populates micros.mixedMeasures and modifies quantity, based on the values in +// measures. +void mixedMeasuresToMicros(const MaybeStackVector &measures, DecimalQuantity *quantity, + MicroProps *micros, UErrorCode status) { + micros->mixedMeasuresCount = measures.length(); + + if (micros->mixedMeasures.getCapacity() < micros->mixedMeasuresCount) { + if (micros->mixedMeasures.resize(micros->mixedMeasuresCount) == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + for (int32_t i = 0; i < micros->mixedMeasuresCount; i++) { + switch (measures[i]->getNumber().getType()) { + case Formattable::kInt64: + micros->mixedMeasures[i] = measures[i]->getNumber().getInt64(); + break; + + case Formattable::kDouble: + U_ASSERT(micros->indexOfQuantity < 0); + quantity->setToDouble(measures[i]->getNumber().getDouble()); + micros->indexOfQuantity = i; + break; + + default: + U_ASSERT(0 == "Found a Measure Number which is neither a double nor an int"); + UPRV_UNREACHABLE_EXIT; + break; + } + + if (U_FAILURE(status)) { + return; + } + } + + if (micros->indexOfQuantity < 0) { + // There is no quantity. + status = U_INTERNAL_PROGRAM_ERROR; + } +} + +UsagePrefsHandler::UsagePrefsHandler(const Locale &locale, + const MeasureUnit &inputUnit, + const StringPiece usage, + const MicroPropsGenerator *parent, + UErrorCode &status) + : fUnitsRouter(inputUnit, locale, usage, status), + fParent(parent) { +} + +void UsagePrefsHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + fParent->processQuantity(quantity, micros, status); + if (U_FAILURE(status)) { + return; + } + + quantity.roundToInfinity(); // Enables toDouble + const units::RouteResult routed = fUnitsRouter.route(quantity.toDouble(), µs.rounder, status); + if (U_FAILURE(status)) { + return; + } + const MaybeStackVector& routedMeasures = routed.measures; + micros.outputUnit = routed.outputUnit.copy(status).build(status); + if (U_FAILURE(status)) { + return; + } + + mixedMeasuresToMicros(routedMeasures, &quantity, µs, status); +} + +UnitConversionHandler::UnitConversionHandler(const MeasureUnit &targetUnit, + const MicroPropsGenerator *parent, UErrorCode &status) + : fOutputUnit(targetUnit), fParent(parent) { + MeasureUnitImpl tempInput, tempOutput; + + ConversionRates conversionRates(status); + if (U_FAILURE(status)) { + return; + } + + const MeasureUnitImpl &targetUnitImpl = + MeasureUnitImpl::forMeasureUnit(targetUnit, tempOutput, status); + fUnitConverter.adoptInsteadAndCheckErrorCode( + new ComplexUnitsConverter(targetUnitImpl, conversionRates, status), status); +} + +void UnitConversionHandler::processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const { + fParent->processQuantity(quantity, micros, status); + if (U_FAILURE(status)) { + return; + } + quantity.roundToInfinity(); // Enables toDouble + MaybeStackVector measures = + fUnitConverter->convert(quantity.toDouble(), µs.rounder, status); + micros.outputUnit = fOutputUnit; + if (U_FAILURE(status)) { + return; + } + + mixedMeasuresToMicros(measures, &quantity, µs, status); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_usageprefs.h b/intl/icu/source/i18n/number_usageprefs.h new file mode 100644 index 0000000000..e90df99d39 --- /dev/null +++ b/intl/icu/source/i18n/number_usageprefs.h @@ -0,0 +1,126 @@ +// © 2020 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_USAGEPREFS_H__ +#define __NUMBER_USAGEPREFS_H__ + +#include "cmemory.h" +#include "number_types.h" +#include "unicode/listformatter.h" +#include "unicode/localpointer.h" +#include "unicode/locid.h" +#include "unicode/measunit.h" +#include "unicode/stringpiece.h" +#include "unicode/uobject.h" +#include "units_converter.h" +#include "units_router.h" + +U_NAMESPACE_BEGIN + +using ::icu::units::ComplexUnitsConverter; +using ::icu::units::UnitsRouter; + +namespace number { +namespace impl { + +/** + * A MicroPropsGenerator which uses UnitsRouter to produce output converted to a + * MeasureUnit appropriate for a particular localized usage: see + * NumberFormatterSettings::usage(). + */ +class U_I18N_API UsagePrefsHandler : public MicroPropsGenerator, public UMemory { + public: + UsagePrefsHandler(const Locale &locale, const MeasureUnit &inputUnit, const StringPiece usage, + const MicroPropsGenerator *parent, UErrorCode &status); + + /** + * Obtains the appropriate output value, MeasureUnit and + * rounding/precision behaviour from the UnitsRouter. + * + * The output unit is passed on to the LongNameHandler via + * micros.outputUnit. + */ + void processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const override; + + /** + * Returns the list of possible output units, i.e. the full set of + * preferences, for the localized, usage-specific unit preferences. + * + * The returned pointer should be valid for the lifetime of the + * UsagePrefsHandler instance. + */ + const MaybeStackVector *getOutputUnits() const { + return fUnitsRouter.getOutputUnits(); + } + + private: + UnitsRouter fUnitsRouter; + const MicroPropsGenerator *fParent; +}; + +} // namespace impl +} // namespace number + +// Export explicit template instantiations of LocalPointerBase and LocalPointer. +// This is required when building DLLs for Windows. (See datefmt.h, +// collationiterator.h, erarules.h and others for similar examples.) +// +// Note: These need to be outside of the number::impl namespace, or Clang will +// generate a compile error. +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#if defined(_MSC_VER) +// Ignore warning 4661 as LocalPointerBase does not use operator== or operator!= +#pragma warning(push) +#pragma warning(disable: 4661) +#endif +template class U_I18N_API LocalPointerBase; +template class U_I18N_API LocalPointer; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + +namespace number { +namespace impl { + +/** + * A MicroPropsGenerator which converts a measurement from one MeasureUnit to + * another. In particular, the output MeasureUnit may be a mixed unit. (The + * input unit may not be a mixed unit.) + */ +class U_I18N_API UnitConversionHandler : public MicroPropsGenerator, public UMemory { + public: + /** + * Constructor. + * + * @param targetUnit Specifies the output MeasureUnit. The input MeasureUnit + * is derived from it: in case of a mixed unit, the biggest unit is + * taken as the input unit. If not a mixed unit, the input unit will be + * the same as the output unit and no unit conversion takes place. + * @param parent The parent MicroPropsGenerator. + * @param status Receives status. + */ + UnitConversionHandler(const MeasureUnit &targetUnit, const MicroPropsGenerator *parent, + UErrorCode &status); + + /** + * Obtains the appropriate output values from the Unit Converter. + */ + void processQuantity(DecimalQuantity &quantity, MicroProps µs, + UErrorCode &status) const override; + private: + MeasureUnit fOutputUnit; + LocalPointer fUnitConverter; + const MicroPropsGenerator *fParent; +}; + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif // __NUMBER_USAGEPREFS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_utils.cpp b/intl/icu/source/i18n/number_utils.cpp new file mode 100644 index 0000000000..ad70532140 --- /dev/null +++ b/intl/icu/source/i18n/number_utils.cpp @@ -0,0 +1,275 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include +#include +#include "number_decnum.h" +#include "number_types.h" +#include "number_utils.h" +#include "charstr.h" +#include "decContext.h" +#include "decNumber.h" +#include "double-conversion.h" +#include "fphdlimp.h" +#include "uresimp.h" +#include "ureslocs.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +using icu::double_conversion::DoubleToStringConverter; + + +namespace { + +const char16_t* +doGetPattern(UResourceBundle* res, const char* nsName, const char* patternKey, UErrorCode& publicStatus, + UErrorCode& localStatus) { + // Construct the path into the resource bundle + CharString key; + key.append("NumberElements/", publicStatus); + key.append(nsName, publicStatus); + key.append("/patterns/", publicStatus); + key.append(patternKey, publicStatus); + if (U_FAILURE(publicStatus)) { + return u""; + } + return ures_getStringByKeyWithFallback(res, key.data(), nullptr, &localStatus); +} + +} + + +const char16_t* utils::getPatternForStyle(const Locale& locale, const char* nsName, CldrPatternStyle style, + UErrorCode& status) { + const char* patternKey; + switch (style) { + case CLDR_PATTERN_STYLE_DECIMAL: + patternKey = "decimalFormat"; + break; + case CLDR_PATTERN_STYLE_CURRENCY: + patternKey = "currencyFormat"; + break; + case CLDR_PATTERN_STYLE_ACCOUNTING: + patternKey = "accountingFormat"; + break; + case CLDR_PATTERN_STYLE_PERCENT: + patternKey = "percentFormat"; + break; + case CLDR_PATTERN_STYLE_SCIENTIFIC: + patternKey = "scientificFormat"; + break; + default: + patternKey = "decimalFormat"; // silence compiler error + UPRV_UNREACHABLE_EXIT; + } + LocalUResourceBundlePointer res(ures_open(nullptr, locale.getName(), &status)); + if (U_FAILURE(status)) { return u""; } + + // Attempt to get the pattern with the native numbering system. + UErrorCode localStatus = U_ZERO_ERROR; + const char16_t* pattern; + pattern = doGetPattern(res.getAlias(), nsName, patternKey, status, localStatus); + if (U_FAILURE(status)) { return u""; } + + // Fall back to latn if native numbering system does not have the right pattern + if (U_FAILURE(localStatus) && uprv_strcmp("latn", nsName) != 0) { + localStatus = U_ZERO_ERROR; + pattern = doGetPattern(res.getAlias(), "latn", patternKey, status, localStatus); + if (U_FAILURE(status)) { return u""; } + } + + return pattern; +} + + +DecNum::DecNum() { + uprv_decContextDefault(&fContext, DEC_INIT_BASE); + uprv_decContextSetRounding(&fContext, DEC_ROUND_HALF_EVEN); + fContext.traps = 0; // no traps, thank you (what does this even mean?) +} + +DecNum::DecNum(const DecNum& other, UErrorCode& status) + : fContext(other.fContext) { + // Allocate memory for the new DecNum. + U_ASSERT(fContext.digits == other.fData.getCapacity()); + if (fContext.digits > kDefaultDigits) { + void* p = fData.resize(fContext.digits, 0); + if (p == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + // Copy the data from the old DecNum to the new one. + uprv_memcpy(fData.getAlias(), other.fData.getAlias(), sizeof(decNumber)); + uprv_memcpy(fData.getArrayStart(), + other.fData.getArrayStart(), + other.fData.getArrayLimit() - other.fData.getArrayStart()); +} + +void DecNum::setTo(StringPiece str, UErrorCode& status) { + // We need NUL-terminated for decNumber; CharString guarantees this, but not StringPiece. + CharString cstr(str, status); + if (U_FAILURE(status)) { return; } + _setTo(cstr.data(), str.length(), status); +} + +void DecNum::setTo(const char* str, UErrorCode& status) { + _setTo(str, static_cast(uprv_strlen(str)), status); +} + +void DecNum::setTo(double d, UErrorCode& status) { + // Need to check for NaN and Infinity before going into DoubleToStringConverter + if (std::isnan(d) != 0 || std::isfinite(d) == 0) { + status = U_UNSUPPORTED_ERROR; + return; + } + + // First convert from double to string, then string to DecNum. + // Allocate enough room for: all digits, "E-324", and NUL-terminator. + char buffer[DoubleToStringConverter::kBase10MaximalLength + 6]; + bool sign; // unused; always positive + int32_t length; + int32_t point; + DoubleToStringConverter::DoubleToAscii( + d, + DoubleToStringConverter::DtoaMode::SHORTEST, + 0, + buffer, + sizeof(buffer), + &sign, + &length, + &point + ); + + // Read initial result as a string. + _setTo(buffer, length, status); + + // Set exponent and bitmask. Note that DoubleToStringConverter does not do negatives. + fData.getAlias()->exponent += point - length; + fData.getAlias()->bits |= static_cast(std::signbit(d) ? DECNEG : 0); +} + +void DecNum::_setTo(const char* str, int32_t maxDigits, UErrorCode& status) { + if (maxDigits > kDefaultDigits) { + fData.resize(maxDigits, 0); + fContext.digits = maxDigits; + } else { + fContext.digits = kDefaultDigits; + } + + static_assert(DECDPUN == 1, "Assumes that DECDPUN is set to 1"); + uprv_decNumberFromString(fData.getAlias(), str, &fContext); + + // Check for invalid syntax and set the corresponding error code. + if ((fContext.status & DEC_Conversion_syntax) != 0) { + status = U_DECIMAL_NUMBER_SYNTAX_ERROR; + return; + } else if (fContext.status != 0) { + // Not a syntax error, but some other error, like an exponent that is too large. + status = U_UNSUPPORTED_ERROR; + return; + } +} + +void +DecNum::setTo(const uint8_t* bcd, int32_t length, int32_t scale, bool isNegative, UErrorCode& status) { + if (length > kDefaultDigits) { + fData.resize(length, 0); + fContext.digits = length; + } else { + fContext.digits = kDefaultDigits; + } + + // "digits is of type int32_t, and must have a value in the range 1 through 999,999,999." + if (length < 1 || length > 999999999) { + // Too large for decNumber + status = U_UNSUPPORTED_ERROR; + return; + } + // "The exponent field holds the exponent of the number. Its range is limited by the requirement that + // "the range of the adjusted exponent of the number be balanced and fit within a whole number of + // "decimal digits (in this implementation, be –999,999,999 through +999,999,999). The adjusted + // "exponent is the exponent that would result if the number were expressed with a single digit before + // "the decimal point, and is therefore given by exponent+digits-1." + if (scale > 999999999 - length + 1 || scale < -999999999 - length + 1) { + // Too large for decNumber + status = U_UNSUPPORTED_ERROR; + return; + } + + fData.getAlias()->digits = length; + fData.getAlias()->exponent = scale; + fData.getAlias()->bits = static_cast(isNegative ? DECNEG : 0); + uprv_decNumberSetBCD(fData, bcd, static_cast(length)); + if (fContext.status != 0) { + // Some error occurred while constructing the decNumber. + status = U_INTERNAL_PROGRAM_ERROR; + } +} + +void DecNum::normalize() { + uprv_decNumberReduce(fData, fData, &fContext); +} + +void DecNum::multiplyBy(const DecNum& rhs, UErrorCode& status) { + uprv_decNumberMultiply(fData, fData, rhs.fData, &fContext); + if (fContext.status != 0) { + status = U_INTERNAL_PROGRAM_ERROR; + } +} + +void DecNum::divideBy(const DecNum& rhs, UErrorCode& status) { + uprv_decNumberDivide(fData, fData, rhs.fData, &fContext); + if ((fContext.status & DEC_Inexact) != 0) { + // Ignore. + } else if (fContext.status != 0) { + status = U_INTERNAL_PROGRAM_ERROR; + } +} + +bool DecNum::isNegative() const { + return decNumberIsNegative(fData.getAlias()); +} + +bool DecNum::isZero() const { + return decNumberIsZero(fData.getAlias()); +} + +bool DecNum::isSpecial() const { + return decNumberIsSpecial(fData.getAlias()); +} + +bool DecNum::isInfinity() const { + return decNumberIsInfinite(fData.getAlias()); +} + +bool DecNum::isNaN() const { + return decNumberIsNaN(fData.getAlias()); +} + +void DecNum::toString(ByteSink& output, UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + // "string must be at least dn->digits+14 characters long" + int32_t minCapacity = fData.getAlias()->digits + 14; + MaybeStackArray buffer(minCapacity, status); + if (U_FAILURE(status)) { + return; + } + uprv_decNumberToString(fData, buffer.getAlias()); + output.Append(buffer.getAlias(), static_cast(uprv_strlen(buffer.getAlias()))); +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_utils.h b/intl/icu/source/i18n/number_utils.h new file mode 100644 index 0000000000..bc369c940f --- /dev/null +++ b/intl/icu/source/i18n/number_utils.h @@ -0,0 +1,112 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMBER_UTILS_H__ +#define __NUMBER_UTILS_H__ + +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "number_scientific.h" +#include "number_patternstring.h" +#include "number_modifiers.h" +#include "number_multiplier.h" +#include "number_roundingutils.h" +#include "decNumber.h" +#include "charstr.h" +#include "formatted_string_builder.h" + +U_NAMESPACE_BEGIN + +namespace number { +namespace impl { + +enum CldrPatternStyle { + CLDR_PATTERN_STYLE_DECIMAL, + CLDR_PATTERN_STYLE_CURRENCY, + CLDR_PATTERN_STYLE_ACCOUNTING, + CLDR_PATTERN_STYLE_PERCENT, + CLDR_PATTERN_STYLE_SCIENTIFIC, + CLDR_PATTERN_STYLE_COUNT, +}; + +// Namespace for naked functions +namespace utils { + +inline int32_t insertDigitFromSymbols(FormattedStringBuilder& output, int32_t index, int8_t digit, + const DecimalFormatSymbols& symbols, Field field, + UErrorCode& status) { + if (symbols.getCodePointZero() != -1) { + return output.insertCodePoint(index, symbols.getCodePointZero() + digit, field, status); + } + return output.insert(index, symbols.getConstDigitSymbol(digit), field, status); +} + +inline bool unitIsCurrency(const MeasureUnit& unit) { + return uprv_strcmp("currency", unit.getType()) == 0; +} + +inline bool unitIsBaseUnit(const MeasureUnit& unit) { + return unit == MeasureUnit(); +} + +inline bool unitIsPercent(const MeasureUnit& unit) { + return uprv_strcmp("percent", unit.getSubtype()) == 0; +} + +inline bool unitIsPermille(const MeasureUnit& unit) { + return uprv_strcmp("permille", unit.getSubtype()) == 0; +} + +// NOTE: In Java, this method is in NumberFormat.java +const char16_t* +getPatternForStyle(const Locale& locale, const char* nsName, CldrPatternStyle style, UErrorCode& status); + +/** + * Computes the plural form for this number based on the specified set of rules. + * + * @param rules A {@link PluralRules} object representing the set of rules. + * @return The {@link StandardPlural} according to the PluralRules. If the plural form is not in + * the set of standard plurals, {@link StandardPlural#OTHER} is returned instead. + */ +inline StandardPlural::Form getStandardPlural(const PluralRules *rules, + const IFixedDecimal &fdec) { + if (rules == nullptr) { + // Fail gracefully if the user didn't provide a PluralRules + return StandardPlural::Form::OTHER; + } else { + UnicodeString ruleString = rules->select(fdec); + return StandardPlural::orOtherFromString(ruleString); + } +} + +/** + * Computes the plural form after copying the number and applying rounding rules. + */ +inline StandardPlural::Form getPluralSafe( + const RoundingImpl& rounder, + const PluralRules* rules, + const DecimalQuantity& dq, + UErrorCode& status) { + // TODO(ICU-20500): Avoid the copy? + DecimalQuantity copy(dq); + rounder.apply(copy, status); + if (U_FAILURE(status)) { + return StandardPlural::Form::OTHER; + } + return getStandardPlural(rules, copy); +} + +} // namespace utils + +} // namespace impl +} // namespace number + +U_NAMESPACE_END + +#endif //__NUMBER_UTILS_H__ + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/number_utypes.h b/intl/icu/source/i18n/number_utypes.h new file mode 100644 index 0000000000..0c13040189 --- /dev/null +++ b/intl/icu/source/i18n/number_utypes.h @@ -0,0 +1,59 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMBER_UTYPES_H__ +#define __SOURCE_NUMBER_UTYPES_H__ + +#include "unicode/numberformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "formatted_string_builder.h" +#include "formattedval_impl.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + + +/** Helper function used in upluralrules.cpp */ +const DecimalQuantity* validateUFormattedNumberToDecimalQuantity( + const UFormattedNumber* uresult, UErrorCode& status); + + +/** + * Struct for data used by FormattedNumber. + * + * This struct is held internally by the C++ version FormattedNumber since the member types are not + * declared in the public header file. + * + * Exported as U_I18N_API for tests + */ +class U_I18N_API UFormattedNumberData : public FormattedValueStringBuilderImpl { +public: + UFormattedNumberData() : FormattedValueStringBuilderImpl(kUndefinedField) {} + virtual ~UFormattedNumberData(); + + UFormattedNumberData(UFormattedNumberData&&) = default; + UFormattedNumberData& operator=(UFormattedNumberData&&) = default; + + // The formatted quantity. + DecimalQuantity quantity; + + // The output unit for the formatted quantity. + // TODO(units,hugovdm): populate this correctly for the general case - it's + // currently only implemented for the .usage() use case. + MeasureUnit outputUnit; + + // The gender of the formatted output. + const char *gender = ""; +}; + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__SOURCE_NUMBER_UTYPES_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numfmt.cpp b/intl/icu/source/i18n/numfmt.cpp new file mode 100644 index 0000000000..74689e1363 --- /dev/null +++ b/intl/icu/source/i18n/numfmt.cpp @@ -0,0 +1,1536 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2015, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File NUMFMT.CPP +* +* Modification History: +* +* Date Name Description +* 02/19/97 aliu Converted from java. +* 03/18/97 clhuang Implemented with C++ APIs. +* 04/17/97 aliu Enlarged MAX_INTEGER_DIGITS to fully accommodate the +* largest double, by default. +* Changed DigitCount to int per code review. +* 07/20/98 stephen Changed operator== to check for grouping +* Changed setMaxIntegerDigits per Java implementation. +* Changed setMinIntegerDigits per Java implementation. +* Changed setMinFractionDigits per Java implementation. +* Changed setMaxFractionDigits per Java implementation. +******************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numfmt.h" +#include "unicode/locid.h" +#include "unicode/dcfmtsym.h" +#include "unicode/decimfmt.h" +#include "unicode/ustring.h" +#include "unicode/ucurr.h" +#include "unicode/curramt.h" +#include "unicode/numsys.h" +#include "unicode/rbnf.h" +#include "unicode/localpointer.h" +#include "unicode/udisplaycontext.h" +#include "charstr.h" +#include "winnmfmt.h" +#include "uresimp.h" +#include "uhash.h" +#include "cmemory.h" +#include "servloc.h" +#include "ucln_in.h" +#include "cstring.h" +#include "putilimp.h" +#include "uassert.h" +#include "umutex.h" +#include "mutex.h" +#include +#include "sharednumberformat.h" +#include "unifiedcache.h" +#include "number_decimalquantity.h" +#include "number_utils.h" + +//#define FMT_DEBUG + +#ifdef FMT_DEBUG +#include +static inline void debugout(UnicodeString s) { + char buf[2000]; + s.extract((int32_t) 0, s.length(), buf); + printf("%s", buf); +} +#define debug(x) printf("%s", x); +#else +#define debugout(x) +#define debug(x) +#endif + +// If no number pattern can be located for a locale, this is the last +// resort. The patterns are same as the ones in root locale. +static const char16_t gLastResortDecimalPat[] = { + 0x23, 0x2C, 0x23, 0x23, 0x30, 0x2E, 0x23, 0x23, 0x23, 0 /* "#,##0.###" */ +}; +static const char16_t gLastResortCurrencyPat[] = { + 0xA4, 0xA0, 0x23, 0x2C, 0x23, 0x23, 0x30, 0x2E, 0x30, 0x30, 0 /* "\u00A4\u00A0#,##0.00" */ +}; +static const char16_t gLastResortPercentPat[] = { + 0x23, 0x2C, 0x23, 0x23, 0x30, 0x25, 0 /* "#,##0%" */ +}; +static const char16_t gLastResortScientificPat[] = { + 0x23, 0x45, 0x30, 0 /* "#E0" */ +}; +static const char16_t gLastResortIsoCurrencyPat[] = { + 0xA4, 0xA4, 0xA0, 0x23, 0x2C, 0x23, 0x23, 0x30, 0x2E, 0x30, 0x30, 0 /* "\u00A4\u00A4\u00A0#,##0.00" */ +}; +static const char16_t gLastResortPluralCurrencyPat[] = { + 0x23, 0x2C, 0x23, 0x23, 0x30, 0x2E, 0x23, 0x23, 0x23, 0x20, 0xA4, 0xA4, 0xA4, 0 /* "#,##0.### \u00A4\u00A4\u00A4*/ +}; +static const char16_t gLastResortAccountingCurrencyPat[] = { + 0xA4, 0xA0, 0x23, 0x2C, 0x23, 0x23, 0x30, 0x2E, 0x30, 0x30, 0 /* "\u00A4\u00A0#,##0.00" */ +}; + +static const char16_t gSingleCurrencySign[] = {0xA4, 0}; +static const char16_t gDoubleCurrencySign[] = {0xA4, 0xA4, 0}; + +static const char16_t gSlash = 0x2f; + +// If the maximum base 10 exponent were 4, then the largest number would +// be 99,999 which has 5 digits. +// On IEEE754 systems gMaxIntegerDigits is 308 + possible denormalized 15 digits + rounding digit +// With big decimal, the max exponent is 999,999,999 and the max number of digits is the same, 999,999,999 +const int32_t icu::NumberFormat::gDefaultMaxIntegerDigits = 2000000000; +const int32_t icu::NumberFormat::gDefaultMinIntegerDigits = 127; + +static const char16_t * const gLastResortNumberPatterns[UNUM_FORMAT_STYLE_COUNT] = { + nullptr, // UNUM_PATTERN_DECIMAL + gLastResortDecimalPat, // UNUM_DECIMAL + gLastResortCurrencyPat, // UNUM_CURRENCY + gLastResortPercentPat, // UNUM_PERCENT + gLastResortScientificPat, // UNUM_SCIENTIFIC + nullptr, // UNUM_SPELLOUT + nullptr, // UNUM_ORDINAL + nullptr, // UNUM_DURATION + gLastResortDecimalPat, // UNUM_NUMBERING_SYSTEM + nullptr, // UNUM_PATTERN_RULEBASED + gLastResortIsoCurrencyPat, // UNUM_CURRENCY_ISO + gLastResortPluralCurrencyPat, // UNUM_CURRENCY_PLURAL + gLastResortAccountingCurrencyPat, // UNUM_CURRENCY_ACCOUNTING + gLastResortCurrencyPat, // UNUM_CASH_CURRENCY + nullptr, // UNUM_DECIMAL_COMPACT_SHORT + nullptr, // UNUM_DECIMAL_COMPACT_LONG + gLastResortCurrencyPat, // UNUM_CURRENCY_STANDARD +}; + +// Keys used for accessing resource bundles + +static const icu::number::impl::CldrPatternStyle gFormatCldrStyles[UNUM_FORMAT_STYLE_COUNT] = { + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_PATTERN_DECIMAL + icu::number::impl::CLDR_PATTERN_STYLE_DECIMAL, // UNUM_DECIMAL + icu::number::impl::CLDR_PATTERN_STYLE_CURRENCY, // UNUM_CURRENCY + icu::number::impl::CLDR_PATTERN_STYLE_PERCENT, // UNUM_PERCENT + icu::number::impl::CLDR_PATTERN_STYLE_SCIENTIFIC, // UNUM_SCIENTIFIC + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_SPELLOUT + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_ORDINAL + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_DURATION + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_NUMBERING_SYSTEM + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_PATTERN_RULEBASED + // For UNUM_CURRENCY_ISO and UNUM_CURRENCY_PLURAL, + // the pattern is the same as the pattern of UNUM_CURRENCY + // except for replacing the single currency sign with + // double currency sign or triple currency sign. + icu::number::impl::CLDR_PATTERN_STYLE_CURRENCY, // UNUM_CURRENCY_ISO + icu::number::impl::CLDR_PATTERN_STYLE_CURRENCY, // UNUM_CURRENCY_PLURAL + icu::number::impl::CLDR_PATTERN_STYLE_ACCOUNTING, // UNUM_CURRENCY_ACCOUNTING + icu::number::impl::CLDR_PATTERN_STYLE_CURRENCY, // UNUM_CASH_CURRENCY + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_DECIMAL_COMPACT_SHORT + /* nullptr */ icu::number::impl::CLDR_PATTERN_STYLE_COUNT, // UNUM_DECIMAL_COMPACT_LONG + icu::number::impl::CLDR_PATTERN_STYLE_CURRENCY, // UNUM_CURRENCY_STANDARD +}; + +// Static hashtable cache of NumberingSystem objects used by NumberFormat +static UHashtable * NumberingSystem_cache = nullptr; +static icu::UInitOnce gNSCacheInitOnce {}; + +#if !UCONFIG_NO_SERVICE +static icu::ICULocaleService* gService = nullptr; +static icu::UInitOnce gServiceInitOnce {}; +#endif + +/** + * Release all static memory held by Number Format. + */ +U_CDECL_BEGIN +static void U_CALLCONV +deleteNumberingSystem(void *obj) { + delete (icu::NumberingSystem *)obj; +} + +static UBool U_CALLCONV numfmt_cleanup() { +#if !UCONFIG_NO_SERVICE + gServiceInitOnce.reset(); + if (gService) { + delete gService; + gService = nullptr; + } +#endif + gNSCacheInitOnce.reset(); + if (NumberingSystem_cache) { + // delete NumberingSystem_cache; + uhash_close(NumberingSystem_cache); + NumberingSystem_cache = nullptr; + } + return true; +} +U_CDECL_END + +// ***************************************************************************** +// class NumberFormat +// ***************************************************************************** + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(NumberFormat) + +#if !UCONFIG_NO_SERVICE +// ------------------------------------- +// SimpleNumberFormatFactory implementation +NumberFormatFactory::~NumberFormatFactory() {} +SimpleNumberFormatFactory::SimpleNumberFormatFactory(const Locale& locale, UBool visible) + : _visible(visible) +{ + LocaleUtility::initNameFromLocale(locale, _id); +} + +SimpleNumberFormatFactory::~SimpleNumberFormatFactory() {} + +UBool SimpleNumberFormatFactory::visible() const { + return _visible; +} + +const UnicodeString * +SimpleNumberFormatFactory::getSupportedIDs(int32_t &count, UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + count = 1; + return &_id; + } + count = 0; + return nullptr; +} +#endif /* #if !UCONFIG_NO_SERVICE */ + +// ------------------------------------- +// default constructor +NumberFormat::NumberFormat() +: fGroupingUsed(true), + fMaxIntegerDigits(gDefaultMaxIntegerDigits), + fMinIntegerDigits(1), + fMaxFractionDigits(3), // invariant, >= minFractionDigits + fMinFractionDigits(0), + fParseIntegerOnly(false), + fLenient(false), + fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE) +{ + fCurrency[0] = 0; +} + +// ------------------------------------- + +NumberFormat::~NumberFormat() +{ +} + +SharedNumberFormat::~SharedNumberFormat() { + delete ptr; +} + +// ------------------------------------- +// copy constructor + +NumberFormat::NumberFormat(const NumberFormat &source) +: Format(source) +{ + *this = source; +} + +// ------------------------------------- +// assignment operator + +NumberFormat& +NumberFormat::operator=(const NumberFormat& rhs) +{ + if (this != &rhs) + { + Format::operator=(rhs); + fGroupingUsed = rhs.fGroupingUsed; + fMaxIntegerDigits = rhs.fMaxIntegerDigits; + fMinIntegerDigits = rhs.fMinIntegerDigits; + fMaxFractionDigits = rhs.fMaxFractionDigits; + fMinFractionDigits = rhs.fMinFractionDigits; + fParseIntegerOnly = rhs.fParseIntegerOnly; + u_strncpy(fCurrency, rhs.fCurrency, 3); + fCurrency[3] = 0; + fLenient = rhs.fLenient; + fCapitalizationContext = rhs.fCapitalizationContext; + } + return *this; +} + +// ------------------------------------- + +bool +NumberFormat::operator==(const Format& that) const +{ + // Format::operator== guarantees this cast is safe + NumberFormat* other = (NumberFormat*)&that; + +#ifdef FMT_DEBUG + // This code makes it easy to determine why two format objects that should + // be equal aren't. + UBool first = true; + if (!Format::operator==(that)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("Format::!="); + } + if (!(fMaxIntegerDigits == other->fMaxIntegerDigits && + fMinIntegerDigits == other->fMinIntegerDigits)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("Integer digits !="); + } + if (!(fMaxFractionDigits == other->fMaxFractionDigits && + fMinFractionDigits == other->fMinFractionDigits)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("Fraction digits !="); + } + if (!(fGroupingUsed == other->fGroupingUsed)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("fGroupingUsed != "); + } + if (!(fParseIntegerOnly == other->fParseIntegerOnly)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("fParseIntegerOnly != "); + } + if (!(u_strcmp(fCurrency, other->fCurrency) == 0)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("fCurrency !="); + } + if (!(fLenient == other->fLenient)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("fLenient != "); + } + if (!(fCapitalizationContext == other->fCapitalizationContext)) { + if (first) { printf("[ "); first = false; } else { printf(", "); } + debug("fCapitalizationContext != "); + } + if (!first) { printf(" ]"); } +#endif + + return ((this == &that) || + ((Format::operator==(that) && + fMaxIntegerDigits == other->fMaxIntegerDigits && + fMinIntegerDigits == other->fMinIntegerDigits && + fMaxFractionDigits == other->fMaxFractionDigits && + fMinFractionDigits == other->fMinFractionDigits && + fGroupingUsed == other->fGroupingUsed && + fParseIntegerOnly == other->fParseIntegerOnly && + u_strcmp(fCurrency, other->fCurrency) == 0 && + fLenient == other->fLenient && + fCapitalizationContext == other->fCapitalizationContext))); +} + +// ------------------------------------- +// Default implementation sets unsupported error; subclasses should +// override. + +UnicodeString& +NumberFormat::format(double /* unused number */, + UnicodeString& toAppendTo, + FieldPositionIterator* /* unused posIter */, + UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + status = U_UNSUPPORTED_ERROR; + } + return toAppendTo; +} + +// ------------------------------------- +// Default implementation sets unsupported error; subclasses should +// override. + +UnicodeString& +NumberFormat::format(int32_t /* unused number */, + UnicodeString& toAppendTo, + FieldPositionIterator* /* unused posIter */, + UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + status = U_UNSUPPORTED_ERROR; + } + return toAppendTo; +} + +// ------------------------------------- +// Default implementation sets unsupported error; subclasses should +// override. + +UnicodeString& +NumberFormat::format(int64_t /* unused number */, + UnicodeString& toAppendTo, + FieldPositionIterator* /* unused posIter */, + UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + status = U_UNSUPPORTED_ERROR; + } + return toAppendTo; +} + +// ------------------------------------------ +// These functions add the status code, just fall back to the non-status versions +UnicodeString& +NumberFormat::format(double number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode &status) const { + if(U_SUCCESS(status)) { + return format(number,appendTo,pos); + } else { + return appendTo; + } +} + +UnicodeString& +NumberFormat::format(int32_t number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode &status) const { + if(U_SUCCESS(status)) { + return format(number,appendTo,pos); + } else { + return appendTo; + } +} + +UnicodeString& +NumberFormat::format(int64_t number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode &status) const { + if(U_SUCCESS(status)) { + return format(number,appendTo,pos); + } else { + return appendTo; + } +} + + + +// ------------------------------------- +// Decimal Number format() default implementation +// Subclasses do not normally override this function, but rather the DigitList +// formatting functions.. +// The expected call chain from here is +// this function -> +// NumberFormat::format(Formattable -> +// DecimalFormat::format(DigitList +// +// Or, for subclasses of Formattable that do not know about DigitList, +// this Function -> +// NumberFormat::format(Formattable -> +// NumberFormat::format(DigitList -> +// XXXFormat::format(double + +UnicodeString& +NumberFormat::format(StringPiece decimalNum, + UnicodeString& toAppendTo, + FieldPositionIterator* fpi, + UErrorCode& status) const +{ + Formattable f; + f.setDecimalNumber(decimalNum, status); + format(f, toAppendTo, fpi, status); + return toAppendTo; +} + +/** + * +// Formats the number object and save the format +// result in the toAppendTo string buffer. + +// utility to save/restore state, used in two overloads +// of format(const Formattable&...) below. +* +* Old purpose of ArgExtractor was to avoid const. Not thread safe! +* +* keeping it around as a shim. +*/ +class ArgExtractor { + const Formattable* num; + char16_t save[4]; + UBool fWasCurrency; + + public: + ArgExtractor(const NumberFormat& nf, const Formattable& obj, UErrorCode& status); + ~ArgExtractor(); + + const Formattable* number() const; + const char16_t *iso() const; + UBool wasCurrency() const; +}; + +inline const Formattable* +ArgExtractor::number() const { + return num; +} + +inline UBool +ArgExtractor::wasCurrency() const { + return fWasCurrency; +} + +inline const char16_t * +ArgExtractor::iso() const { + return save; +} + +ArgExtractor::ArgExtractor(const NumberFormat& /*nf*/, const Formattable& obj, UErrorCode& /*status*/) + : num(&obj), fWasCurrency(false) { + + const UObject* o = obj.getObject(); // most commonly o==nullptr + const CurrencyAmount* amt; + if (o != nullptr && (amt = dynamic_cast(o)) != nullptr) { + // getISOCurrency() returns a pointer to internal storage, so we + // copy it to retain it across the call to setCurrency(). + //const char16_t* curr = amt->getISOCurrency(); + u_strcpy(save, amt->getISOCurrency()); + num = &amt->getNumber(); + fWasCurrency=true; + } else { + save[0]=0; + } +} + +ArgExtractor::~ArgExtractor() { +} + +UnicodeString& NumberFormat::format(const number::impl::DecimalQuantity &number, + UnicodeString& appendTo, + FieldPositionIterator* posIter, + UErrorCode& status) const { + // DecimalFormat overrides this function, and handles DigitList based big decimals. + // Other subclasses (ChoiceFormat) do not (yet) handle DigitLists, + // so this default implementation falls back to formatting decimal numbers as doubles. + if (U_FAILURE(status)) { + return appendTo; + } + double dnum = number.toDouble(); + format(dnum, appendTo, posIter, status); + return appendTo; +} + + + +UnicodeString& +NumberFormat::format(const number::impl::DecimalQuantity &number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode &status) const { + // DecimalFormat overrides this function, and handles DigitList based big decimals. + // Other subclasses (ChoiceFormat) do not (yet) handle DigitLists, + // so this default implementation falls back to formatting decimal numbers as doubles. + if (U_FAILURE(status)) { + return appendTo; + } + double dnum = number.toDouble(); + format(dnum, appendTo, pos, status); + return appendTo; +} + +UnicodeString& +NumberFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return appendTo; + + ArgExtractor arg(*this, obj, status); + const Formattable *n = arg.number(); + const char16_t *iso = arg.iso(); + + if(arg.wasCurrency() && u_strcmp(iso, getCurrency())) { + // trying to format a different currency. + // Right now, we clone. + LocalPointer cloneFmt(this->clone()); + cloneFmt->setCurrency(iso, status); + // next line should NOT recurse, because n is numeric whereas obj was a wrapper around currency amount. + return cloneFmt->format(*n, appendTo, pos, status); + } + + if (n->isNumeric() && n->getDecimalQuantity() != nullptr) { + // Decimal Number. We will have a DigitList available if the value was + // set to a decimal number, or if the value originated with a parse. + // + // The default implementation for formatting a DigitList converts it + // to a double, and formats that, allowing formatting classes that don't + // know about DigitList to continue to operate as they had. + // + // DecimalFormat overrides the DigitList formatting functions. + format(*n->getDecimalQuantity(), appendTo, pos, status); + } else { + switch (n->getType()) { + case Formattable::kDouble: + format(n->getDouble(), appendTo, pos, status); + break; + case Formattable::kLong: + format(n->getLong(), appendTo, pos, status); + break; + case Formattable::kInt64: + format(n->getInt64(), appendTo, pos, status); + break; + default: + status = U_INVALID_FORMAT_ERROR; + break; + } + } + + return appendTo; +} + +// -------------------------------------x +// Formats the number object and save the format +// result in the toAppendTo string buffer. + +UnicodeString& +NumberFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPositionIterator* posIter, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return appendTo; + + ArgExtractor arg(*this, obj, status); + const Formattable *n = arg.number(); + const char16_t *iso = arg.iso(); + + if(arg.wasCurrency() && u_strcmp(iso, getCurrency())) { + // trying to format a different currency. + // Right now, we clone. + LocalPointer cloneFmt(this->clone()); + cloneFmt->setCurrency(iso, status); + // next line should NOT recurse, because n is numeric whereas obj was a wrapper around currency amount. + return cloneFmt->format(*n, appendTo, posIter, status); + } + + if (n->isNumeric() && n->getDecimalQuantity() != nullptr) { + // Decimal Number + format(*n->getDecimalQuantity(), appendTo, posIter, status); + } else { + switch (n->getType()) { + case Formattable::kDouble: + format(n->getDouble(), appendTo, posIter, status); + break; + case Formattable::kLong: + format(n->getLong(), appendTo, posIter, status); + break; + case Formattable::kInt64: + format(n->getInt64(), appendTo, posIter, status); + break; + default: + status = U_INVALID_FORMAT_ERROR; + break; + } + } + + return appendTo; +} + +// ------------------------------------- + +UnicodeString& +NumberFormat::format(int64_t number, + UnicodeString& appendTo, + FieldPosition& pos) const +{ + // default so we don't introduce a new abstract method + return format((int32_t)number, appendTo, pos); +} + +// ------------------------------------- +// Parses the string and save the result object as well +// as the final parsed position. + +void +NumberFormat::parseObject(const UnicodeString& source, + Formattable& result, + ParsePosition& parse_pos) const +{ + parse(source, result, parse_pos); +} + +// ------------------------------------- +// Formats a double number and save the result in a string. + +UnicodeString& +NumberFormat::format(double number, UnicodeString& appendTo) const +{ + FieldPosition pos(FieldPosition::DONT_CARE); + return format(number, appendTo, pos); +} + +// ------------------------------------- +// Formats a long number and save the result in a string. + +UnicodeString& +NumberFormat::format(int32_t number, UnicodeString& appendTo) const +{ + FieldPosition pos(FieldPosition::DONT_CARE); + return format(number, appendTo, pos); +} + +// ------------------------------------- +// Formats a long number and save the result in a string. + +UnicodeString& +NumberFormat::format(int64_t number, UnicodeString& appendTo) const +{ + FieldPosition pos(FieldPosition::DONT_CARE); + return format(number, appendTo, pos); +} + +// ------------------------------------- +// Parses the text and save the result object. If the returned +// parse position is 0, that means the parsing failed, the status +// code needs to be set to failure. Ignores the returned parse +// position, otherwise. + +void +NumberFormat::parse(const UnicodeString& text, + Formattable& result, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return; + + ParsePosition parsePosition(0); + parse(text, result, parsePosition); + if (parsePosition.getIndex() == 0) { + status = U_INVALID_FORMAT_ERROR; + } +} + +CurrencyAmount* NumberFormat::parseCurrency(const UnicodeString& text, + ParsePosition& pos) const { + // Default implementation only -- subclasses should override + Formattable parseResult; + int32_t start = pos.getIndex(); + parse(text, parseResult, pos); + if (pos.getIndex() != start) { + char16_t curr[4]; + UErrorCode ec = U_ZERO_ERROR; + getEffectiveCurrency(curr, ec); + if (U_SUCCESS(ec)) { + LocalPointer currAmt(new CurrencyAmount(parseResult, curr, ec), ec); + if (U_FAILURE(ec)) { + pos.setIndex(start); // indicate failure + } else { + return currAmt.orphan(); + } + } + } + return nullptr; +} + +// ------------------------------------- +// Sets to only parse integers. + +void +NumberFormat::setParseIntegerOnly(UBool value) +{ + fParseIntegerOnly = value; +} + +// ------------------------------------- +// Sets whether lenient parse is enabled. + +void +NumberFormat::setLenient(UBool enable) +{ + fLenient = enable; +} + +// ------------------------------------- +// Create a number style NumberFormat instance with the default locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createInstance(UErrorCode& status) +{ + return createInstance(Locale::getDefault(), UNUM_DECIMAL, status); +} + +// ------------------------------------- +// Create a number style NumberFormat instance with the inLocale locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createInstance(const Locale& inLocale, UErrorCode& status) +{ + return createInstance(inLocale, UNUM_DECIMAL, status); +} + +// ------------------------------------- +// Create a currency style NumberFormat instance with the default locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createCurrencyInstance(UErrorCode& status) +{ + return createCurrencyInstance(Locale::getDefault(), status); +} + +// ------------------------------------- +// Create a currency style NumberFormat instance with the inLocale locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createCurrencyInstance(const Locale& inLocale, UErrorCode& status) +{ + return createInstance(inLocale, UNUM_CURRENCY, status); +} + +// ------------------------------------- +// Create a percent style NumberFormat instance with the default locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createPercentInstance(UErrorCode& status) +{ + return createInstance(Locale::getDefault(), UNUM_PERCENT, status); +} + +// ------------------------------------- +// Create a percent style NumberFormat instance with the inLocale locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createPercentInstance(const Locale& inLocale, UErrorCode& status) +{ + return createInstance(inLocale, UNUM_PERCENT, status); +} + +// ------------------------------------- +// Create a scientific style NumberFormat instance with the default locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createScientificInstance(UErrorCode& status) +{ + return createInstance(Locale::getDefault(), UNUM_SCIENTIFIC, status); +} + +// ------------------------------------- +// Create a scientific style NumberFormat instance with the inLocale locale. + +NumberFormat* U_EXPORT2 +NumberFormat::createScientificInstance(const Locale& inLocale, UErrorCode& status) +{ + return createInstance(inLocale, UNUM_SCIENTIFIC, status); +} + +// ------------------------------------- + +const Locale* U_EXPORT2 +NumberFormat::getAvailableLocales(int32_t& count) +{ + return Locale::getAvailableLocales(count); +} + +// ------------------------------------------ +// +// Registration +// +//------------------------------------------- + +#if !UCONFIG_NO_SERVICE + +// ------------------------------------- + +class ICUNumberFormatFactory : public ICUResourceBundleFactory { +public: + virtual ~ICUNumberFormatFactory(); +protected: + virtual UObject* handleCreate(const Locale& loc, int32_t kind, const ICUService* /* service */, UErrorCode& status) const override { + return NumberFormat::makeInstance(loc, (UNumberFormatStyle)kind, status); + } +}; + +ICUNumberFormatFactory::~ICUNumberFormatFactory() {} + +// ------------------------------------- + +class NFFactory : public LocaleKeyFactory { +private: + NumberFormatFactory* _delegate; + Hashtable* _ids; + +public: + NFFactory(NumberFormatFactory* delegate) + : LocaleKeyFactory(delegate->visible() ? VISIBLE : INVISIBLE) + , _delegate(delegate) + , _ids(nullptr) + { + } + + virtual ~NFFactory(); + + virtual UObject* create(const ICUServiceKey& key, const ICUService* service, UErrorCode& status) const override + { + if (handlesKey(key, status)) { + const LocaleKey* lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + Locale loc; + lkey->canonicalLocale(loc); + int32_t kind = lkey->kind(); + + UObject* result = _delegate->createFormat(loc, (UNumberFormatStyle)kind); + if (result == nullptr) { + result = service->getKey(const_cast(key) /* cast away const */, nullptr, this, status); + } + return result; + } + return nullptr; + } + +protected: + /** + * Return the set of ids that this factory supports (visible or + * otherwise). This can be called often and might need to be + * cached if it is expensive to create. + */ + virtual const Hashtable* getSupportedIDs(UErrorCode& status) const override + { + if (U_SUCCESS(status)) { + if (!_ids) { + int32_t count = 0; + const UnicodeString * const idlist = _delegate->getSupportedIDs(count, status); + ((NFFactory*)this)->_ids = new Hashtable(status); /* cast away const */ + if (_ids) { + for (int i = 0; i < count; ++i) { + _ids->put(idlist[i], (void*)this, status); + } + } + } + return _ids; + } + return nullptr; + } +}; + +NFFactory::~NFFactory() +{ + delete _delegate; + delete _ids; +} + +class ICUNumberFormatService : public ICULocaleService { +public: + ICUNumberFormatService() + : ICULocaleService(UNICODE_STRING_SIMPLE("Number Format")) + { + UErrorCode status = U_ZERO_ERROR; + registerFactory(new ICUNumberFormatFactory(), status); + } + + virtual ~ICUNumberFormatService(); + + virtual UObject* cloneInstance(UObject* instance) const override { + return ((NumberFormat*)instance)->clone(); + } + + virtual UObject* handleDefault(const ICUServiceKey& key, UnicodeString* /* actualID */, UErrorCode& status) const override { + const LocaleKey* lkey = dynamic_cast(&key); + U_ASSERT(lkey != nullptr); + int32_t kind = lkey->kind(); + Locale loc; + lkey->currentLocale(loc); + return NumberFormat::makeInstance(loc, (UNumberFormatStyle)kind, status); + } + + virtual UBool isDefault() const override { + return countFactories() == 1; + } +}; + +ICUNumberFormatService::~ICUNumberFormatService() {} + +// ------------------------------------- + +static void U_CALLCONV initNumberFormatService() { + U_ASSERT(gService == nullptr); + ucln_i18n_registerCleanup(UCLN_I18N_NUMFMT, numfmt_cleanup); + gService = new ICUNumberFormatService(); +} + +static ICULocaleService* +getNumberFormatService() +{ + umtx_initOnce(gServiceInitOnce, &initNumberFormatService); + return gService; +} + +static UBool haveService() { + return !gServiceInitOnce.isReset() && (getNumberFormatService() != nullptr); +} + +// ------------------------------------- + +URegistryKey U_EXPORT2 +NumberFormat::registerFactory(NumberFormatFactory* toAdopt, UErrorCode& status) +{ + if (U_FAILURE(status)) { + delete toAdopt; + return nullptr; + } + ICULocaleService *service = getNumberFormatService(); + if (service) { + NFFactory *tempnnf = new NFFactory(toAdopt); + if (tempnnf != nullptr) { + return service->registerFactory(tempnnf, status); + } + } + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; +} + +// ------------------------------------- + +UBool U_EXPORT2 +NumberFormat::unregister(URegistryKey key, UErrorCode& status) +{ + if (U_FAILURE(status)) { + return false; + } + if (haveService()) { + return gService->unregister(key, status); + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return false; + } +} + +// ------------------------------------- +StringEnumeration* U_EXPORT2 +NumberFormat::getAvailableLocales() +{ + ICULocaleService *service = getNumberFormatService(); + if (service) { + return service->getAvailableLocales(); + } + return nullptr; // no way to return error condition +} +#endif /* UCONFIG_NO_SERVICE */ +// ------------------------------------- + +enum { kKeyValueLenMax = 32 }; + +NumberFormat* +NumberFormat::internalCreateInstance(const Locale& loc, UNumberFormatStyle kind, UErrorCode& status) { + if (kind == UNUM_CURRENCY) { + char cfKeyValue[kKeyValueLenMax] = {0}; + UErrorCode kvStatus = U_ZERO_ERROR; + int32_t kLen = loc.getKeywordValue("cf", cfKeyValue, kKeyValueLenMax, kvStatus); + if (U_SUCCESS(kvStatus) && kLen > 0 && uprv_strcmp(cfKeyValue,"account")==0) { + kind = UNUM_CURRENCY_ACCOUNTING; + } + } +#if !UCONFIG_NO_SERVICE + if (haveService()) { + return (NumberFormat*)gService->get(loc, kind, status); + } +#endif + return makeInstance(loc, kind, status); +} + +NumberFormat* U_EXPORT2 +NumberFormat::createInstance(const Locale& loc, UNumberFormatStyle kind, UErrorCode& status) { + if (kind != UNUM_DECIMAL) { + return internalCreateInstance(loc, kind, status); + } + const SharedNumberFormat *shared = createSharedInstance(loc, kind, status); + if (U_FAILURE(status)) { + return nullptr; + } + NumberFormat *result = (*shared)->clone(); + shared->removeRef(); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return result; +} + + +// ------------------------------------- +// Checks if the thousand/10 thousand grouping is used in the +// NumberFormat instance. + +UBool +NumberFormat::isGroupingUsed() const +{ + return fGroupingUsed; +} + +// ------------------------------------- +// Sets to use the thousand/10 thousand grouping in the +// NumberFormat instance. + +void +NumberFormat::setGroupingUsed(UBool newValue) +{ + fGroupingUsed = newValue; +} + +// ------------------------------------- +// Gets the maximum number of digits for the integral part for +// this NumberFormat instance. + +int32_t NumberFormat::getMaximumIntegerDigits() const +{ + return fMaxIntegerDigits; +} + +// ------------------------------------- +// Sets the maximum number of digits for the integral part for +// this NumberFormat instance. + +void +NumberFormat::setMaximumIntegerDigits(int32_t newValue) +{ + fMaxIntegerDigits = uprv_max(0, uprv_min(newValue, gDefaultMaxIntegerDigits)); + if(fMinIntegerDigits > fMaxIntegerDigits) + fMinIntegerDigits = fMaxIntegerDigits; +} + +// ------------------------------------- +// Gets the minimum number of digits for the integral part for +// this NumberFormat instance. + +int32_t +NumberFormat::getMinimumIntegerDigits() const +{ + return fMinIntegerDigits; +} + +// ------------------------------------- +// Sets the minimum number of digits for the integral part for +// this NumberFormat instance. + +void +NumberFormat::setMinimumIntegerDigits(int32_t newValue) +{ + fMinIntegerDigits = uprv_max(0, uprv_min(newValue, gDefaultMinIntegerDigits)); + if(fMinIntegerDigits > fMaxIntegerDigits) + fMaxIntegerDigits = fMinIntegerDigits; +} + +// ------------------------------------- +// Gets the maximum number of digits for the fractional part for +// this NumberFormat instance. + +int32_t +NumberFormat::getMaximumFractionDigits() const +{ + return fMaxFractionDigits; +} + +// ------------------------------------- +// Sets the maximum number of digits for the fractional part for +// this NumberFormat instance. + +void +NumberFormat::setMaximumFractionDigits(int32_t newValue) +{ + fMaxFractionDigits = uprv_max(0, uprv_min(newValue, gDefaultMaxIntegerDigits)); + if(fMaxFractionDigits < fMinFractionDigits) + fMinFractionDigits = fMaxFractionDigits; +} + +// ------------------------------------- +// Gets the minimum number of digits for the fractional part for +// this NumberFormat instance. + +int32_t +NumberFormat::getMinimumFractionDigits() const +{ + return fMinFractionDigits; +} + +// ------------------------------------- +// Sets the minimum number of digits for the fractional part for +// this NumberFormat instance. + +void +NumberFormat::setMinimumFractionDigits(int32_t newValue) +{ + fMinFractionDigits = uprv_max(0, uprv_min(newValue, gDefaultMinIntegerDigits)); + if (fMaxFractionDigits < fMinFractionDigits) + fMaxFractionDigits = fMinFractionDigits; +} + +// ------------------------------------- + +void NumberFormat::setCurrency(const char16_t* theCurrency, UErrorCode& ec) { + if (U_FAILURE(ec)) { + return; + } + if (theCurrency) { + u_strncpy(fCurrency, theCurrency, 3); + fCurrency[3] = 0; + } else { + fCurrency[0] = 0; + } +} + +const char16_t* NumberFormat::getCurrency() const { + return fCurrency; +} + +void NumberFormat::getEffectiveCurrency(char16_t* result, UErrorCode& ec) const { + const char16_t* c = getCurrency(); + if (*c != 0) { + u_strncpy(result, c, 3); + result[3] = 0; + } else { + const char* loc = getLocaleID(ULOC_VALID_LOCALE, ec); + if (loc == nullptr) { + loc = uloc_getDefault(); + } + ucurr_forLocale(loc, result, 4, &ec); + } +} + +//---------------------------------------------------------------------- + + +void NumberFormat::setContext(UDisplayContext value, UErrorCode& status) +{ + if (U_FAILURE(status)) + return; + if ( (UDisplayContextType)((uint32_t)value >> 8) == UDISPCTX_TYPE_CAPITALIZATION ) { + fCapitalizationContext = value; + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + } +} + + +UDisplayContext NumberFormat::getContext(UDisplayContextType type, UErrorCode& status) const +{ + if (U_FAILURE(status)) + return (UDisplayContext)0; + if (type != UDISPCTX_TYPE_CAPITALIZATION) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return (UDisplayContext)0; + } + return fCapitalizationContext; +} + + +// ------------------------------------- +// Creates the NumberFormat instance of the specified style (number, currency, +// or percent) for the desired locale. + +static void U_CALLCONV nscacheInit() { + U_ASSERT(NumberingSystem_cache == nullptr); + ucln_i18n_registerCleanup(UCLN_I18N_NUMFMT, numfmt_cleanup); + UErrorCode status = U_ZERO_ERROR; + NumberingSystem_cache = uhash_open(uhash_hashLong, + uhash_compareLong, + nullptr, + &status); + if (U_FAILURE(status)) { + // Number Format code will run with no cache if creation fails. + NumberingSystem_cache = nullptr; + return; + } + uhash_setValueDeleter(NumberingSystem_cache, deleteNumberingSystem); +} + +template<> U_I18N_API +const SharedNumberFormat *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); + NumberFormat *nf = NumberFormat::internalCreateInstance( + localeId, UNUM_DECIMAL, status); + if (U_FAILURE(status)) { + return nullptr; + } + SharedNumberFormat *result = new SharedNumberFormat(nf); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + delete nf; + return nullptr; + } + result->addRef(); + return result; +} + +const SharedNumberFormat* U_EXPORT2 +NumberFormat::createSharedInstance(const Locale& loc, UNumberFormatStyle kind, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + if (kind != UNUM_DECIMAL) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + const SharedNumberFormat *result = nullptr; + UnifiedCache::getByLocale(loc, result, status); + return result; +} + +UBool +NumberFormat::isStyleSupported(UNumberFormatStyle style) { + return gLastResortNumberPatterns[style] != nullptr; +} + +NumberFormat* +NumberFormat::makeInstance(const Locale& desiredLocale, + UNumberFormatStyle style, + UErrorCode& status) { + return makeInstance(desiredLocale, style, false, status); +} + +NumberFormat* +NumberFormat::makeInstance(const Locale& desiredLocale, + UNumberFormatStyle style, + UBool mustBeDecimalFormat, + UErrorCode& status) { + if (U_FAILURE(status)) return nullptr; + + if (style < 0 || style >= UNUM_FORMAT_STYLE_COUNT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + // For the purposes of general number formatting, UNUM_NUMBERING_SYSTEM should behave the same + // was as UNUM_DECIMAL. In both cases, you get either a DecimalFormat or a RuleBasedNumberFormat + // depending on the locale's numbering system (either the default one for the locale or a specific + // one specified by using the "@numbers=" or "-u-nu-" parameter in the locale ID. + if (style == UNUM_NUMBERING_SYSTEM) { + style = UNUM_DECIMAL; + } + + // Some styles are not supported. This is a result of merging + // the @draft ICU 4.2 NumberFormat::EStyles into the long-existing UNumberFormatStyle. + // Ticket #8503 is for reviewing/fixing/merging the two relevant implementations: + // this one and unum_open(). + // The UNUM_PATTERN_ styles are not supported here + // because this method does not take a pattern string. + if (!isStyleSupported(style)) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + +#if U_PLATFORM_USES_ONLY_WIN32_API + if (!mustBeDecimalFormat) { + char buffer[8]; + int32_t count = desiredLocale.getKeywordValue("compat", buffer, sizeof(buffer), status); + + // if the locale has "@compat=host", create a host-specific NumberFormat + if (U_SUCCESS(status) && count > 0 && uprv_strcmp(buffer, "host") == 0) { + UBool curr = true; + + switch (style) { + case UNUM_DECIMAL: + curr = false; + // fall-through + U_FALLTHROUGH; + + case UNUM_CURRENCY: + case UNUM_CURRENCY_ISO: // do not support plural formatting here + case UNUM_CURRENCY_PLURAL: + case UNUM_CURRENCY_ACCOUNTING: + case UNUM_CASH_CURRENCY: + case UNUM_CURRENCY_STANDARD: + { + LocalPointer f(new Win32NumberFormat(desiredLocale, curr, status), status); + if (U_SUCCESS(status)) { + return f.orphan(); + } + } + break; + default: + break; + } + } + } +#endif + // Use numbering system cache hashtable + umtx_initOnce(gNSCacheInitOnce, &nscacheInit); + + // Get cached numbering system + LocalPointer ownedNs; + NumberingSystem *ns = nullptr; + if (NumberingSystem_cache != nullptr) { + // TODO: Bad hash key usage, see ticket #8504. + int32_t hashKey = desiredLocale.hashCode(); + + static UMutex nscacheMutex; + Mutex lock(&nscacheMutex); + ns = (NumberingSystem *)uhash_iget(NumberingSystem_cache, hashKey); + if (ns == nullptr) { + ns = NumberingSystem::createInstance(desiredLocale,status); + uhash_iput(NumberingSystem_cache, hashKey, (void*)ns, &status); + } + } else { + ownedNs.adoptInstead(NumberingSystem::createInstance(desiredLocale,status)); + ns = ownedNs.getAlias(); + } + + // check results of getting a numbering system + if (U_FAILURE(status)) { + return nullptr; + } + + if (mustBeDecimalFormat && ns->isAlgorithmic()) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + + LocalPointer symbolsToAdopt; + UnicodeString pattern; + LocalUResourceBundlePointer ownedResource(ures_open(nullptr, desiredLocale.getName(), &status)); + if (U_FAILURE(status)) { + return nullptr; + } + else { + // Loads the decimal symbols of the desired locale. + symbolsToAdopt.adoptInsteadAndCheckErrorCode(new DecimalFormatSymbols(desiredLocale, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + + // Load the pattern from data using the common library function + const char16_t* patternPtr = number::impl::utils::getPatternForStyle( + desiredLocale, + ns->getName(), + gFormatCldrStyles[style], + status); + pattern = UnicodeString(true, patternPtr, -1); + } + if (U_FAILURE(status)) { + return nullptr; + } + if(style==UNUM_CURRENCY || style == UNUM_CURRENCY_ISO || style == UNUM_CURRENCY_ACCOUNTING + || style == UNUM_CASH_CURRENCY || style == UNUM_CURRENCY_STANDARD){ + const char16_t* currPattern = symbolsToAdopt->getCurrencyPattern(); + if(currPattern!=nullptr){ + pattern.setTo(currPattern, u_strlen(currPattern)); + } + } + + LocalPointer f; + if (ns->isAlgorithmic()) { + UnicodeString nsDesc; + UnicodeString nsRuleSetGroup; + UnicodeString nsRuleSetName; + Locale nsLoc; + URBNFRuleSetTag desiredRulesType = URBNF_NUMBERING_SYSTEM; + + nsDesc.setTo(ns->getDescription()); + int32_t firstSlash = nsDesc.indexOf(gSlash); + int32_t lastSlash = nsDesc.lastIndexOf(gSlash); + if ( lastSlash > firstSlash ) { + CharString nsLocID; + + nsLocID.appendInvariantChars(nsDesc.tempSubString(0, firstSlash), status); + nsRuleSetGroup.setTo(nsDesc,firstSlash+1,lastSlash-firstSlash-1); + nsRuleSetName.setTo(nsDesc,lastSlash+1); + + nsLoc = Locale::createFromName(nsLocID.data()); + + UnicodeString SpelloutRules = UNICODE_STRING_SIMPLE("SpelloutRules"); + if ( nsRuleSetGroup.compare(SpelloutRules) == 0 ) { + desiredRulesType = URBNF_SPELLOUT; + } + } else { + nsLoc = desiredLocale; + nsRuleSetName.setTo(nsDesc); + } + + RuleBasedNumberFormat *r = new RuleBasedNumberFormat(desiredRulesType,nsLoc,status); + if (r == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + r->setDefaultRuleSet(nsRuleSetName,status); + f.adoptInstead(r); + } else { + // replace single currency sign in the pattern with double currency sign + // if the style is UNUM_CURRENCY_ISO + if (style == UNUM_CURRENCY_ISO) { + pattern.findAndReplace(UnicodeString(true, gSingleCurrencySign, 1), + UnicodeString(true, gDoubleCurrencySign, 2)); + } + + // "new DecimalFormat()" does not adopt the symbols argument if its memory allocation fails. + // So we can't use adoptInsteadAndCheckErrorCode as we need to know if the 'new' failed. + DecimalFormatSymbols *syms = symbolsToAdopt.getAlias(); + LocalPointer df(new DecimalFormat(pattern, syms, style, status)); + + if (df.isValid()) { + // if the DecimalFormat object was successfully new'ed, then it will own symbolsToAdopt, even if the status is a failure. + symbolsToAdopt.orphan(); + } + else { + status = U_MEMORY_ALLOCATION_ERROR; + } + + if (U_FAILURE(status)) { + return nullptr; + } + + // if it is cash currency style, setCurrencyUsage with usage + if (style == UNUM_CASH_CURRENCY){ + df->setCurrencyUsage(UCURR_USAGE_CASH, &status); + } + + if (U_FAILURE(status)) { + return nullptr; + } + + f.adoptInstead(df.orphan()); + } + + f->setLocaleIDs(ures_getLocaleByType(ownedResource.getAlias(), ULOC_VALID_LOCALE, &status), + ures_getLocaleByType(ownedResource.getAlias(), ULOC_ACTUAL_LOCALE, &status)); + if (U_FAILURE(status)) { + return nullptr; + } + return f.orphan(); +} + +/** + * Get the rounding mode. + * @return A rounding mode + */ +NumberFormat::ERoundingMode NumberFormat::getRoundingMode() const { + // Default value. ICU4J throws an exception and we can't change this API. + return NumberFormat::ERoundingMode::kRoundUnnecessary; +} + +/** + * Set the rounding mode. This has no effect unless the rounding + * increment is greater than zero. + * @param roundingMode A rounding mode + */ +void NumberFormat::setRoundingMode(NumberFormat::ERoundingMode /*roundingMode*/) { + // No-op ICU4J throws an exception, and we can't change this API. +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/numparse_affixes.cpp b/intl/icu/source/i18n/numparse_affixes.cpp new file mode 100644 index 0000000000..ad3d48b473 --- /dev/null +++ b/intl/icu/source/i18n/numparse_affixes.cpp @@ -0,0 +1,463 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_affixes.h" +#include "numparse_utils.h" +#include "number_utils.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; +using namespace icu::number; +using namespace icu::number::impl; + + +namespace { + +/** + * Helper method to return whether the given AffixPatternMatcher equals the given pattern string. + * Either both arguments must be null or the pattern string inside the AffixPatternMatcher must equal + * the given pattern string. + */ +static bool matched(const AffixPatternMatcher* affix, const UnicodeString& patternString) { + return (affix == nullptr && patternString.isBogus()) || + (affix != nullptr && affix->getPattern() == patternString); +} + +/** + * Helper method to return the length of the given AffixPatternMatcher. Returns 0 for null. + */ +static int32_t length(const AffixPatternMatcher* matcher) { + return matcher == nullptr ? 0 : matcher->getPattern().length(); +} + +/** + * Helper method to return whether (1) both lhs and rhs are null/invalid, or (2) if they are both + * valid, whether they are equal according to operator==. Similar to Java Objects.equals() + */ +static bool equals(const AffixPatternMatcher* lhs, const AffixPatternMatcher* rhs) { + if (lhs == nullptr && rhs == nullptr) { + return true; + } + if (lhs == nullptr || rhs == nullptr) { + return false; + } + return *lhs == *rhs; +} + +} + + +AffixPatternMatcherBuilder::AffixPatternMatcherBuilder(const UnicodeString& pattern, + AffixTokenMatcherWarehouse& warehouse, + IgnorablesMatcher* ignorables) + : fMatchersLen(0), + fLastTypeOrCp(0), + fPattern(pattern), + fWarehouse(warehouse), + fIgnorables(ignorables) {} + +void AffixPatternMatcherBuilder::consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) { + // This is called by AffixUtils.iterateWithConsumer() for each token. + + // Add an ignorables matcher between tokens except between two literals, and don't put two + // ignorables matchers in a row. + if (fIgnorables != nullptr && fMatchersLen > 0 && + (fLastTypeOrCp < 0 || !fIgnorables->getSet()->contains(fLastTypeOrCp))) { + addMatcher(*fIgnorables); + } + + if (type != TYPE_CODEPOINT) { + // Case 1: the token is a symbol. + switch (type) { + case TYPE_MINUS_SIGN: + addMatcher(fWarehouse.minusSign()); + break; + case TYPE_PLUS_SIGN: + addMatcher(fWarehouse.plusSign()); + break; + case TYPE_PERCENT: + addMatcher(fWarehouse.percent()); + break; + case TYPE_PERMILLE: + addMatcher(fWarehouse.permille()); + break; + case TYPE_CURRENCY_SINGLE: + case TYPE_CURRENCY_DOUBLE: + case TYPE_CURRENCY_TRIPLE: + case TYPE_CURRENCY_QUAD: + case TYPE_CURRENCY_QUINT: + // All currency symbols use the same matcher + addMatcher(fWarehouse.currency(status)); + break; + default: + UPRV_UNREACHABLE_EXIT; + } + + } else if (fIgnorables != nullptr && fIgnorables->getSet()->contains(cp)) { + // Case 2: the token is an ignorable literal. + // No action necessary: the ignorables matcher has already been added. + + } else { + // Case 3: the token is a non-ignorable literal. + if (auto* ptr = fWarehouse.nextCodePointMatcher(cp, status)) { + addMatcher(*ptr); + } else { + // OOM; unwind the stack + return; + } + } + fLastTypeOrCp = type != TYPE_CODEPOINT ? type : cp; +} + +void AffixPatternMatcherBuilder::addMatcher(NumberParseMatcher& matcher) { + if (fMatchersLen >= fMatchers.getCapacity()) { + fMatchers.resize(fMatchersLen * 2, fMatchersLen); + } + fMatchers[fMatchersLen++] = &matcher; +} + +AffixPatternMatcher AffixPatternMatcherBuilder::build(UErrorCode& status) { + return AffixPatternMatcher(fMatchers, fMatchersLen, fPattern, status); +} + +AffixTokenMatcherWarehouse::AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData) + : fSetupData(setupData) {} + +NumberParseMatcher& AffixTokenMatcherWarehouse::minusSign() { + return fMinusSign = {fSetupData->dfs, true}; +} + +NumberParseMatcher& AffixTokenMatcherWarehouse::plusSign() { + return fPlusSign = {fSetupData->dfs, true}; +} + +NumberParseMatcher& AffixTokenMatcherWarehouse::percent() { + return fPercent = {fSetupData->dfs}; +} + +NumberParseMatcher& AffixTokenMatcherWarehouse::permille() { + return fPermille = {fSetupData->dfs}; +} + +NumberParseMatcher& AffixTokenMatcherWarehouse::currency(UErrorCode& status) { + return fCurrency = {fSetupData->currencySymbols, fSetupData->dfs, fSetupData->parseFlags, status}; +} + +IgnorablesMatcher& AffixTokenMatcherWarehouse::ignorables() { + return fSetupData->ignorables; +} + +NumberParseMatcher* AffixTokenMatcherWarehouse::nextCodePointMatcher(UChar32 cp, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + auto* result = fCodePoints.create(cp); + if (result == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return result; +} + +bool AffixTokenMatcherWarehouse::hasEmptyCurrencySymbol() const { + return fSetupData->currencySymbols.hasEmptyCurrencySymbol(); +} + + +CodePointMatcher::CodePointMatcher(UChar32 cp) + : fCp(cp) {} + +bool CodePointMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const { + if (segment.startsWith(fCp)) { + segment.adjustOffsetByCodePoint(); + result.setCharsConsumed(segment); + } + return false; +} + +bool CodePointMatcher::smokeTest(const StringSegment& segment) const { + return segment.startsWith(fCp); +} + +UnicodeString CodePointMatcher::toString() const { + return u""; +} + + +AffixPatternMatcher AffixPatternMatcher::fromAffixPattern(const UnicodeString& affixPattern, + AffixTokenMatcherWarehouse& tokenWarehouse, + parse_flags_t parseFlags, bool* success, + UErrorCode& status) { + if (affixPattern.isEmpty()) { + *success = false; + return {}; + } + *success = true; + + IgnorablesMatcher* ignorables; + if (0 != (parseFlags & PARSE_FLAG_EXACT_AFFIX)) { + ignorables = nullptr; + } else { + ignorables = &tokenWarehouse.ignorables(); + } + + AffixPatternMatcherBuilder builder(affixPattern, tokenWarehouse, ignorables); + AffixUtils::iterateWithConsumer(affixPattern, builder, status); + return builder.build(status); +} + +AffixPatternMatcher::AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen, + const UnicodeString& pattern, UErrorCode& status) + : ArraySeriesMatcher(matchers, matchersLen), fPattern(pattern, status) { +} + +UnicodeString AffixPatternMatcher::getPattern() const { + return fPattern.toAliasedUnicodeString(); +} + +bool AffixPatternMatcher::operator==(const AffixPatternMatcher& other) const { + return fPattern == other.fPattern; +} + + +AffixMatcherWarehouse::AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse) + : fTokenWarehouse(tokenWarehouse) { +} + +bool AffixMatcherWarehouse::isInteresting(const AffixPatternProvider& patternInfo, + const IgnorablesMatcher& ignorables, parse_flags_t parseFlags, + UErrorCode& status) { + UnicodeString posPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_PREFIX); + UnicodeString posSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_SUFFIX); + UnicodeString negPrefixString; + UnicodeString negSuffixString; + if (patternInfo.hasNegativeSubpattern()) { + negPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_PREFIX); + negSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_SUFFIX); + } + + if (0 == (parseFlags & PARSE_FLAG_USE_FULL_AFFIXES) && + AffixUtils::containsOnlySymbolsAndIgnorables(posPrefixString, *ignorables.getSet(), status) && + AffixUtils::containsOnlySymbolsAndIgnorables(posSuffixString, *ignorables.getSet(), status) && + AffixUtils::containsOnlySymbolsAndIgnorables(negPrefixString, *ignorables.getSet(), status) && + AffixUtils::containsOnlySymbolsAndIgnorables(negSuffixString, *ignorables.getSet(), status) + // HACK: Plus and minus sign are a special case: we accept them trailing only if they are + // trailing in the pattern string. + && !AffixUtils::containsType(posSuffixString, TYPE_PLUS_SIGN, status) && + !AffixUtils::containsType(posSuffixString, TYPE_MINUS_SIGN, status) && + !AffixUtils::containsType(negSuffixString, TYPE_PLUS_SIGN, status) && + !AffixUtils::containsType(negSuffixString, TYPE_MINUS_SIGN, status)) { + // The affixes contain only symbols and ignorables. + // No need to generate affix matchers. + return false; + } + return true; +} + +void AffixMatcherWarehouse::createAffixMatchers(const AffixPatternProvider& patternInfo, + MutableMatcherCollection& output, + const IgnorablesMatcher& ignorables, + parse_flags_t parseFlags, UErrorCode& status) { + if (!isInteresting(patternInfo, ignorables, parseFlags, status)) { + return; + } + + // The affixes have interesting characters, or we are in strict mode. + // Use initial capacity of 6, the highest possible number of AffixMatchers. + UnicodeString sb; + bool includeUnpaired = 0 != (parseFlags & PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES); + + int32_t numAffixMatchers = 0; + int32_t numAffixPatternMatchers = 0; + + AffixPatternMatcher* posPrefix = nullptr; + AffixPatternMatcher* posSuffix = nullptr; + + // Pre-process the affix strings to resolve LDML rules like sign display. + for (int8_t typeInt = 0; typeInt < PATTERN_SIGN_TYPE_COUNT * 2; typeInt++) { + auto type = static_cast(typeInt / 2); + bool dropCurrencySymbols = (typeInt % 2) == 1; + + if (dropCurrencySymbols && !patternInfo.hasCurrencySign()) { + continue; + } + if (dropCurrencySymbols && !fTokenWarehouse->hasEmptyCurrencySymbol()) { + continue; + } + + // Skip affixes in some cases + if (type == PATTERN_SIGN_TYPE_POS + && 0 != (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) { + continue; + } + if (type == PATTERN_SIGN_TYPE_POS_SIGN + && 0 == (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) { + continue; + } + + // Generate Prefix + // TODO: Handle approximately sign? + bool hasPrefix = false; + PatternStringUtils::patternInfoToStringBuilder( + patternInfo, true, type, false, StandardPlural::OTHER, false, dropCurrencySymbols, sb); + fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern( + sb, *fTokenWarehouse, parseFlags, &hasPrefix, status); + AffixPatternMatcher* prefix = hasPrefix ? &fAffixPatternMatchers[numAffixPatternMatchers++] + : nullptr; + + // Generate Suffix + // TODO: Handle approximately sign? + bool hasSuffix = false; + PatternStringUtils::patternInfoToStringBuilder( + patternInfo, false, type, false, StandardPlural::OTHER, false, dropCurrencySymbols, sb); + fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern( + sb, *fTokenWarehouse, parseFlags, &hasSuffix, status); + AffixPatternMatcher* suffix = hasSuffix ? &fAffixPatternMatchers[numAffixPatternMatchers++] + : nullptr; + + if (type == PATTERN_SIGN_TYPE_POS) { + posPrefix = prefix; + posSuffix = suffix; + } else if (equals(prefix, posPrefix) && equals(suffix, posSuffix)) { + // Skip adding these matchers (we already have equivalents) + continue; + } + + // Flags for setting in the ParsedNumber; the token matchers may add more. + int flags = (type == PATTERN_SIGN_TYPE_NEG) ? FLAG_NEGATIVE : 0; + + // Note: it is indeed possible for posPrefix and posSuffix to both be null. + // We still need to add that matcher for strict mode to work. + fAffixMatchers[numAffixMatchers++] = {prefix, suffix, flags}; + if (includeUnpaired && prefix != nullptr && suffix != nullptr) { + // The following if statements are designed to prevent adding two identical matchers. + if (type == PATTERN_SIGN_TYPE_POS || !equals(prefix, posPrefix)) { + fAffixMatchers[numAffixMatchers++] = {prefix, nullptr, flags}; + } + if (type == PATTERN_SIGN_TYPE_POS || !equals(suffix, posSuffix)) { + fAffixMatchers[numAffixMatchers++] = {nullptr, suffix, flags}; + } + } + } + + // Put the AffixMatchers in order, and then add them to the output. + // Since there are at most 9 elements, do a simple-to-implement bubble sort. + bool madeChanges; + do { + madeChanges = false; + for (int32_t i = 1; i < numAffixMatchers; i++) { + if (fAffixMatchers[i - 1].compareTo(fAffixMatchers[i]) > 0) { + madeChanges = true; + AffixMatcher temp = std::move(fAffixMatchers[i - 1]); + fAffixMatchers[i - 1] = std::move(fAffixMatchers[i]); + fAffixMatchers[i] = std::move(temp); + } + } + } while (madeChanges); + + for (int32_t i = 0; i < numAffixMatchers; i++) { + // Enable the following line to debug affixes + //std::cout << "Adding affix matcher: " << CStr(fAffixMatchers[i].toString())() << std::endl; + output.addMatcher(fAffixMatchers[i]); + } +} + + +AffixMatcher::AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags) + : fPrefix(prefix), fSuffix(suffix), fFlags(flags) {} + +bool AffixMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { + if (!result.seenNumber()) { + // Prefix + // Do not match if: + // 1. We have already seen a prefix (result.prefix != null) + // 2. The prefix in this AffixMatcher is empty (prefix == null) + if (!result.prefix.isBogus() || fPrefix == nullptr) { + return false; + } + + // Attempt to match the prefix. + int initialOffset = segment.getOffset(); + bool maybeMore = fPrefix->match(segment, result, status); + if (initialOffset != segment.getOffset()) { + result.prefix = fPrefix->getPattern(); + } + return maybeMore; + + } else { + // Suffix + // Do not match if: + // 1. We have already seen a suffix (result.suffix != null) + // 2. The suffix in this AffixMatcher is empty (suffix == null) + // 3. The matched prefix does not equal this AffixMatcher's prefix + if (!result.suffix.isBogus() || fSuffix == nullptr || !matched(fPrefix, result.prefix)) { + return false; + } + + // Attempt to match the suffix. + int initialOffset = segment.getOffset(); + bool maybeMore = fSuffix->match(segment, result, status); + if (initialOffset != segment.getOffset()) { + result.suffix = fSuffix->getPattern(); + } + return maybeMore; + } +} + +bool AffixMatcher::smokeTest(const StringSegment& segment) const { + return (fPrefix != nullptr && fPrefix->smokeTest(segment)) || + (fSuffix != nullptr && fSuffix->smokeTest(segment)); +} + +void AffixMatcher::postProcess(ParsedNumber& result) const { + // Check to see if our affix is the one that was matched. If so, set the flags in the result. + if (matched(fPrefix, result.prefix) && matched(fSuffix, result.suffix)) { + // Fill in the result prefix and suffix with non-null values (empty string). + // Used by strict mode to determine whether an entire affix pair was matched. + if (result.prefix.isBogus()) { + result.prefix = UnicodeString(); + } + if (result.suffix.isBogus()) { + result.suffix = UnicodeString(); + } + result.flags |= fFlags; + if (fPrefix != nullptr) { + fPrefix->postProcess(result); + } + if (fSuffix != nullptr) { + fSuffix->postProcess(result); + } + } +} + +int8_t AffixMatcher::compareTo(const AffixMatcher& rhs) const { + const AffixMatcher& lhs = *this; + if (length(lhs.fPrefix) != length(rhs.fPrefix)) { + return length(lhs.fPrefix) > length(rhs.fPrefix) ? -1 : 1; + } else if (length(lhs.fSuffix) != length(rhs.fSuffix)) { + return length(lhs.fSuffix) > length(rhs.fSuffix) ? -1 : 1; + } else { + return 0; + } +} + +UnicodeString AffixMatcher::toString() const { + bool isNegative = 0 != (fFlags & FLAG_NEGATIVE); + return UnicodeString(u"getPattern() : u"null") + u"#" + + (fSuffix ? fSuffix->getPattern() : u"null") + u">"; + +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_affixes.h b/intl/icu/source/i18n/numparse_affixes.h new file mode 100644 index 0000000000..81b633c262 --- /dev/null +++ b/intl/icu/source/i18n/numparse_affixes.h @@ -0,0 +1,230 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_AFFIXES_H__ +#define __NUMPARSE_AFFIXES_H__ + +#include "cmemory.h" + +#include "numparse_types.h" +#include "numparse_symbols.h" +#include "numparse_currency.h" +#include "number_affixutils.h" +#include "number_currencysymbols.h" + +U_NAMESPACE_BEGIN +namespace numparse { +namespace impl { + +// Forward-declaration of implementation classes for friending +class AffixPatternMatcherBuilder; +class AffixPatternMatcher; + +using ::icu::number::impl::AffixPatternProvider; +using ::icu::number::impl::TokenConsumer; +using ::icu::number::impl::CurrencySymbols; + + +class U_I18N_API CodePointMatcher : public NumberParseMatcher, public UMemory { + public: + CodePointMatcher() = default; // WARNING: Leaves the object in an unusable state + + CodePointMatcher(UChar32 cp); + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool smokeTest(const StringSegment& segment) const override; + + UnicodeString toString() const override; + + private: + UChar32 fCp; +}; + +} // namespace impl +} // namespace numparse + +// Export a explicit template instantiations of MaybeStackArray, MemoryPool and CompactUnicodeString. +// When building DLLs for Windows this is required even though no direct access leaks out of the i18n library. +// (See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples.) +// Note: These need to be outside of the numparse::impl namespace, or Clang will generate a compile error. +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +template class U_I18N_API MaybeStackArray; +template class U_I18N_API MemoryPool; +template class U_I18N_API numparse::impl::CompactUnicodeString<4>; +#endif + +namespace numparse { +namespace impl { + +struct AffixTokenMatcherSetupData { + const CurrencySymbols& currencySymbols; + const DecimalFormatSymbols& dfs; + IgnorablesMatcher& ignorables; + const Locale& locale; + parse_flags_t parseFlags; +}; + + +/** + * Small helper class that generates matchers for individual tokens for AffixPatternMatcher. + * + * In Java, this is called AffixTokenMatcherFactory (a "factory"). However, in C++, it is called a + * "warehouse", because in addition to generating the matchers, it also retains ownership of them. The + * warehouse must stay in scope for the whole lifespan of the AffixPatternMatcher that uses matchers from + * the warehouse. + * + * @author sffc + */ +// Exported as U_I18N_API for tests +class U_I18N_API AffixTokenMatcherWarehouse : public UMemory { + public: + AffixTokenMatcherWarehouse() = default; // WARNING: Leaves the object in an unusable state + + AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData); + + NumberParseMatcher& minusSign(); + + NumberParseMatcher& plusSign(); + + NumberParseMatcher& percent(); + + NumberParseMatcher& permille(); + + NumberParseMatcher& currency(UErrorCode& status); + + IgnorablesMatcher& ignorables(); + + NumberParseMatcher* nextCodePointMatcher(UChar32 cp, UErrorCode& status); + + bool hasEmptyCurrencySymbol() const; + + private: + // NOTE: The following field may be unsafe to access after construction is done! + const AffixTokenMatcherSetupData* fSetupData; + + // NOTE: These are default-constructed and should not be used until initialized. + MinusSignMatcher fMinusSign; + PlusSignMatcher fPlusSign; + PercentMatcher fPercent; + PermilleMatcher fPermille; + CombinedCurrencyMatcher fCurrency; + + // Use a child class for code point matchers, since it requires non-default operators. + MemoryPool fCodePoints; + + friend class AffixPatternMatcherBuilder; + friend class AffixPatternMatcher; +}; + + +class AffixPatternMatcherBuilder : public TokenConsumer, public MutableMatcherCollection { + public: + AffixPatternMatcherBuilder(const UnicodeString& pattern, AffixTokenMatcherWarehouse& warehouse, + IgnorablesMatcher* ignorables); + + void consumeToken(::icu::number::impl::AffixPatternType type, UChar32 cp, UErrorCode& status) override; + + /** NOTE: You can build only once! */ + AffixPatternMatcher build(UErrorCode& status); + + private: + ArraySeriesMatcher::MatcherArray fMatchers; + int32_t fMatchersLen; + int32_t fLastTypeOrCp; + + const UnicodeString& fPattern; + AffixTokenMatcherWarehouse& fWarehouse; + IgnorablesMatcher* fIgnorables; + + void addMatcher(NumberParseMatcher& matcher) override; +}; + + +// Exported as U_I18N_API for tests +class U_I18N_API AffixPatternMatcher : public ArraySeriesMatcher { + public: + AffixPatternMatcher() = default; // WARNING: Leaves the object in an unusable state + + static AffixPatternMatcher fromAffixPattern(const UnicodeString& affixPattern, + AffixTokenMatcherWarehouse& warehouse, + parse_flags_t parseFlags, bool* success, + UErrorCode& status); + + UnicodeString getPattern() const; + + bool operator==(const AffixPatternMatcher& other) const; + + private: + CompactUnicodeString<4> fPattern; + + AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen, const UnicodeString& pattern, + UErrorCode& status); + + friend class AffixPatternMatcherBuilder; +}; + + +class AffixMatcher : public NumberParseMatcher, public UMemory { + public: + AffixMatcher() = default; // WARNING: Leaves the object in an unusable state + + AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags); + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + void postProcess(ParsedNumber& result) const override; + + bool smokeTest(const StringSegment& segment) const override; + + int8_t compareTo(const AffixMatcher& rhs) const; + + UnicodeString toString() const override; + + private: + AffixPatternMatcher* fPrefix; + AffixPatternMatcher* fSuffix; + result_flags_t fFlags; +}; + + +/** + * A C++-only class to retain ownership of the AffixMatchers needed for parsing. + */ +class AffixMatcherWarehouse { + public: + AffixMatcherWarehouse() = default; // WARNING: Leaves the object in an unusable state + + AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse); + + void createAffixMatchers(const AffixPatternProvider& patternInfo, MutableMatcherCollection& output, + const IgnorablesMatcher& ignorables, parse_flags_t parseFlags, + UErrorCode& status); + + private: + // 18 is the limit: positive, zero, and negative, each with prefix, suffix, and prefix+suffix, + // and doubled since there may be an empty currency symbol + AffixMatcher fAffixMatchers[18]; + // 6 is the limit: positive, zero, and negative, a prefix and a suffix for each, + // and doubled since there may be an empty currency symbol + AffixPatternMatcher fAffixPatternMatchers[12]; + // Reference to the warehouse for tokens used by the AffixPatternMatchers + AffixTokenMatcherWarehouse* fTokenWarehouse; + + friend class AffixMatcher; + + static bool isInteresting(const AffixPatternProvider& patternInfo, const IgnorablesMatcher& ignorables, + parse_flags_t parseFlags, UErrorCode& status); +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_AFFIXES_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_compositions.cpp b/intl/icu/source/i18n/numparse_compositions.cpp new file mode 100644 index 0000000000..2f7e1ab28d --- /dev/null +++ b/intl/icu/source/i18n/numparse_compositions.cpp @@ -0,0 +1,108 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_compositions.h" +#include "string_segment.h" +#include "unicode/uniset.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +bool SeriesMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { + ParsedNumber backup(result); + + int32_t initialOffset = segment.getOffset(); + bool maybeMore = true; + for (auto* it = begin(); it < end();) { + const NumberParseMatcher* matcher = *it; + int matcherOffset = segment.getOffset(); + if (segment.length() != 0) { + maybeMore = matcher->match(segment, result, status); + } else { + // Nothing for this matcher to match; ask for more. + maybeMore = true; + } + + bool success = (segment.getOffset() != matcherOffset); + bool isFlexible = matcher->isFlexible(); + if (success && isFlexible) { + // Match succeeded, and this is a flexible matcher. Re-run it. + } else if (success) { + // Match succeeded, and this is NOT a flexible matcher. Proceed to the next matcher. + it++; + // Small hack: if there is another matcher coming, do not accept trailing weak chars. + // Needed for proper handling of currency spacing. + if (it < end() && segment.getOffset() != result.charEnd && result.charEnd > matcherOffset) { + segment.setOffset(result.charEnd); + } + } else if (isFlexible) { + // Match failed, and this is a flexible matcher. Try again with the next matcher. + it++; + } else { + // Match failed, and this is NOT a flexible matcher. Exit. + segment.setOffset(initialOffset); + result = backup; + return maybeMore; + } + } + + // All matchers in the series succeeded. + return maybeMore; +} + +bool SeriesMatcher::smokeTest(const StringSegment& segment) const { + // NOTE: The range-based for loop calls the virtual begin() and end() methods. + // NOTE: We only want the first element. Use the for loop for boundary checking. + for (auto& matcher : *this) { + // SeriesMatchers are never allowed to start with a Flexible matcher. + U_ASSERT(!matcher->isFlexible()); + return matcher->smokeTest(segment); + } + return false; +} + +void SeriesMatcher::postProcess(ParsedNumber& result) const { + // NOTE: The range-based for loop calls the virtual begin() and end() methods. + for (auto* matcher : *this) { + matcher->postProcess(result); + } +} + + +ArraySeriesMatcher::ArraySeriesMatcher() + : fMatchersLen(0) { +} + +ArraySeriesMatcher::ArraySeriesMatcher(MatcherArray& matchers, int32_t matchersLen) + : fMatchers(std::move(matchers)), fMatchersLen(matchersLen) { +} + +int32_t ArraySeriesMatcher::length() const { + return fMatchersLen; +} + +const NumberParseMatcher* const* ArraySeriesMatcher::begin() const { + return fMatchers.getAlias(); +} + +const NumberParseMatcher* const* ArraySeriesMatcher::end() const { + return fMatchers.getAlias() + fMatchersLen; +} + +UnicodeString ArraySeriesMatcher::toString() const { + return u""; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_compositions.h b/intl/icu/source/i18n/numparse_compositions.h new file mode 100644 index 0000000000..f085912def --- /dev/null +++ b/intl/icu/source/i18n/numparse_compositions.h @@ -0,0 +1,124 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMPARSE_COMPOSITIONS__ +#define __SOURCE_NUMPARSE_COMPOSITIONS__ + +#include "numparse_types.h" + +U_NAMESPACE_BEGIN + +// Export an explicit template instantiation of the MaybeStackArray that is used as a data member of ArraySeriesMatcher. +// When building DLLs for Windows this is required even though no direct access to the MaybeStackArray leaks out of the i18n library. +// (See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +#endif + +namespace numparse { +namespace impl { + +/** + * Base class for AnyMatcher and SeriesMatcher. + */ +// Exported as U_I18N_API for tests +class U_I18N_API CompositionMatcher : public NumberParseMatcher { + protected: + // No construction except by subclasses! + CompositionMatcher() = default; + + // To be overridden by subclasses (used for iteration): + virtual const NumberParseMatcher* const* begin() const = 0; + + // To be overridden by subclasses (used for iteration): + virtual const NumberParseMatcher* const* end() const = 0; +}; + + +// NOTE: AnyMatcher is no longer being used. The previous definition is shown below. +// The implementation can be found in SVN source control, deleted around March 30, 2018. +///** +// * Composes a number of matchers, and succeeds if any of the matchers succeed. Always greedily chooses +// * the first matcher in the list to succeed. +// * +// * NOTE: In C++, this is a base class, unlike ICU4J, which uses a factory-style interface. +// * +// * @author sffc +// * @see SeriesMatcher +// */ +//class AnyMatcher : public CompositionMatcher { +// public: +// bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; +// +// bool smokeTest(const StringSegment& segment) const override; +// +// void postProcess(ParsedNumber& result) const override; +// +// protected: +// // No construction except by subclasses! +// AnyMatcher() = default; +//}; + + +/** + * Composes a number of matchers, running one after another. Matches the input string only if all of the + * matchers in the series succeed. Performs greedy matches within the context of the series. + * + * @author sffc + * @see AnyMatcher + */ +// Exported as U_I18N_API for tests +class U_I18N_API SeriesMatcher : public CompositionMatcher { + public: + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool smokeTest(const StringSegment& segment) const override; + + void postProcess(ParsedNumber& result) const override; + + virtual int32_t length() const = 0; + + protected: + // No construction except by subclasses! + SeriesMatcher() = default; +}; + +/** + * An implementation of SeriesMatcher that references an array of matchers. + * + * The object adopts the array, but NOT the matchers contained inside the array. + */ +// Exported as U_I18N_API for tests +class U_I18N_API ArraySeriesMatcher : public SeriesMatcher { + public: + ArraySeriesMatcher(); // WARNING: Leaves the object in an unusable state + + typedef MaybeStackArray MatcherArray; + + /** The array is std::move'd */ + ArraySeriesMatcher(MatcherArray& matchers, int32_t matchersLen); + + UnicodeString toString() const override; + + int32_t length() const override; + + protected: + const NumberParseMatcher* const* begin() const override; + + const NumberParseMatcher* const* end() const override; + + private: + MatcherArray fMatchers; + int32_t fMatchersLen; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__SOURCE_NUMPARSE_COMPOSITIONS__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_currency.cpp b/intl/icu/source/i18n/numparse_currency.cpp new file mode 100644 index 0000000000..7bbb060f3d --- /dev/null +++ b/intl/icu/source/i18n/numparse_currency.cpp @@ -0,0 +1,189 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_currency.h" +#include "ucurrimp.h" +#include "unicode/errorcode.h" +#include "numparse_utils.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +CombinedCurrencyMatcher::CombinedCurrencyMatcher(const CurrencySymbols& currencySymbols, const DecimalFormatSymbols& dfs, + parse_flags_t parseFlags, UErrorCode& status) + : fCurrency1(currencySymbols.getCurrencySymbol(status)), + fCurrency2(currencySymbols.getIntlCurrencySymbol(status)), + fUseFullCurrencyData(0 == (parseFlags & PARSE_FLAG_NO_FOREIGN_CURRENCY)), + afterPrefixInsert(dfs.getPatternForCurrencySpacing(UNUM_CURRENCY_INSERT, false, status)), + beforeSuffixInsert(dfs.getPatternForCurrencySpacing(UNUM_CURRENCY_INSERT, true, status)), + fLocaleName(dfs.getLocale().getName(), -1, status) { + utils::copyCurrencyCode(fCurrencyCode, currencySymbols.getIsoCode()); + + // Pre-load the long names for the current locale and currency + // if we are parsing without the full currency data. + if (!fUseFullCurrencyData) { + for (int32_t i=0; i(i); + fLocalLongNames[i] = currencySymbols.getPluralName(plural, status); + } + } + + // TODO: Figure out how to make this faster and re-enable. + // Computing the "lead code points" set for fastpathing is too slow to use in production. + // See https://unicode-org.atlassian.net/browse/ICU-13584 +// // Compute the full set of characters that could be the first in a currency to allow for +// // efficient smoke test. +// fLeadCodePoints.add(fCurrency1.char32At(0)); +// fLeadCodePoints.add(fCurrency2.char32At(0)); +// fLeadCodePoints.add(beforeSuffixInsert.char32At(0)); +// uprv_currencyLeads(fLocaleName.data(), fLeadCodePoints, status); +// // Always apply case mapping closure for currencies +// fLeadCodePoints.closeOver(USET_ADD_CASE_MAPPINGS); +// fLeadCodePoints.freeze(); +} + +bool +CombinedCurrencyMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { + if (result.currencyCode[0] != 0) { + return false; + } + + // Try to match a currency spacing separator. + int32_t initialOffset = segment.getOffset(); + bool maybeMore = false; + if (result.seenNumber() && !beforeSuffixInsert.isEmpty()) { + int32_t overlap = segment.getCommonPrefixLength(beforeSuffixInsert); + if (overlap == beforeSuffixInsert.length()) { + segment.adjustOffset(overlap); + // Note: let currency spacing be a weak match. Don't update chars consumed. + } + maybeMore = maybeMore || overlap == segment.length(); + } + + // Match the currency string, and reset if we didn't find one. + maybeMore = maybeMore || matchCurrency(segment, result, status); + if (result.currencyCode[0] == 0) { + segment.setOffset(initialOffset); + return maybeMore; + } + + // Try to match a currency spacing separator. + if (!result.seenNumber() && !afterPrefixInsert.isEmpty()) { + int32_t overlap = segment.getCommonPrefixLength(afterPrefixInsert); + if (overlap == afterPrefixInsert.length()) { + segment.adjustOffset(overlap); + // Note: let currency spacing be a weak match. Don't update chars consumed. + } + maybeMore = maybeMore || overlap == segment.length(); + } + + return maybeMore; +} + +bool CombinedCurrencyMatcher::matchCurrency(StringSegment& segment, ParsedNumber& result, + UErrorCode& status) const { + bool maybeMore = false; + + int32_t overlap1; + if (!fCurrency1.isEmpty()) { + overlap1 = segment.getCaseSensitivePrefixLength(fCurrency1); + } else { + overlap1 = -1; + } + maybeMore = maybeMore || overlap1 == segment.length(); + if (overlap1 == fCurrency1.length()) { + utils::copyCurrencyCode(result.currencyCode, fCurrencyCode); + segment.adjustOffset(overlap1); + result.setCharsConsumed(segment); + return maybeMore; + } + + int32_t overlap2; + if (!fCurrency2.isEmpty()) { + // ISO codes should be accepted case-insensitive. + // https://unicode-org.atlassian.net/browse/ICU-13696 + overlap2 = segment.getCommonPrefixLength(fCurrency2); + } else { + overlap2 = -1; + } + maybeMore = maybeMore || overlap2 == segment.length(); + if (overlap2 == fCurrency2.length()) { + utils::copyCurrencyCode(result.currencyCode, fCurrencyCode); + segment.adjustOffset(overlap2); + result.setCharsConsumed(segment); + return maybeMore; + } + + if (fUseFullCurrencyData) { + // Use the full currency data. + // NOTE: This call site should be improved with #13584. + const UnicodeString segmentString = segment.toTempUnicodeString(); + + // Try to parse the currency + ParsePosition ppos(0); + int32_t partialMatchLen = 0; + uprv_parseCurrency( + fLocaleName.data(), + segmentString, + ppos, + UCURR_SYMBOL_NAME, // checks for both UCURR_SYMBOL_NAME and UCURR_LONG_NAME + &partialMatchLen, + result.currencyCode, + status); + maybeMore = maybeMore || partialMatchLen == segment.length(); + + if (U_SUCCESS(status) && ppos.getIndex() != 0) { + // Complete match. + // NOTE: The currency code should already be saved in the ParsedNumber. + segment.adjustOffset(ppos.getIndex()); + result.setCharsConsumed(segment); + return maybeMore; + } + + } else { + // Use the locale long names. + int32_t longestFullMatch = 0; + for (int32_t i=0; i longestFullMatch) { + longestFullMatch = name.length(); + } + maybeMore = maybeMore || overlap > 0; + } + if (longestFullMatch > 0) { + utils::copyCurrencyCode(result.currencyCode, fCurrencyCode); + segment.adjustOffset(longestFullMatch); + result.setCharsConsumed(segment); + return maybeMore; + } + } + + // No match found. + return maybeMore; +} + +bool CombinedCurrencyMatcher::smokeTest(const StringSegment&) const { + // TODO: See constructor + return true; + //return segment.startsWith(fLeadCodePoints); +} + +UnicodeString CombinedCurrencyMatcher::toString() const { + return u""; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_currency.h b/intl/icu/source/i18n/numparse_currency.h new file mode 100644 index 0000000000..4e99334a31 --- /dev/null +++ b/intl/icu/source/i18n/numparse_currency.h @@ -0,0 +1,74 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_CURRENCY_H__ +#define __NUMPARSE_CURRENCY_H__ + +#include "numparse_types.h" +#include "numparse_compositions.h" +#include "charstr.h" +#include "number_currencysymbols.h" +#include "unicode/uniset.h" + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { + +using ::icu::number::impl::CurrencySymbols; + +/** + * Matches a currency, either a custom currency or one from the data bundle. The class is called + * "combined" to emphasize that the currency string may come from one of multiple sources. + * + * Will match currency spacing either before or after the number depending on whether we are currently in + * the prefix or suffix. + * + * The implementation of this class is slightly different between J and C. See #13584 for a follow-up. + * + * @author sffc + */ +// Exported as U_I18N_API for tests +class U_I18N_API CombinedCurrencyMatcher : public NumberParseMatcher, public UMemory { + public: + CombinedCurrencyMatcher() = default; // WARNING: Leaves the object in an unusable state + + CombinedCurrencyMatcher(const CurrencySymbols& currencySymbols, const DecimalFormatSymbols& dfs, + parse_flags_t parseFlags, UErrorCode& status); + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool smokeTest(const StringSegment& segment) const override; + + UnicodeString toString() const override; + + private: + char16_t fCurrencyCode[4]; + UnicodeString fCurrency1; + UnicodeString fCurrency2; + + bool fUseFullCurrencyData; + UnicodeString fLocalLongNames[StandardPlural::COUNT]; + + UnicodeString afterPrefixInsert; + UnicodeString beforeSuffixInsert; + + // We could use Locale instead of CharString here, but + // Locale has a non-trivial default constructor. + CharString fLocaleName; + + // TODO: See comments in constructor in numparse_currency.cpp + // UnicodeSet fLeadCodePoints; + + /** Matches the currency string without concern for currency spacing. */ + bool matchCurrency(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_CURRENCY_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_decimal.cpp b/intl/icu/source/i18n/numparse_decimal.cpp new file mode 100644 index 0000000000..8b99fd7ad4 --- /dev/null +++ b/intl/icu/source/i18n/numparse_decimal.cpp @@ -0,0 +1,459 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_decimal.h" +#include "static_unicode_sets.h" +#include "numparse_utils.h" +#include "unicode/uchar.h" +#include "putilimp.h" +#include "number_decimalquantity.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +DecimalMatcher::DecimalMatcher(const DecimalFormatSymbols& symbols, const Grouper& grouper, + parse_flags_t parseFlags) { + if (0 != (parseFlags & PARSE_FLAG_MONETARY_SEPARATORS)) { + groupingSeparator = symbols.getConstSymbol(DecimalFormatSymbols::kMonetaryGroupingSeparatorSymbol); + decimalSeparator = symbols.getConstSymbol(DecimalFormatSymbols::kMonetarySeparatorSymbol); + } else { + groupingSeparator = symbols.getConstSymbol(DecimalFormatSymbols::kGroupingSeparatorSymbol); + decimalSeparator = symbols.getConstSymbol(DecimalFormatSymbols::kDecimalSeparatorSymbol); + } + bool strictSeparators = 0 != (parseFlags & PARSE_FLAG_STRICT_SEPARATORS); + unisets::Key groupingKey = strictSeparators ? unisets::STRICT_ALL_SEPARATORS + : unisets::ALL_SEPARATORS; + + // Attempt to find separators in the static cache + + groupingUniSet = unisets::get(groupingKey); + unisets::Key decimalKey = unisets::chooseFrom( + decimalSeparator, + strictSeparators ? unisets::STRICT_COMMA : unisets::COMMA, + strictSeparators ? unisets::STRICT_PERIOD : unisets::PERIOD); + if (decimalKey >= 0) { + decimalUniSet = unisets::get(decimalKey); + } else if (!decimalSeparator.isEmpty()) { + auto* set = new UnicodeSet(); + set->add(decimalSeparator.char32At(0)); + set->freeze(); + decimalUniSet = set; + fLocalDecimalUniSet.adoptInstead(set); + } else { + decimalUniSet = unisets::get(unisets::EMPTY); + } + + if (groupingKey >= 0 && decimalKey >= 0) { + // Everything is available in the static cache + separatorSet = groupingUniSet; + leadSet = unisets::get( + strictSeparators ? unisets::DIGITS_OR_ALL_SEPARATORS + : unisets::DIGITS_OR_STRICT_ALL_SEPARATORS); + } else { + auto* set = new UnicodeSet(); + set->addAll(*groupingUniSet); + set->addAll(*decimalUniSet); + set->freeze(); + separatorSet = set; + fLocalSeparatorSet.adoptInstead(set); + leadSet = nullptr; + } + + UChar32 cpZero = symbols.getCodePointZero(); + if (cpZero == -1 || !u_isdigit(cpZero) || u_digit(cpZero, 10) != 0) { + // Uncommon case: okay to allocate. + auto digitStrings = new UnicodeString[10]; + fLocalDigitStrings.adoptInstead(digitStrings); + for (int32_t i = 0; i <= 9; i++) { + digitStrings[i] = symbols.getConstDigitSymbol(i); + } + } + + requireGroupingMatch = 0 != (parseFlags & PARSE_FLAG_STRICT_GROUPING_SIZE); + groupingDisabled = 0 != (parseFlags & PARSE_FLAG_GROUPING_DISABLED); + integerOnly = 0 != (parseFlags & PARSE_FLAG_INTEGER_ONLY); + grouping1 = grouper.getPrimary(); + grouping2 = grouper.getSecondary(); + + // Fraction grouping parsing is disabled for now but could be enabled later. + // See https://unicode-org.atlassian.net/browse/ICU-10794 + // fractionGrouping = 0 != (parseFlags & PARSE_FLAG_FRACTION_GROUPING_ENABLED); +} + +bool DecimalMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { + return match(segment, result, 0, status); +} + +bool DecimalMatcher::match(StringSegment& segment, ParsedNumber& result, int8_t exponentSign, + UErrorCode&) const { + if (result.seenNumber() && exponentSign == 0) { + // A number has already been consumed. + return false; + } else if (exponentSign != 0) { + // scientific notation always comes after the number + U_ASSERT(!result.quantity.bogus); + } + + // Initial offset before any character consumption. + int32_t initialOffset = segment.getOffset(); + + // Return value: whether to ask for more characters. + bool maybeMore = false; + + // All digits consumed so far. + number::impl::DecimalQuantity digitsConsumed; + digitsConsumed.bogus = true; + + // The total number of digits after the decimal place, used for scaling the result. + int32_t digitsAfterDecimalPlace = 0; + + // The actual grouping and decimal separators used in the string. + // If non-null, we have seen that token. + UnicodeString actualGroupingString; + UnicodeString actualDecimalString; + actualGroupingString.setToBogus(); + actualDecimalString.setToBogus(); + + // Information for two groups: the previous group and the current group. + // + // Each group has three pieces of information: + // + // Offset: the string position of the beginning of the group, including a leading separator + // if there was a leading separator. This is needed in case we need to rewind the parse to + // that position. + // + // Separator type: + // 0 => beginning of string + // 1 => lead separator is a grouping separator + // 2 => lead separator is a decimal separator + // + // Count: the number of digits in the group. If -1, the group has been validated. + int32_t currGroupOffset = 0; + int32_t currGroupSepType = 0; + int32_t currGroupCount = 0; + int32_t prevGroupOffset = -1; + int32_t prevGroupSepType = -1; + int32_t prevGroupCount = -1; + + while (segment.length() > 0) { + maybeMore = false; + + // Attempt to match a digit. + int8_t digit = -1; + + // Try by code point digit value. + UChar32 cp = segment.getCodePoint(); + if (u_isdigit(cp)) { + segment.adjustOffset(U16_LENGTH(cp)); + digit = static_cast(u_digit(cp, 10)); + } + + // Try by digit string. + if (digit == -1 && !fLocalDigitStrings.isNull()) { + for (int32_t i = 0; i < 10; i++) { + const UnicodeString& str = fLocalDigitStrings[i]; + if (str.isEmpty()) { + continue; + } + int32_t overlap = segment.getCommonPrefixLength(str); + if (overlap == str.length()) { + segment.adjustOffset(overlap); + digit = static_cast(i); + break; + } + maybeMore = maybeMore || (overlap == segment.length()); + } + } + + if (digit >= 0) { + // Digit was found. + if (digitsConsumed.bogus) { + digitsConsumed.bogus = false; + digitsConsumed.clear(); + } + digitsConsumed.appendDigit(digit, 0, true); + currGroupCount++; + if (!actualDecimalString.isBogus()) { + digitsAfterDecimalPlace++; + } + continue; + } + + // Attempt to match a literal grouping or decimal separator. + bool isDecimal = false; + bool isGrouping = false; + + // 1) Attempt the decimal separator string literal. + // if (we have not seen a decimal separator yet) { ... } + if (actualDecimalString.isBogus() && !decimalSeparator.isEmpty()) { + int32_t overlap = segment.getCommonPrefixLength(decimalSeparator); + maybeMore = maybeMore || (overlap == segment.length()); + if (overlap == decimalSeparator.length()) { + isDecimal = true; + actualDecimalString = decimalSeparator; + } + } + + // 2) Attempt to match the actual grouping string literal. + if (!actualGroupingString.isBogus()) { + int32_t overlap = segment.getCommonPrefixLength(actualGroupingString); + maybeMore = maybeMore || (overlap == segment.length()); + if (overlap == actualGroupingString.length()) { + isGrouping = true; + } + } + + // 2.5) Attempt to match a new the grouping separator string literal. + // if (we have not seen a grouping or decimal separator yet) { ... } + if (!groupingDisabled && actualGroupingString.isBogus() && actualDecimalString.isBogus() && + !groupingSeparator.isEmpty()) { + int32_t overlap = segment.getCommonPrefixLength(groupingSeparator); + maybeMore = maybeMore || (overlap == segment.length()); + if (overlap == groupingSeparator.length()) { + isGrouping = true; + actualGroupingString = groupingSeparator; + } + } + + // 3) Attempt to match a decimal separator from the equivalence set. + // if (we have not seen a decimal separator yet) { ... } + // The !isGrouping is to confirm that we haven't yet matched the current character. + if (!isGrouping && actualDecimalString.isBogus()) { + if (decimalUniSet->contains(cp)) { + isDecimal = true; + actualDecimalString = UnicodeString(cp); + } + } + + // 4) Attempt to match a grouping separator from the equivalence set. + // if (we have not seen a grouping or decimal separator yet) { ... } + if (!groupingDisabled && actualGroupingString.isBogus() && actualDecimalString.isBogus()) { + if (groupingUniSet->contains(cp)) { + isGrouping = true; + actualGroupingString = UnicodeString(cp); + } + } + + // Leave if we failed to match this as a separator. + if (!isDecimal && !isGrouping) { + break; + } + + // Check for conditions when we don't want to accept the separator. + if (isDecimal && integerOnly) { + break; + } else if (currGroupSepType == 2 && isGrouping) { + // Fraction grouping + break; + } + + // Validate intermediate grouping sizes. + bool prevValidSecondary = validateGroup(prevGroupSepType, prevGroupCount, false); + bool currValidPrimary = validateGroup(currGroupSepType, currGroupCount, true); + if (!prevValidSecondary || (isDecimal && !currValidPrimary)) { + // Invalid grouping sizes. + if (isGrouping && currGroupCount == 0) { + // Trailing grouping separators: these are taken care of below + U_ASSERT(currGroupSepType == 1); + } else if (requireGroupingMatch) { + // Strict mode: reject the parse + digitsConsumed.clear(); + digitsConsumed.bogus = true; + } + break; + } else if (requireGroupingMatch && currGroupCount == 0 && currGroupSepType == 1) { + break; + } else { + // Grouping sizes OK so far. + prevGroupOffset = currGroupOffset; + prevGroupCount = currGroupCount; + if (isDecimal) { + // Do not validate this group any more. + prevGroupSepType = -1; + } else { + prevGroupSepType = currGroupSepType; + } + } + + // OK to accept the separator. + // Special case: don't update currGroup if it is empty; this allows two grouping + // separators in a row in lenient mode. + if (currGroupCount != 0) { + currGroupOffset = segment.getOffset(); + } + currGroupSepType = isGrouping ? 1 : 2; + currGroupCount = 0; + if (isGrouping) { + segment.adjustOffset(actualGroupingString.length()); + } else { + segment.adjustOffset(actualDecimalString.length()); + } + } + + // End of main loop. + // Back up if there was a trailing grouping separator. + // Shift prev -> curr so we can check it as a final group. + if (currGroupSepType != 2 && currGroupCount == 0) { + maybeMore = true; + segment.setOffset(currGroupOffset); + currGroupOffset = prevGroupOffset; + currGroupSepType = prevGroupSepType; + currGroupCount = prevGroupCount; + prevGroupOffset = -1; + prevGroupSepType = 0; + prevGroupCount = 1; + } + + // Validate final grouping sizes. + bool prevValidSecondary = validateGroup(prevGroupSepType, prevGroupCount, false); + bool currValidPrimary = validateGroup(currGroupSepType, currGroupCount, true); + if (!requireGroupingMatch) { + // The cases we need to handle here are lone digits. + // Examples: "1,1" "1,1," "1,1,1" "1,1,1," ",1" (all parse as 1) + // See more examples in numberformattestspecification.txt + int32_t digitsToRemove = 0; + if (!prevValidSecondary) { + segment.setOffset(prevGroupOffset); + digitsToRemove += prevGroupCount; + digitsToRemove += currGroupCount; + } else if (!currValidPrimary && (prevGroupSepType != 0 || prevGroupCount != 0)) { + maybeMore = true; + segment.setOffset(currGroupOffset); + digitsToRemove += currGroupCount; + } + if (digitsToRemove != 0) { + digitsConsumed.adjustMagnitude(-digitsToRemove); + digitsConsumed.truncate(); + } + prevValidSecondary = true; + currValidPrimary = true; + } + if (currGroupSepType != 2 && (!prevValidSecondary || !currValidPrimary)) { + // Grouping failure. + digitsConsumed.bogus = true; + } + + // Strings that start with a separator but have no digits, + // or strings that failed a grouping size check. + if (digitsConsumed.bogus) { + maybeMore = maybeMore || (segment.length() == 0); + segment.setOffset(initialOffset); + return maybeMore; + } + + // We passed all inspections. Start post-processing. + + // Adjust for fraction part. + digitsConsumed.adjustMagnitude(-digitsAfterDecimalPlace); + + // Set the digits, either normal or exponent. + if (exponentSign != 0 && segment.getOffset() != initialOffset) { + bool overflow = false; + if (digitsConsumed.fitsInLong()) { + int64_t exponentLong = digitsConsumed.toLong(false); + U_ASSERT(exponentLong >= 0); + if (exponentLong <= INT32_MAX) { + auto exponentInt = static_cast(exponentLong); + if (result.quantity.adjustMagnitude(exponentSign * exponentInt)) { + overflow = true; + } + } else { + overflow = true; + } + } else { + overflow = true; + } + if (overflow) { + if (exponentSign == -1) { + // Set to zero + result.quantity.clear(); + } else { + // Set to infinity + result.quantity.bogus = true; + result.flags |= FLAG_INFINITY; + } + } + } else { + result.quantity = digitsConsumed; + } + + // Set other information into the result and return. + if (!actualDecimalString.isBogus()) { + result.flags |= FLAG_HAS_DECIMAL_SEPARATOR; + } + result.setCharsConsumed(segment); + return segment.length() == 0 || maybeMore; +} + +bool DecimalMatcher::validateGroup(int32_t sepType, int32_t count, bool isPrimary) const { + if (requireGroupingMatch) { + if (sepType == -1) { + // No such group (prevGroup before first shift). + return true; + } else if (sepType == 0) { + // First group. + if (isPrimary) { + // No grouping separators is OK. + return true; + } else { + return count != 0 && count <= grouping2; + } + } else if (sepType == 1) { + // Middle group. + if (isPrimary) { + return count == grouping1; + } else { + return count == grouping2; + } + } else { + U_ASSERT(sepType == 2); + // After the decimal separator. + return true; + } + } else { + if (sepType == 1) { + // #11230: don't accept middle groups with only 1 digit. + return count != 1; + } else { + return true; + } + } +} + +bool DecimalMatcher::smokeTest(const StringSegment& segment) const { + // The common case uses a static leadSet for efficiency. + if (fLocalDigitStrings.isNull() && leadSet != nullptr) { + return segment.startsWith(*leadSet); + } + if (segment.startsWith(*separatorSet) || u_isdigit(segment.getCodePoint())) { + return true; + } + if (fLocalDigitStrings.isNull()) { + return false; + } + for (int32_t i = 0; i < 10; i++) { + if (segment.startsWith(fLocalDigitStrings[i])) { + return true; + } + } + return false; +} + +UnicodeString DecimalMatcher::toString() const { + return u""; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_decimal.h b/intl/icu/source/i18n/numparse_decimal.h new file mode 100644 index 0000000000..07c9afeccc --- /dev/null +++ b/intl/icu/source/i18n/numparse_decimal.h @@ -0,0 +1,76 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_DECIMAL_H__ +#define __NUMPARSE_DECIMAL_H__ + +#include "unicode/uniset.h" +#include "numparse_types.h" + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { + +using ::icu::number::impl::Grouper; + +class DecimalMatcher : public NumberParseMatcher, public UMemory { + public: + DecimalMatcher() = default; // WARNING: Leaves the object in an unusable state + + DecimalMatcher(const DecimalFormatSymbols& symbols, const Grouper& grouper, + parse_flags_t parseFlags); + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool + match(StringSegment& segment, ParsedNumber& result, int8_t exponentSign, UErrorCode& status) const; + + bool smokeTest(const StringSegment& segment) const override; + + UnicodeString toString() const override; + + private: + /** If true, only accept strings whose grouping sizes match the locale */ + bool requireGroupingMatch; + + /** If true, do not accept grouping separators at all */ + bool groupingDisabled; + + // Fraction grouping parsing is disabled for now but could be enabled later. + // See https://unicode-org.atlassian.net/browse/ICU-10794 + // bool fractionGrouping; + + /** If true, do not accept numbers in the fraction */ + bool integerOnly; + + int16_t grouping1; + int16_t grouping2; + + UnicodeString groupingSeparator; + UnicodeString decimalSeparator; + + // Assumption: these sets all consist of single code points. If this assumption needs to be broken, + // fix getLeadCodePoints() as well as matching logic. Be careful of the performance impact. + const UnicodeSet* groupingUniSet; + const UnicodeSet* decimalUniSet; + const UnicodeSet* separatorSet; + const UnicodeSet* leadSet; + + // Make this class the owner of a few objects that could be allocated. + // The first three LocalPointers are used for assigning ownership only. + LocalPointer fLocalDecimalUniSet; + LocalPointer fLocalSeparatorSet; + LocalArray fLocalDigitStrings; + + bool validateGroup(int32_t sepType, int32_t count, bool isPrimary) const; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_DECIMAL_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_impl.cpp b/intl/icu/source/i18n/numparse_impl.cpp new file mode 100644 index 0000000000..91c60747f2 --- /dev/null +++ b/intl/icu/source/i18n/numparse_impl.cpp @@ -0,0 +1,365 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include +#include +#include "number_types.h" +#include "number_patternstring.h" +#include "numparse_types.h" +#include "numparse_impl.h" +#include "numparse_symbols.h" +#include "numparse_decimal.h" +#include "unicode/numberformatter.h" +#include "cstr.h" +#include "number_mapper.h" +#include "static_unicode_sets.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +NumberParseMatcher::~NumberParseMatcher() = default; + + +NumberParserImpl* +NumberParserImpl::createSimpleParser(const Locale& locale, const UnicodeString& patternString, + parse_flags_t parseFlags, UErrorCode& status) { + + LocalPointer parser(new NumberParserImpl(parseFlags)); + DecimalFormatSymbols symbols(locale, status); + + parser->fLocalMatchers.ignorables = {parseFlags}; + IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; + + DecimalFormatSymbols dfs(locale, status); + dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$"); + dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU"); + CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status); + + ParsedPatternInfo patternInfo; + PatternParser::parseToPatternInfo(patternString, patternInfo, status); + + // The following statements set up the affix matchers. + AffixTokenMatcherSetupData affixSetupData = { + currencySymbols, symbols, ignorables, locale, parseFlags}; + parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; + parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; + parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( + patternInfo, *parser, ignorables, parseFlags, status); + + Grouper grouper = Grouper::forStrategy(UNUM_GROUPING_AUTO); + grouper.setLocaleData(patternInfo, locale); + + parser->addMatcher(parser->fLocalMatchers.ignorables); + parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); + parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); + parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); + parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); + parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); + parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); + parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); + parser->addMatcher(parser->fLocalMatchers.padding = {u"@"}); + parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); + parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); + parser->addMatcher(parser->fLocalValidators.number = {}); + + parser->freeze(); + return parser.orphan(); +} + +NumberParserImpl* +NumberParserImpl::createParserFromProperties(const number::impl::DecimalFormatProperties& properties, + const DecimalFormatSymbols& symbols, bool parseCurrency, + UErrorCode& status) { + Locale locale = symbols.getLocale(); + AutoAffixPatternProvider affixProvider(properties, status); + if (U_FAILURE(status)) { return nullptr; } + CurrencyUnit currency = resolveCurrency(properties, locale, status); + CurrencySymbols currencySymbols(currency, locale, symbols, status); + bool isStrict = properties.parseMode.getOrDefault(PARSE_MODE_STRICT) == PARSE_MODE_STRICT; + Grouper grouper = Grouper::forProperties(properties); + int parseFlags = 0; + if (U_FAILURE(status)) { return nullptr; } + if (!properties.parseCaseSensitive) { + parseFlags |= PARSE_FLAG_IGNORE_CASE; + } + if (properties.parseIntegerOnly) { + parseFlags |= PARSE_FLAG_INTEGER_ONLY; + } + if (properties.signAlwaysShown) { + parseFlags |= PARSE_FLAG_PLUS_SIGN_ALLOWED; + } + if (isStrict) { + parseFlags |= PARSE_FLAG_STRICT_GROUPING_SIZE; + parseFlags |= PARSE_FLAG_STRICT_SEPARATORS; + parseFlags |= PARSE_FLAG_USE_FULL_AFFIXES; + parseFlags |= PARSE_FLAG_EXACT_AFFIX; + parseFlags |= PARSE_FLAG_STRICT_IGNORABLES; + } else { + parseFlags |= PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES; + } + if (grouper.getPrimary() <= 0) { + parseFlags |= PARSE_FLAG_GROUPING_DISABLED; + } + if (parseCurrency || affixProvider.get().hasCurrencySign()) { + parseFlags |= PARSE_FLAG_MONETARY_SEPARATORS; + } + if (!parseCurrency) { + parseFlags |= PARSE_FLAG_NO_FOREIGN_CURRENCY; + } + + LocalPointer parser(new NumberParserImpl(parseFlags)); + + parser->fLocalMatchers.ignorables = {parseFlags}; + IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; + + ////////////////////// + /// AFFIX MATCHERS /// + ////////////////////// + + // The following statements set up the affix matchers. + AffixTokenMatcherSetupData affixSetupData = { + currencySymbols, symbols, ignorables, locale, parseFlags}; + parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; + parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; + parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( + affixProvider.get(), *parser, ignorables, parseFlags, status); + + //////////////////////// + /// CURRENCY MATCHER /// + //////////////////////// + + if (parseCurrency || affixProvider.get().hasCurrencySign()) { + parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); + } + + /////////////// + /// PERCENT /// + /////////////// + + // ICU-TC meeting, April 11, 2018: accept percent/permille only if it is in the pattern, + // and to maintain regressive behavior, divide by 100 even if no percent sign is present. + if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERCENT, status)) { + parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); + } + if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERMILLE, status)) { + parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); + } + + /////////////////////////////// + /// OTHER STANDARD MATCHERS /// + /////////////////////////////// + + if (!isStrict) { + parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); + parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); + } + parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); + parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); + UnicodeString padString = properties.padString; + if (!padString.isBogus() && !ignorables.getSet()->contains(padString)) { + parser->addMatcher(parser->fLocalMatchers.padding = {padString}); + } + parser->addMatcher(parser->fLocalMatchers.ignorables); + parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); + // NOTE: parseNoExponent doesn't disable scientific parsing if we have a scientific formatter + if (!properties.parseNoExponent || properties.minimumExponentDigits > 0) { + parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); + } + + ////////////////// + /// VALIDATORS /// + ////////////////// + + parser->addMatcher(parser->fLocalValidators.number = {}); + if (isStrict) { + parser->addMatcher(parser->fLocalValidators.affix = {}); + } + if (parseCurrency) { + parser->addMatcher(parser->fLocalValidators.currency = {}); + } + if (properties.decimalPatternMatchRequired) { + bool patternHasDecimalSeparator = + properties.decimalSeparatorAlwaysShown || properties.maximumFractionDigits != 0; + parser->addMatcher(parser->fLocalValidators.decimalSeparator = {patternHasDecimalSeparator}); + } + // The multiplier takes care of scaling percentages. + Scale multiplier = scaleFromProperties(properties); + if (multiplier.isValid()) { + parser->addMatcher(parser->fLocalValidators.multiplier = {multiplier}); + } + + parser->freeze(); + return parser.orphan(); +} + +NumberParserImpl::NumberParserImpl(parse_flags_t parseFlags) + : fParseFlags(parseFlags) { +} + +NumberParserImpl::~NumberParserImpl() { + fNumMatchers = 0; +} + +void NumberParserImpl::addMatcher(NumberParseMatcher& matcher) { + if (fNumMatchers + 1 > fMatchers.getCapacity()) { + fMatchers.resize(fNumMatchers * 2, fNumMatchers); + } + fMatchers[fNumMatchers] = &matcher; + fNumMatchers++; +} + +void NumberParserImpl::freeze() { + fFrozen = true; +} + +parse_flags_t NumberParserImpl::getParseFlags() const { + return fParseFlags; +} + +void NumberParserImpl::parse(const UnicodeString& input, bool greedy, ParsedNumber& result, + UErrorCode& status) const { + return parse(input, 0, greedy, result, status); +} + +void NumberParserImpl::parse(const UnicodeString& input, int32_t start, bool greedy, ParsedNumber& result, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + U_ASSERT(fFrozen); + // TODO: Check start >= 0 and start < input.length() + StringSegment segment(input, 0 != (fParseFlags & PARSE_FLAG_IGNORE_CASE)); + segment.adjustOffset(start); + if (greedy) { + parseGreedy(segment, result, status); + } else if (0 != (fParseFlags & PARSE_FLAG_ALLOW_INFINITE_RECURSION)) { + // Start at 1 so that recursionLevels never gets to 0 + parseLongestRecursive(segment, result, 1, status); + } else { + // Arbitrary recursion safety limit: 100 levels. + parseLongestRecursive(segment, result, -100, status); + } + for (int32_t i = 0; i < fNumMatchers; i++) { + fMatchers[i]->postProcess(result); + } + result.postProcess(); +} + +void NumberParserImpl::parseGreedy(StringSegment& segment, ParsedNumber& result, + UErrorCode& status) const { + // Note: this method is not recursive in order to avoid stack overflow. + for (int i = 0; i smokeTest(segment)) { + // Matcher failed smoke test: try the next one + i++; + continue; + } + int32_t initialOffset = segment.getOffset(); + matcher->match(segment, result, status); + if (U_FAILURE(status)) { + return; + } + if (segment.getOffset() != initialOffset) { + // Greedy heuristic: accept the match and loop back + i = 0; + continue; + } else { + // Matcher did not match: try the next one + i++; + continue; + } + UPRV_UNREACHABLE_EXIT; + } + + // NOTE: If we get here, the greedy parse completed without consuming the entire string. +} + +void NumberParserImpl::parseLongestRecursive(StringSegment& segment, ParsedNumber& result, + int32_t recursionLevels, + UErrorCode& status) const { + // Base Case + if (segment.length() == 0) { + return; + } + + // Safety against stack overflow + if (recursionLevels == 0) { + return; + } + + // TODO: Give a nice way for the matcher to reset the ParsedNumber? + ParsedNumber initial(result); + ParsedNumber candidate; + + int initialOffset = segment.getOffset(); + for (int32_t i = 0; i < fNumMatchers; i++) { + const NumberParseMatcher* matcher = fMatchers[i]; + if (!matcher->smokeTest(segment)) { + continue; + } + + // In a non-greedy parse, we attempt all possible matches and pick the best. + for (int32_t charsToConsume = 0; charsToConsume < segment.length();) { + charsToConsume += U16_LENGTH(segment.codePointAt(charsToConsume)); + + // Run the matcher on a segment of the current length. + candidate = initial; + segment.setLength(charsToConsume); + bool maybeMore = matcher->match(segment, candidate, status); + segment.resetLength(); + if (U_FAILURE(status)) { + return; + } + + // If the entire segment was consumed, recurse. + if (segment.getOffset() - initialOffset == charsToConsume) { + parseLongestRecursive(segment, candidate, recursionLevels + 1, status); + if (U_FAILURE(status)) { + return; + } + if (candidate.isBetterThan(result)) { + result = candidate; + } + } + + // Since the segment can be re-used, reset the offset. + // This does not have an effect if the matcher did not consume any chars. + segment.setOffset(initialOffset); + + // Unless the matcher wants to see the next char, continue to the next matcher. + if (!maybeMore) { + break; + } + } + } +} + +UnicodeString NumberParserImpl::toString() const { + UnicodeString result(u"toString()); + } + result.append(u" ]>", -1); + return result; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_impl.h b/intl/icu/source/i18n/numparse_impl.h new file mode 100644 index 0000000000..380d9aa4c6 --- /dev/null +++ b/intl/icu/source/i18n/numparse_impl.h @@ -0,0 +1,111 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_IMPL_H__ +#define __NUMPARSE_IMPL_H__ + +#include "numparse_types.h" +#include "numparse_decimal.h" +#include "numparse_symbols.h" +#include "numparse_scientific.h" +#include "unicode/uniset.h" +#include "numparse_currency.h" +#include "numparse_affixes.h" +#include "number_decimfmtprops.h" +#include "unicode/localpointer.h" +#include "numparse_validators.h" +#include "number_multiplier.h" +#include "string_segment.h" + +U_NAMESPACE_BEGIN + +// Export an explicit template instantiation of the MaybeStackArray that is used as a data member of NumberParserImpl. +// When building DLLs for Windows this is required even though no direct access to the MaybeStackArray leaks out of the i18n library. +// (See numparse_compositions.h, numparse_affixes.h, datefmt.h, and others for similar examples.) +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +template class U_I18N_API MaybeStackArray; +#endif + +namespace numparse { +namespace impl { + +// Exported as U_I18N_API for tests +class U_I18N_API NumberParserImpl : public MutableMatcherCollection, public UMemory { + public: + virtual ~NumberParserImpl(); + + static NumberParserImpl* createSimpleParser(const Locale& locale, const UnicodeString& patternString, + parse_flags_t parseFlags, UErrorCode& status); + + static NumberParserImpl* createParserFromProperties( + const number::impl::DecimalFormatProperties& properties, const DecimalFormatSymbols& symbols, + bool parseCurrency, UErrorCode& status); + + /** + * Does NOT take ownership of the matcher. The matcher MUST remain valid for the lifespan of the + * NumberParserImpl. + * @param matcher The matcher to reference. + */ + void addMatcher(NumberParseMatcher& matcher) override; + + void freeze(); + + parse_flags_t getParseFlags() const; + + void parse(const UnicodeString& input, bool greedy, ParsedNumber& result, UErrorCode& status) const; + + void parse(const UnicodeString& input, int32_t start, bool greedy, ParsedNumber& result, + UErrorCode& status) const; + + UnicodeString toString() const; + + private: + parse_flags_t fParseFlags; + int32_t fNumMatchers = 0; + // NOTE: The stack capacity for fMatchers and fLeads should be the same + MaybeStackArray fMatchers; + bool fFrozen = false; + + // WARNING: All of these matchers start in an undefined state (default-constructed). + // You must use an assignment operator on them before using. + struct { + IgnorablesMatcher ignorables; + InfinityMatcher infinity; + MinusSignMatcher minusSign; + NanMatcher nan; + PaddingMatcher padding; + PercentMatcher percent; + PermilleMatcher permille; + PlusSignMatcher plusSign; + DecimalMatcher decimal; + ScientificMatcher scientific; + CombinedCurrencyMatcher currency; + AffixMatcherWarehouse affixMatcherWarehouse; + AffixTokenMatcherWarehouse affixTokenMatcherWarehouse; + } fLocalMatchers; + struct { + RequireAffixValidator affix; + RequireCurrencyValidator currency; + RequireDecimalSeparatorValidator decimalSeparator; + RequireNumberValidator number; + MultiplierParseHandler multiplier; + } fLocalValidators; + + explicit NumberParserImpl(parse_flags_t parseFlags); + + void parseGreedy(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const; + + void parseLongestRecursive( + StringSegment& segment, ParsedNumber& result, int32_t recursionLevels, UErrorCode& status) const; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_IMPL_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_parsednumber.cpp b/intl/icu/source/i18n/numparse_parsednumber.cpp new file mode 100644 index 0000000000..4b373a3c31 --- /dev/null +++ b/intl/icu/source/i18n/numparse_parsednumber.cpp @@ -0,0 +1,126 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "number_decimalquantity.h" +#include "string_segment.h" +#include "putilimp.h" +#include + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +ParsedNumber::ParsedNumber() { + clear(); +} + +void ParsedNumber::clear() { + quantity.bogus = true; + charEnd = 0; + flags = 0; + prefix.setToBogus(); + suffix.setToBogus(); + currencyCode[0] = 0; +} + +void ParsedNumber::setCharsConsumed(const StringSegment& segment) { + charEnd = segment.getOffset(); +} + +void ParsedNumber::postProcess() { + if (!quantity.bogus && 0 != (flags & FLAG_NEGATIVE)) { + quantity.negate(); + } +} + +bool ParsedNumber::success() const { + return charEnd > 0 && 0 == (flags & FLAG_FAIL); +} + +bool ParsedNumber::seenNumber() const { + return !quantity.bogus || 0 != (flags & FLAG_NAN) || 0 != (flags & FLAG_INFINITY); +} + +double ParsedNumber::getDouble(UErrorCode& status) const { + bool sawNaN = 0 != (flags & FLAG_NAN); + bool sawInfinity = 0 != (flags & FLAG_INFINITY); + + // Check for NaN, infinity, and -0.0 + if (sawNaN) { + // Can't use NAN or std::nan because the byte pattern is platform-dependent; + // MSVC sets the sign bit, but Clang and GCC do not + return uprv_getNaN(); + } + if (sawInfinity) { + if (0 != (flags & FLAG_NEGATIVE)) { + return -INFINITY; + } else { + return INFINITY; + } + } + if (quantity.bogus) { + status = U_INVALID_STATE_ERROR; + return 0.0; + } + if (quantity.isZeroish() && quantity.isNegative()) { + return -0.0; + } + + if (quantity.fitsInLong()) { + return static_cast(quantity.toLong()); + } else { + return quantity.toDouble(); + } +} + +void ParsedNumber::populateFormattable(Formattable& output, parse_flags_t parseFlags) const { + bool sawNaN = 0 != (flags & FLAG_NAN); + bool sawInfinity = 0 != (flags & FLAG_INFINITY); + bool integerOnly = 0 != (parseFlags & PARSE_FLAG_INTEGER_ONLY); + + // Check for NaN, infinity, and -0.0 + if (sawNaN) { + // Can't use NAN or std::nan because the byte pattern is platform-dependent; + // MSVC sets the sign bit, but Clang and GCC do not + output.setDouble(uprv_getNaN()); + return; + } + if (sawInfinity) { + if (0 != (flags & FLAG_NEGATIVE)) { + output.setDouble(-INFINITY); + return; + } else { + output.setDouble(INFINITY); + return; + } + } + U_ASSERT(!quantity.bogus); + if (quantity.isZeroish() && quantity.isNegative() && !integerOnly) { + output.setDouble(-0.0); + return; + } + + // All other numbers + output.adoptDecimalQuantity(new DecimalQuantity(quantity)); +} + +bool ParsedNumber::isBetterThan(const ParsedNumber& other) { + // Favor results with strictly more characters consumed. + return charEnd > other.charEnd; +} + + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_scientific.cpp b/intl/icu/source/i18n/numparse_scientific.cpp new file mode 100644 index 0000000000..4b88cd998f --- /dev/null +++ b/intl/icu/source/i18n/numparse_scientific.cpp @@ -0,0 +1,163 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_scientific.h" +#include "static_unicode_sets.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +namespace { + +inline const UnicodeSet& minusSignSet() { + return *unisets::get(unisets::MINUS_SIGN); +} + +inline const UnicodeSet& plusSignSet() { + return *unisets::get(unisets::PLUS_SIGN); +} + +} // namespace + + +ScientificMatcher::ScientificMatcher(const DecimalFormatSymbols& dfs, const Grouper& grouper) + : fExponentSeparatorString(dfs.getConstSymbol(DecimalFormatSymbols::kExponentialSymbol)), + fExponentMatcher(dfs, grouper, PARSE_FLAG_INTEGER_ONLY | PARSE_FLAG_GROUPING_DISABLED), + fIgnorablesMatcher(PARSE_FLAG_STRICT_IGNORABLES) { + + const UnicodeString& minusSign = dfs.getConstSymbol(DecimalFormatSymbols::kMinusSignSymbol); + if (minusSignSet().contains(minusSign)) { + fCustomMinusSign.setToBogus(); + } else { + fCustomMinusSign = minusSign; + } + + const UnicodeString& plusSign = dfs.getConstSymbol(DecimalFormatSymbols::kPlusSignSymbol); + if (plusSignSet().contains(plusSign)) { + fCustomPlusSign.setToBogus(); + } else { + fCustomPlusSign = plusSign; + } +} + +bool ScientificMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { + // Only accept scientific notation after the mantissa. + if (!result.seenNumber()) { + return false; + } + + // Only accept one exponent per string. + if (0 != (result.flags & FLAG_HAS_EXPONENT)) { + return false; + } + + // First match the scientific separator, and then match another number after it. + // NOTE: This is guarded by the smoke test; no need to check fExponentSeparatorString length again. + int32_t initialOffset = segment.getOffset(); + int32_t overlap = segment.getCommonPrefixLength(fExponentSeparatorString); + if (overlap == fExponentSeparatorString.length()) { + // Full exponent separator match. + + // First attempt to get a code point, returning true if we can't get one. + if (segment.length() == overlap) { + return true; + } + segment.adjustOffset(overlap); + + // Allow ignorables before the sign. + // Note: call site is guarded by the segment.length() check above. + // Note: the ignorables matcher should not touch the result. + fIgnorablesMatcher.match(segment, result, status); + if (segment.length() == 0) { + segment.setOffset(initialOffset); + return true; + } + + // Allow a sign, and then try to match digits. + int8_t exponentSign = 1; + if (segment.startsWith(minusSignSet())) { + exponentSign = -1; + segment.adjustOffsetByCodePoint(); + } else if (segment.startsWith(plusSignSet())) { + segment.adjustOffsetByCodePoint(); + } else if (segment.startsWith(fCustomMinusSign)) { + overlap = segment.getCommonPrefixLength(fCustomMinusSign); + if (overlap != fCustomMinusSign.length()) { + // Partial custom sign match + segment.setOffset(initialOffset); + return true; + } + exponentSign = -1; + segment.adjustOffset(overlap); + } else if (segment.startsWith(fCustomPlusSign)) { + overlap = segment.getCommonPrefixLength(fCustomPlusSign); + if (overlap != fCustomPlusSign.length()) { + // Partial custom sign match + segment.setOffset(initialOffset); + return true; + } + segment.adjustOffset(overlap); + } + + // Return true if the segment is empty. + if (segment.length() == 0) { + segment.setOffset(initialOffset); + return true; + } + + // Allow ignorables after the sign. + // Note: call site is guarded by the segment.length() check above. + // Note: the ignorables matcher should not touch the result. + fIgnorablesMatcher.match(segment, result, status); + if (segment.length() == 0) { + segment.setOffset(initialOffset); + return true; + } + + // We are supposed to accept E0 after NaN, so we need to make sure result.quantity is available. + bool wasBogus = result.quantity.bogus; + result.quantity.bogus = false; + int digitsOffset = segment.getOffset(); + bool digitsReturnValue = fExponentMatcher.match(segment, result, exponentSign, status); + result.quantity.bogus = wasBogus; + + if (segment.getOffset() != digitsOffset) { + // At least one exponent digit was matched. + result.flags |= FLAG_HAS_EXPONENT; + } else { + // No exponent digits were matched + segment.setOffset(initialOffset); + } + return digitsReturnValue; + + } else if (overlap == segment.length()) { + // Partial exponent separator match + return true; + } + + // No match + return false; +} + +bool ScientificMatcher::smokeTest(const StringSegment& segment) const { + return segment.startsWith(fExponentSeparatorString); +} + +UnicodeString ScientificMatcher::toString() const { + return u""; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_scientific.h b/intl/icu/source/i18n/numparse_scientific.h new file mode 100644 index 0000000000..5617c0c6a6 --- /dev/null +++ b/intl/icu/source/i18n/numparse_scientific.h @@ -0,0 +1,47 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_SCIENTIFIC_H__ +#define __NUMPARSE_SCIENTIFIC_H__ + +#include "numparse_types.h" +#include "numparse_decimal.h" +#include "numparse_symbols.h" +#include "unicode/numberformatter.h" + +using icu::number::impl::Grouper; + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { + + +class ScientificMatcher : public NumberParseMatcher, public UMemory { + public: + ScientificMatcher() = default; // WARNING: Leaves the object in an unusable state + + ScientificMatcher(const DecimalFormatSymbols& dfs, const Grouper& grouper); + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool smokeTest(const StringSegment& segment) const override; + + UnicodeString toString() const override; + + private: + UnicodeString fExponentSeparatorString; + DecimalMatcher fExponentMatcher; + IgnorablesMatcher fIgnorablesMatcher; + UnicodeString fCustomMinusSign; + UnicodeString fCustomPlusSign; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_SCIENTIFIC_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_symbols.cpp b/intl/icu/source/i18n/numparse_symbols.cpp new file mode 100644 index 0000000000..608f4f5c8b --- /dev/null +++ b/intl/icu/source/i18n/numparse_symbols.cpp @@ -0,0 +1,198 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_symbols.h" +#include "numparse_utils.h" +#include "string_segment.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +SymbolMatcher::SymbolMatcher(const UnicodeString& symbolString, unisets::Key key) { + fUniSet = unisets::get(key); + if (fUniSet->contains(symbolString)) { + fString.setToBogus(); + } else { + fString = symbolString; + } +} + +const UnicodeSet* SymbolMatcher::getSet() const { + return fUniSet; +} + +bool SymbolMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const { + // Smoke test first; this matcher might be disabled. + if (isDisabled(result)) { + return false; + } + + // Test the string first in order to consume trailing chars greedily. + int overlap = 0; + if (!fString.isEmpty()) { + overlap = segment.getCommonPrefixLength(fString); + if (overlap == fString.length()) { + segment.adjustOffset(fString.length()); + accept(segment, result); + return false; + } + } + + int cp = segment.getCodePoint(); + if (cp != -1 && fUniSet->contains(cp)) { + segment.adjustOffset(U16_LENGTH(cp)); + accept(segment, result); + return false; + } + + return overlap == segment.length(); +} + +bool SymbolMatcher::smokeTest(const StringSegment& segment) const { + return segment.startsWith(*fUniSet) || segment.startsWith(fString); +} + +UnicodeString SymbolMatcher::toString() const { + // TODO: Customize output for each symbol + return u""; +} + + +IgnorablesMatcher::IgnorablesMatcher(parse_flags_t parseFlags) : + SymbolMatcher( + {}, + (0 != (parseFlags & PARSE_FLAG_STRICT_IGNORABLES)) ? + unisets::STRICT_IGNORABLES : + unisets::DEFAULT_IGNORABLES) { +} + +bool IgnorablesMatcher::isFlexible() const { + return true; +} + +UnicodeString IgnorablesMatcher::toString() const { + return u""; +} + +bool IgnorablesMatcher::isDisabled(const ParsedNumber&) const { + return false; +} + +void IgnorablesMatcher::accept(StringSegment&, ParsedNumber&) const { + // No-op +} + + +InfinityMatcher::InfinityMatcher(const DecimalFormatSymbols& dfs) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kInfinitySymbol), unisets::INFINITY_SIGN) { +} + +bool InfinityMatcher::isDisabled(const ParsedNumber& result) const { + return 0 != (result.flags & FLAG_INFINITY); +} + +void InfinityMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.flags |= FLAG_INFINITY; + result.setCharsConsumed(segment); +} + + +MinusSignMatcher::MinusSignMatcher(const DecimalFormatSymbols& dfs, bool allowTrailing) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kMinusSignSymbol), unisets::MINUS_SIGN), + fAllowTrailing(allowTrailing) { +} + +bool MinusSignMatcher::isDisabled(const ParsedNumber& result) const { + return !fAllowTrailing && result.seenNumber(); +} + +void MinusSignMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.flags |= FLAG_NEGATIVE; + result.setCharsConsumed(segment); +} + + +NanMatcher::NanMatcher(const DecimalFormatSymbols& dfs) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kNaNSymbol), unisets::EMPTY) { +} + +bool NanMatcher::isDisabled(const ParsedNumber& result) const { + return result.seenNumber(); +} + +void NanMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.flags |= FLAG_NAN; + result.setCharsConsumed(segment); +} + + +PaddingMatcher::PaddingMatcher(const UnicodeString& padString) + : SymbolMatcher(padString, unisets::EMPTY) {} + +bool PaddingMatcher::isFlexible() const { + return true; +} + +bool PaddingMatcher::isDisabled(const ParsedNumber&) const { + return false; +} + +void PaddingMatcher::accept(StringSegment&, ParsedNumber&) const { + // No-op +} + + +PercentMatcher::PercentMatcher(const DecimalFormatSymbols& dfs) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kPercentSymbol), unisets::PERCENT_SIGN) { +} + +bool PercentMatcher::isDisabled(const ParsedNumber& result) const { + return 0 != (result.flags & FLAG_PERCENT); +} + +void PercentMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.flags |= FLAG_PERCENT; + result.setCharsConsumed(segment); +} + + +PermilleMatcher::PermilleMatcher(const DecimalFormatSymbols& dfs) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kPerMillSymbol), unisets::PERMILLE_SIGN) { +} + +bool PermilleMatcher::isDisabled(const ParsedNumber& result) const { + return 0 != (result.flags & FLAG_PERMILLE); +} + +void PermilleMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.flags |= FLAG_PERMILLE; + result.setCharsConsumed(segment); +} + + +PlusSignMatcher::PlusSignMatcher(const DecimalFormatSymbols& dfs, bool allowTrailing) + : SymbolMatcher(dfs.getConstSymbol(DecimalFormatSymbols::kPlusSignSymbol), unisets::PLUS_SIGN), + fAllowTrailing(allowTrailing) { +} + +bool PlusSignMatcher::isDisabled(const ParsedNumber& result) const { + return !fAllowTrailing && result.seenNumber(); +} + +void PlusSignMatcher::accept(StringSegment& segment, ParsedNumber& result) const { + result.setCharsConsumed(segment); +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_symbols.h b/intl/icu/source/i18n/numparse_symbols.h new file mode 100644 index 0000000000..beb133f7d0 --- /dev/null +++ b/intl/icu/source/i18n/numparse_symbols.h @@ -0,0 +1,173 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_SYMBOLS_H__ +#define __NUMPARSE_SYMBOLS_H__ + +#include "numparse_types.h" +#include "unicode/uniset.h" +#include "static_unicode_sets.h" + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { + + +/** + * A base class for many matchers that performs a simple match against a UnicodeString and/or UnicodeSet. + * + * @author sffc + */ +// Exported as U_I18N_API for tests +class U_I18N_API SymbolMatcher : public NumberParseMatcher, public UMemory { + public: + SymbolMatcher() = default; // WARNING: Leaves the object in an unusable state + + const UnicodeSet* getSet() const; + + bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const override; + + bool smokeTest(const StringSegment& segment) const override; + + UnicodeString toString() const override; + + virtual bool isDisabled(const ParsedNumber& result) const = 0; + + virtual void accept(StringSegment& segment, ParsedNumber& result) const = 0; + + protected: + UnicodeString fString; + const UnicodeSet* fUniSet; // a reference from numparse_unisets.h; never owned + + SymbolMatcher(const UnicodeString& symbolString, unisets::Key key); +}; + + +// Exported as U_I18N_API for tests +class U_I18N_API IgnorablesMatcher : public SymbolMatcher { + public: + IgnorablesMatcher() = default; // WARNING: Leaves the object in an unusable state + + IgnorablesMatcher(parse_flags_t parseFlags); + + bool isFlexible() const override; + + UnicodeString toString() const override; + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + + +class InfinityMatcher : public SymbolMatcher { + public: + InfinityMatcher() = default; // WARNING: Leaves the object in an unusable state + + InfinityMatcher(const DecimalFormatSymbols& dfs); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + + +// Exported as U_I18N_API for tests +class U_I18N_API MinusSignMatcher : public SymbolMatcher { + public: + MinusSignMatcher() = default; // WARNING: Leaves the object in an unusable state + + MinusSignMatcher(const DecimalFormatSymbols& dfs, bool allowTrailing); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; + + private: + bool fAllowTrailing; +}; + + +class NanMatcher : public SymbolMatcher { + public: + NanMatcher() = default; // WARNING: Leaves the object in an unusable state + + NanMatcher(const DecimalFormatSymbols& dfs); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + + +class PaddingMatcher : public SymbolMatcher { + public: + PaddingMatcher() = default; // WARNING: Leaves the object in an unusable state + + PaddingMatcher(const UnicodeString& padString); + + bool isFlexible() const override; + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + + +// Exported as U_I18N_API for tests +class U_I18N_API PercentMatcher : public SymbolMatcher { + public: + PercentMatcher() = default; // WARNING: Leaves the object in an unusable state + + PercentMatcher(const DecimalFormatSymbols& dfs); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + +// Exported as U_I18N_API for tests +class U_I18N_API PermilleMatcher : public SymbolMatcher { + public: + PermilleMatcher() = default; // WARNING: Leaves the object in an unusable state + + PermilleMatcher(const DecimalFormatSymbols& dfs); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; +}; + + +// Exported as U_I18N_API for tests +class U_I18N_API PlusSignMatcher : public SymbolMatcher { + public: + PlusSignMatcher() = default; // WARNING: Leaves the object in an unusable state + + PlusSignMatcher(const DecimalFormatSymbols& dfs, bool allowTrailing); + + protected: + bool isDisabled(const ParsedNumber& result) const override; + + void accept(StringSegment& segment, ParsedNumber& result) const override; + + private: + bool fAllowTrailing; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_SYMBOLS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_types.h b/intl/icu/source/i18n/numparse_types.h new file mode 100644 index 0000000000..8e881793fd --- /dev/null +++ b/intl/icu/source/i18n/numparse_types.h @@ -0,0 +1,272 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_TYPES_H__ +#define __NUMPARSE_TYPES_H__ + +#include "unicode/uobject.h" +#include "number_decimalquantity.h" +#include "string_segment.h" + +U_NAMESPACE_BEGIN +namespace numparse { +namespace impl { + +// Forward-declarations +class ParsedNumber; + +typedef int32_t result_flags_t; +typedef int32_t parse_flags_t; + +/** Flags for the type result_flags_t */ +enum ResultFlags { + FLAG_NEGATIVE = 0x0001, + FLAG_PERCENT = 0x0002, + FLAG_PERMILLE = 0x0004, + FLAG_HAS_EXPONENT = 0x0008, + // FLAG_HAS_DEFAULT_CURRENCY = 0x0010, // no longer used + FLAG_HAS_DECIMAL_SEPARATOR = 0x0020, + FLAG_NAN = 0x0040, + FLAG_INFINITY = 0x0080, + FLAG_FAIL = 0x0100, +}; + +/** Flags for the type parse_flags_t */ +enum ParseFlags { + PARSE_FLAG_IGNORE_CASE = 0x0001, + PARSE_FLAG_MONETARY_SEPARATORS = 0x0002, + PARSE_FLAG_STRICT_SEPARATORS = 0x0004, + PARSE_FLAG_STRICT_GROUPING_SIZE = 0x0008, + PARSE_FLAG_INTEGER_ONLY = 0x0010, + PARSE_FLAG_GROUPING_DISABLED = 0x0020, + // PARSE_FLAG_FRACTION_GROUPING_ENABLED = 0x0040, // see #10794 + PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES = 0x0080, + PARSE_FLAG_USE_FULL_AFFIXES = 0x0100, + PARSE_FLAG_EXACT_AFFIX = 0x0200, + PARSE_FLAG_PLUS_SIGN_ALLOWED = 0x0400, + // PARSE_FLAG_OPTIMIZE = 0x0800, // no longer used + // PARSE_FLAG_FORCE_BIG_DECIMAL = 0x1000, // not used in ICU4C + PARSE_FLAG_NO_FOREIGN_CURRENCY = 0x2000, + PARSE_FLAG_ALLOW_INFINITE_RECURSION = 0x4000, + PARSE_FLAG_STRICT_IGNORABLES = 0x8000, +}; + + +// TODO: Is this class worthwhile? +template +class CompactUnicodeString { + public: + CompactUnicodeString() { + static_assert(stackCapacity > 0, "cannot have zero space on stack"); + fBuffer[0] = 0; + } + + CompactUnicodeString(const UnicodeString& text, UErrorCode& status) + : fBuffer(text.length() + 1, status) { + if (U_FAILURE(status)) { return; } + uprv_memcpy(fBuffer.getAlias(), text.getBuffer(), sizeof(char16_t) * text.length()); + fBuffer[text.length()] = 0; + } + + inline UnicodeString toAliasedUnicodeString() const { + return UnicodeString(true, fBuffer.getAlias(), -1); + } + + bool operator==(const CompactUnicodeString& other) const { + // Use the alias-only constructor and then call UnicodeString operator== + return toAliasedUnicodeString() == other.toAliasedUnicodeString(); + } + + private: + MaybeStackArray fBuffer; +}; + + +/** + * Struct-like class to hold the results of a parsing routine. + * + * @author sffc + */ +// Exported as U_I18N_API for tests +class U_I18N_API ParsedNumber { + public: + + /** + * The numerical value that was parsed. + */ + ::icu::number::impl::DecimalQuantity quantity; + + /** + * The index of the last char consumed during parsing. If parsing started at index 0, this is equal + * to the number of chars consumed. This is NOT necessarily the same as the StringSegment offset; + * "weak" chars, like whitespace, change the offset, but the charsConsumed is not touched until a + * "strong" char is encountered. + */ + int32_t charEnd; + + /** + * Boolean flags (see constants above). + */ + result_flags_t flags; + + /** + * The pattern string corresponding to the prefix that got consumed. + */ + UnicodeString prefix; + + /** + * The pattern string corresponding to the suffix that got consumed. + */ + UnicodeString suffix; + + /** + * The currency that got consumed. + */ + char16_t currencyCode[4]; + + ParsedNumber(); + + ParsedNumber(const ParsedNumber& other) = default; + + ParsedNumber& operator=(const ParsedNumber& other) = default; + + void clear(); + + /** + * Call this method to register that a "strong" char was consumed. This should be done after calling + * {@link StringSegment#setOffset} or {@link StringSegment#adjustOffset} except when the char is + * "weak", like whitespace. + * + *

    + * What is a strong versus weak char? The behavior of number parsing is to "stop" + * after reading the number, even if there is other content following the number. For example, after + * parsing the string "123 " (123 followed by a space), the cursor should be set to 3, not 4, even + * though there are matchers that accept whitespace. In this example, the digits are strong, whereas + * the whitespace is weak. Grouping separators are weak, whereas decimal separators are strong. Most + * other chars are strong. + * + * @param segment + * The current StringSegment, usually immediately following a call to setOffset. + */ + void setCharsConsumed(const StringSegment& segment); + + /** Apply certain number-related flags to the DecimalQuantity. */ + void postProcess(); + + /** + * Returns whether this the parse was successful. To be successful, at least one char must have been + * consumed, and the failure flag must not be set. + */ + bool success() const; + + bool seenNumber() const; + + double getDouble(UErrorCode& status) const; + + void populateFormattable(Formattable& output, parse_flags_t parseFlags) const; + + bool isBetterThan(const ParsedNumber& other); +}; + + +/** + * The core interface implemented by all matchers used for number parsing. + * + * Given a string, there should NOT be more than one way to consume the string with the same matcher + * applied multiple times. If there is, the non-greedy parsing algorithm will be unhappy and may enter an + * exponential-time loop. For example, consider the "A Matcher" that accepts "any number of As". Given + * the string "AAAA", there are 2^N = 8 ways to apply the A Matcher to this string: you could have the A + * Matcher apply 4 times to each character; you could have it apply just once to all the characters; you + * could have it apply to the first 2 characters and the second 2 characters; and so on. A better version + * of the "A Matcher" would be for it to accept exactly one A, and allow the algorithm to run it + * repeatedly to consume a string of multiple As. The A Matcher can implement the Flexible interface + * below to signal that it can be applied multiple times in a row. + * + * @author sffc + */ +// Exported as U_I18N_API for tests +class U_I18N_API NumberParseMatcher { + public: + virtual ~NumberParseMatcher(); + + /** + * Matchers can override this method to return true to indicate that they are optional and can be run + * repeatedly. Used by SeriesMatcher, primarily in the context of IgnorablesMatcher. + */ + virtual bool isFlexible() const { + return false; + } + + /** + * Runs this matcher starting at the beginning of the given StringSegment. If this matcher finds + * something interesting in the StringSegment, it should update the offset of the StringSegment + * corresponding to how many chars were matched. + * + * This method is thread-safe. + * + * @param segment + * The StringSegment to match against. Matches always start at the beginning of the + * segment. The segment is guaranteed to contain at least one char. + * @param result + * The data structure to store results if the match succeeds. + * @return Whether this matcher thinks there may be more interesting chars beyond the end of the + * string segment. + */ + virtual bool match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const = 0; + + /** + * Performs a fast "smoke check" for whether or not this matcher could possibly match against the + * given string segment. The test should be as fast as possible but also as restrictive as possible. + * For example, matchers can maintain a UnicodeSet of all code points that count possibly start a + * match. Matchers should use the {@link StringSegment#startsWith} method in order to correctly + * handle case folding. + * + * @param segment + * The segment to check against. + * @return true if the matcher might be able to match against this segment; false if it definitely + * will not be able to match. + */ + virtual bool smokeTest(const StringSegment& segment) const = 0; + + /** + * Method called at the end of a parse, after all matchers have failed to consume any more chars. + * Allows a matcher to make final modifications to the result given the knowledge that no more + * matches are possible. + * + * @param result + * The data structure to store results. + */ + virtual void postProcess(ParsedNumber&) const { + // Default implementation: no-op + } + + // String for debugging + virtual UnicodeString toString() const = 0; + + protected: + // No construction except by subclasses! + NumberParseMatcher() = default; +}; + + +/** + * Interface for use in arguments. + */ +// Exported as U_I18N_API for tests +class U_I18N_API MutableMatcherCollection { + public: + virtual ~MutableMatcherCollection() = default; + + virtual void addMatcher(NumberParseMatcher& matcher) = 0; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_TYPES_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_utils.h b/intl/icu/source/i18n/numparse_utils.h new file mode 100644 index 0000000000..8fda4f4369 --- /dev/null +++ b/intl/icu/source/i18n/numparse_utils.h @@ -0,0 +1,43 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __NUMPARSE_UTILS_H__ +#define __NUMPARSE_UTILS_H__ + +#include "numparse_types.h" +#include "unicode/uniset.h" + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { +namespace utils { + + +inline static void putLeadCodePoints(const UnicodeSet* input, UnicodeSet* output) { + for (int32_t i = 0; i < input->getRangeCount(); i++) { + output->add(input->getRangeStart(i), input->getRangeEnd(i)); + } + // TODO: ANDY: How to iterate over the strings in ICU4C UnicodeSet? +} + +inline static void putLeadCodePoint(const UnicodeString& input, UnicodeSet* output) { + if (!input.isEmpty()) { + output->add(input.char32At(0)); + } +} + +inline static void copyCurrencyCode(char16_t* dest, const char16_t* src) { + uprv_memcpy(dest, src, sizeof(char16_t) * 3); + dest[3] = 0; +} + + +} // namespace utils +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__NUMPARSE_UTILS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_validators.cpp b/intl/icu/source/i18n/numparse_validators.cpp new file mode 100644 index 0000000000..12d3465c4e --- /dev/null +++ b/intl/icu/source/i18n/numparse_validators.cpp @@ -0,0 +1,85 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numparse_types.h" +#include "numparse_validators.h" +#include "static_unicode_sets.h" + +using namespace icu; +using namespace icu::numparse; +using namespace icu::numparse::impl; + + +void RequireAffixValidator::postProcess(ParsedNumber& result) const { + if (result.prefix.isBogus() || result.suffix.isBogus()) { + // We saw a prefix or a suffix but not both. Fail the parse. + result.flags |= FLAG_FAIL; + } +} + +UnicodeString RequireAffixValidator::toString() const { + return u""; +} + + +void RequireCurrencyValidator::postProcess(ParsedNumber& result) const { + if (result.currencyCode[0] == 0) { + result.flags |= FLAG_FAIL; + } +} + +UnicodeString RequireCurrencyValidator::toString() const { + return u""; +} + + +RequireDecimalSeparatorValidator::RequireDecimalSeparatorValidator(bool patternHasDecimalSeparator) + : fPatternHasDecimalSeparator(patternHasDecimalSeparator) { +} + +void RequireDecimalSeparatorValidator::postProcess(ParsedNumber& result) const { + bool parseHasDecimalSeparator = 0 != (result.flags & FLAG_HAS_DECIMAL_SEPARATOR); + if (parseHasDecimalSeparator != fPatternHasDecimalSeparator) { + result.flags |= FLAG_FAIL; + } +} + +UnicodeString RequireDecimalSeparatorValidator::toString() const { + return u""; +} + + +void RequireNumberValidator::postProcess(ParsedNumber& result) const { + // Require that a number is matched. + if (!result.seenNumber()) { + result.flags |= FLAG_FAIL; + } +} + +UnicodeString RequireNumberValidator::toString() const { + return u""; +} + +MultiplierParseHandler::MultiplierParseHandler(::icu::number::Scale multiplier) + : fMultiplier(std::move(multiplier)) {} + +void MultiplierParseHandler::postProcess(ParsedNumber& result) const { + if (!result.quantity.bogus) { + fMultiplier.applyReciprocalTo(result.quantity); + // NOTE: It is okay if the multiplier was negative. + } +} + +UnicodeString MultiplierParseHandler::toString() const { + return u""; +} + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numparse_validators.h b/intl/icu/source/i18n/numparse_validators.h new file mode 100644 index 0000000000..9bb4b482b5 --- /dev/null +++ b/intl/icu/source/i18n/numparse_validators.h @@ -0,0 +1,95 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMPARSE_VALIDATORS_H__ +#define __SOURCE_NUMPARSE_VALIDATORS_H__ + +#include "numparse_types.h" +#include "static_unicode_sets.h" + +U_NAMESPACE_BEGIN namespace numparse { +namespace impl { + + +class ValidationMatcher : public NumberParseMatcher { + public: + bool match(StringSegment&, ParsedNumber&, UErrorCode&) const override { + // No-op + return false; + } + + bool smokeTest(const StringSegment&) const override { + // No-op + return false; + } + + void postProcess(ParsedNumber& result) const override = 0; +}; + + +class RequireAffixValidator : public ValidationMatcher, public UMemory { + public: + void postProcess(ParsedNumber& result) const override; + + UnicodeString toString() const override; +}; + + +class RequireCurrencyValidator : public ValidationMatcher, public UMemory { + public: + void postProcess(ParsedNumber& result) const override; + + UnicodeString toString() const override; +}; + + +class RequireDecimalSeparatorValidator : public ValidationMatcher, public UMemory { + public: + RequireDecimalSeparatorValidator() = default; // leaves instance in valid but undefined state + + RequireDecimalSeparatorValidator(bool patternHasDecimalSeparator); + + void postProcess(ParsedNumber& result) const override; + + UnicodeString toString() const override; + + private: + bool fPatternHasDecimalSeparator; +}; + + +class RequireNumberValidator : public ValidationMatcher, public UMemory { + public: + void postProcess(ParsedNumber& result) const override; + + UnicodeString toString() const override; +}; + + +/** + * Wraps a {@link Multiplier} for use in the number parsing pipeline. + */ +class MultiplierParseHandler : public ValidationMatcher, public UMemory { + public: + MultiplierParseHandler() = default; // leaves instance in valid but undefined state + + MultiplierParseHandler(::icu::number::Scale multiplier); + + void postProcess(ParsedNumber& result) const override; + + UnicodeString toString() const override; + + private: + ::icu::number::Scale fMultiplier; +}; + + +} // namespace impl +} // namespace numparse +U_NAMESPACE_END + +#endif //__SOURCE_NUMPARSE_VALIDATORS_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numrange_capi.cpp b/intl/icu/source/i18n/numrange_capi.cpp new file mode 100644 index 0000000000..9222969eb4 --- /dev/null +++ b/intl/icu/source/i18n/numrange_capi.cpp @@ -0,0 +1,198 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "fphdlimp.h" +#include "number_utypes.h" +#include "numparse_types.h" +#include "formattedval_impl.h" +#include "numrange_impl.h" +#include "number_decnum.h" +#include "unicode/numberrangeformatter.h" +#include "unicode/unumberrangeformatter.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +U_NAMESPACE_BEGIN +namespace number { +namespace impl { + +/** + * Implementation class for UNumberRangeFormatter. Wraps a LocalizedRangeNumberFormatter. + */ +struct UNumberRangeFormatterData : public UMemory, + // Magic number as ASCII == "NRF" (NumberRangeFormatter) + public IcuCApiHelper { + LocalizedNumberRangeFormatter fFormatter; +}; + +struct UFormattedNumberRangeImpl; + +// Magic number as ASCII == "FDN" (FormatteDNumber) +typedef IcuCApiHelper UFormattedNumberRangeApiHelper; + +struct UFormattedNumberRangeImpl : public UFormattedValueImpl, public UFormattedNumberRangeApiHelper { + UFormattedNumberRangeImpl(); + ~UFormattedNumberRangeImpl(); + + FormattedNumberRange fImpl; + UFormattedNumberRangeData fData; +}; + +UFormattedNumberRangeImpl::UFormattedNumberRangeImpl() + : fImpl(&fData) { + fFormattedValue = &fImpl; +} + +UFormattedNumberRangeImpl::~UFormattedNumberRangeImpl() { + // Disown the data from fImpl so it doesn't get deleted twice + fImpl.fData = nullptr; +} + +} // namespace impl +} // namespace number +U_NAMESPACE_END + + +UPRV_FORMATTED_VALUE_CAPI_NO_IMPLTYPE_AUTO_IMPL( + UFormattedNumberRange, + UFormattedNumberRangeImpl, + UFormattedNumberRangeApiHelper, + unumrf) + + +const UFormattedNumberRangeData* number::impl::validateUFormattedNumberRange( + const UFormattedNumberRange* uresult, UErrorCode& status) { + auto* result = UFormattedNumberRangeApiHelper::validate(uresult, status); + if (U_FAILURE(status)) { + return nullptr; + } + return &result->fData; +} + + +U_CAPI UNumberRangeFormatter* U_EXPORT2 +unumrf_openForSkeletonWithCollapseAndIdentityFallback( + const char16_t* skeleton, + int32_t skeletonLen, + UNumberRangeCollapse collapse, + UNumberRangeIdentityFallback identityFallback, + const char* locale, + UParseError* perror, + UErrorCode* ec) { + auto* impl = new UNumberRangeFormatterData(); + if (impl == nullptr) { + *ec = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + // Readonly-alias constructor (first argument is whether we are NUL-terminated) + UnicodeString skeletonString(skeletonLen == -1, skeleton, skeletonLen); + UParseError tempParseError; + impl->fFormatter = NumberRangeFormatter::withLocale(locale) + .numberFormatterBoth(NumberFormatter::forSkeleton(skeletonString, (perror == nullptr) ? tempParseError : *perror, *ec)) + .collapse(collapse) + .identityFallback(identityFallback); + return impl->exportForC(); +} + +U_CAPI void U_EXPORT2 +unumrf_formatDoubleRange( + const UNumberRangeFormatter* uformatter, + double first, + double second, + UFormattedNumberRange* uresult, + UErrorCode* ec) { + const UNumberRangeFormatterData* formatter = UNumberRangeFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberRangeApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + result->fData.resetString(); + result->fData.quantity1.clear(); + result->fData.quantity2.clear(); + result->fData.quantity1.setToDouble(first); + result->fData.quantity2.setToDouble(second); + formatter->fFormatter.formatImpl(result->fData, first == second, *ec); +} + +U_CAPI void U_EXPORT2 +unumrf_formatDecimalRange( + const UNumberRangeFormatter* uformatter, + const char* first, int32_t firstLen, + const char* second, int32_t secondLen, + UFormattedNumberRange* uresult, + UErrorCode* ec) { + const UNumberRangeFormatterData* formatter = UNumberRangeFormatterData::validate(uformatter, *ec); + auto* result = UFormattedNumberRangeApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { return; } + + result->fData.resetString(); + result->fData.quantity1.clear(); + result->fData.quantity2.clear(); + result->fData.quantity1.setToDecNumber({first, firstLen}, *ec); + result->fData.quantity2.setToDecNumber({second, secondLen}, *ec); + formatter->fFormatter.formatImpl(result->fData, first == second, *ec); +} + +U_CAPI UNumberRangeIdentityResult U_EXPORT2 +unumrf_resultGetIdentityResult( + const UFormattedNumberRange* uresult, + UErrorCode* ec) { + auto* result = UFormattedNumberRangeApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return UNUM_IDENTITY_RESULT_COUNT; + } + return result->fData.identityResult; +} + +U_CAPI int32_t U_EXPORT2 +unumrf_resultGetFirstDecimalNumber( + const UFormattedNumberRange* uresult, + char* dest, + int32_t destCapacity, + UErrorCode* ec) { + const auto* result = UFormattedNumberRangeApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + DecNum decnum; + return result->fData.quantity1.toDecNum(decnum, *ec) + .toCharString(*ec) + .extract(dest, destCapacity, *ec); +} + +U_CAPI int32_t U_EXPORT2 +unumrf_resultGetSecondDecimalNumber( + const UFormattedNumberRange* uresult, + char* dest, + int32_t destCapacity, + UErrorCode* ec) { + const auto* result = UFormattedNumberRangeApiHelper::validate(uresult, *ec); + if (U_FAILURE(*ec)) { + return 0; + } + DecNum decnum; + return result->fData.quantity2 + .toDecNum(decnum, *ec) + .toCharString(*ec) + .extract(dest, destCapacity, *ec); +} + +U_CAPI void U_EXPORT2 +unumrf_close(UNumberRangeFormatter* f) { + UErrorCode localStatus = U_ZERO_ERROR; + const UNumberRangeFormatterData* impl = UNumberRangeFormatterData::validate(f, localStatus); + delete impl; +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numrange_fluent.cpp b/intl/icu/source/i18n/numrange_fluent.cpp new file mode 100644 index 0000000000..1b2ebc6e61 --- /dev/null +++ b/intl/icu/source/i18n/numrange_fluent.cpp @@ -0,0 +1,410 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "numrange_impl.h" +#include "util.h" +#include "number_utypes.h" +#include "number_decnum.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + + +// This function needs to be declared in this namespace so it can be friended. +// NOTE: In Java, this logic is handled in the resolve() function. +void icu::number::impl::touchRangeLocales(RangeMacroProps& macros) { + macros.formatter1.fMacros.locale = macros.locale; + macros.formatter2.fMacros.locale = macros.locale; +} + + +template +Derived NumberRangeFormatterSettings::numberFormatterBoth(const UnlocalizedNumberFormatter& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter1 = formatter; + copy.fMacros.singleFormatter = true; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterBoth(const UnlocalizedNumberFormatter& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter1 = formatter; + move.fMacros.singleFormatter = true; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterBoth(UnlocalizedNumberFormatter&& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter1 = std::move(formatter); + copy.fMacros.singleFormatter = true; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterBoth(UnlocalizedNumberFormatter&& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter1 = std::move(formatter); + move.fMacros.singleFormatter = true; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterFirst(const UnlocalizedNumberFormatter& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter1 = formatter; + copy.fMacros.singleFormatter = false; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterFirst(const UnlocalizedNumberFormatter& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter1 = formatter; + move.fMacros.singleFormatter = false; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterFirst(UnlocalizedNumberFormatter&& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter1 = std::move(formatter); + copy.fMacros.singleFormatter = false; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterFirst(UnlocalizedNumberFormatter&& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter1 = std::move(formatter); + move.fMacros.singleFormatter = false; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterSecond(const UnlocalizedNumberFormatter& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter2 = formatter; + copy.fMacros.singleFormatter = false; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterSecond(const UnlocalizedNumberFormatter& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter2 = formatter; + move.fMacros.singleFormatter = false; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterSecond(UnlocalizedNumberFormatter&& formatter) const& { + Derived copy(*this); + copy.fMacros.formatter2 = std::move(formatter); + copy.fMacros.singleFormatter = false; + touchRangeLocales(copy.fMacros); + return copy; +} + +template +Derived NumberRangeFormatterSettings::numberFormatterSecond(UnlocalizedNumberFormatter&& formatter) && { + Derived move(std::move(*this)); + move.fMacros.formatter2 = std::move(formatter); + move.fMacros.singleFormatter = false; + touchRangeLocales(move.fMacros); + return move; +} + +template +Derived NumberRangeFormatterSettings::collapse(UNumberRangeCollapse collapse) const& { + Derived copy(*this); + copy.fMacros.collapse = collapse; + return copy; +} + +template +Derived NumberRangeFormatterSettings::collapse(UNumberRangeCollapse collapse) && { + Derived move(std::move(*this)); + move.fMacros.collapse = collapse; + return move; +} + +template +Derived NumberRangeFormatterSettings::identityFallback(UNumberRangeIdentityFallback identityFallback) const& { + Derived copy(*this); + copy.fMacros.identityFallback = identityFallback; + return copy; +} + +template +Derived NumberRangeFormatterSettings::identityFallback(UNumberRangeIdentityFallback identityFallback) && { + Derived move(std::move(*this)); + move.fMacros.identityFallback = identityFallback; + return move; +} + +template +LocalPointer NumberRangeFormatterSettings::clone() const & { + return LocalPointer(new Derived(*this)); +} + +template +LocalPointer NumberRangeFormatterSettings::clone() && { + return LocalPointer(new Derived(std::move(*this))); +} + +// Declare all classes that implement NumberRangeFormatterSettings +// See https://stackoverflow.com/a/495056/1407170 +template +class icu::number::NumberRangeFormatterSettings; +template +class icu::number::NumberRangeFormatterSettings; + + +UnlocalizedNumberRangeFormatter NumberRangeFormatter::with() { + UnlocalizedNumberRangeFormatter result; + return result; +} + +LocalizedNumberRangeFormatter NumberRangeFormatter::withLocale(const Locale& locale) { + return with().locale(locale); +} + + +template using NFS = NumberRangeFormatterSettings; +using LNF = LocalizedNumberRangeFormatter; +using UNF = UnlocalizedNumberRangeFormatter; + +UnlocalizedNumberRangeFormatter::UnlocalizedNumberRangeFormatter(const UNF& other) + : UNF(static_cast&>(other)) {} + +UnlocalizedNumberRangeFormatter::UnlocalizedNumberRangeFormatter(const NFS& other) + : NFS(other) { + // No additional fields to assign +} + +// Make default copy constructor call the NumberRangeFormatterSettings copy constructor. +UnlocalizedNumberRangeFormatter::UnlocalizedNumberRangeFormatter(UNF&& src) noexcept + : UNF(static_cast&&>(src)) {} + +UnlocalizedNumberRangeFormatter::UnlocalizedNumberRangeFormatter(NFS&& src) noexcept + : NFS(std::move(src)) { + // No additional fields to assign +} + +UnlocalizedNumberRangeFormatter& UnlocalizedNumberRangeFormatter::operator=(const UNF& other) { + NFS::operator=(static_cast&>(other)); + // No additional fields to assign + return *this; +} + +UnlocalizedNumberRangeFormatter& UnlocalizedNumberRangeFormatter::operator=(UNF&& src) noexcept { + NFS::operator=(static_cast&&>(src)); + // No additional fields to assign + return *this; +} + +// Make default copy constructor call the NumberRangeFormatterSettings copy constructor. +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(const LNF& other) + : LNF(static_cast&>(other)) {} + +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(const NFS& other) + : NFS(other) { + // No additional fields to assign +} + +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(LocalizedNumberRangeFormatter&& src) noexcept + : LNF(static_cast&&>(src)) {} + +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(NFS&& src) noexcept + : NFS(std::move(src)) { + // Steal the compiled formatter + LNF&& _src = static_cast(src); +#ifndef __wasi__ + auto* stolen = _src.fAtomicFormatter.exchange(nullptr); + delete fAtomicFormatter.exchange(stolen); +#else + delete fAtomicFormatter; + fAtomicFormatter = _src.fAtomicFormatter; + _src.fAtomicFormatter = nullptr; +#endif +} + +LocalizedNumberRangeFormatter& LocalizedNumberRangeFormatter::operator=(const LNF& other) { + if (this == &other) { return *this; } // self-assignment: no-op + NFS::operator=(static_cast&>(other)); + // Do not steal; just clear +#ifndef __wasi__ + delete fAtomicFormatter.exchange(nullptr); +#else + delete fAtomicFormatter; +#endif + return *this; +} + +LocalizedNumberRangeFormatter& LocalizedNumberRangeFormatter::operator=(LNF&& src) noexcept { + NFS::operator=(static_cast&&>(src)); + // Steal the compiled formatter +#ifndef __wasi__ + auto* stolen = src.fAtomicFormatter.exchange(nullptr); + delete fAtomicFormatter.exchange(stolen); +#else + delete fAtomicFormatter; + fAtomicFormatter = src.fAtomicFormatter; + src.fAtomicFormatter = nullptr; +#endif + return *this; +} + + +LocalizedNumberRangeFormatter::~LocalizedNumberRangeFormatter() { +#ifndef __wasi__ + delete fAtomicFormatter.exchange(nullptr); +#else + delete fAtomicFormatter; +#endif +} + +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(const RangeMacroProps& macros, const Locale& locale) { + fMacros = macros; + fMacros.locale = locale; + touchRangeLocales(fMacros); +} + +LocalizedNumberRangeFormatter::LocalizedNumberRangeFormatter(RangeMacroProps&& macros, const Locale& locale) { + fMacros = std::move(macros); + fMacros.locale = locale; + touchRangeLocales(fMacros); +} + +LocalizedNumberRangeFormatter UnlocalizedNumberRangeFormatter::locale(const Locale& locale) const& { + return LocalizedNumberRangeFormatter(fMacros, locale); +} + +LocalizedNumberRangeFormatter UnlocalizedNumberRangeFormatter::locale(const Locale& locale)&& { + return LocalizedNumberRangeFormatter(std::move(fMacros), locale); +} + + +FormattedNumberRange LocalizedNumberRangeFormatter::formatFormattableRange( + const Formattable& first, const Formattable& second, UErrorCode& status) const { + if (U_FAILURE(status)) { + return FormattedNumberRange(U_ILLEGAL_ARGUMENT_ERROR); + } + + auto results = new UFormattedNumberRangeData(); + if (results == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return FormattedNumberRange(status); + } + + first.populateDecimalQuantity(results->quantity1, status); + if (U_FAILURE(status)) { + return FormattedNumberRange(status); + } + + second.populateDecimalQuantity(results->quantity2, status); + if (U_FAILURE(status)) { + return FormattedNumberRange(status); + } + + formatImpl(*results, first == second, status); + + // Do not save the results object if we encountered a failure. + if (U_SUCCESS(status)) { + return FormattedNumberRange(results); + } else { + delete results; + return FormattedNumberRange(status); + } +} + +void LocalizedNumberRangeFormatter::formatImpl( + UFormattedNumberRangeData& results, bool equalBeforeRounding, UErrorCode& status) const { + auto* impl = getFormatter(status); + if (U_FAILURE(status)) { + return; + } + if (impl == nullptr) { + status = U_INTERNAL_PROGRAM_ERROR; + return; + } + impl->format(results, equalBeforeRounding, status); + if (U_FAILURE(status)) { + return; + } + results.getStringRef().writeTerminator(status); +} + +const impl::NumberRangeFormatterImpl* +LocalizedNumberRangeFormatter::getFormatter(UErrorCode& status) const { + // TODO: Move this into umutex.h? (similar logic also in decimfmt.cpp) + // See ICU-20146 + + if (U_FAILURE(status)) { + return nullptr; + } + + // First try to get the pre-computed formatter +#ifndef __wasi__ + auto* ptr = fAtomicFormatter.load(); +#else + auto* ptr = fAtomicFormatter; +#endif + if (ptr != nullptr) { + return ptr; + } + + // Try computing the formatter on our own + auto* temp = new NumberRangeFormatterImpl(fMacros, status); + if (U_FAILURE(status)) { + delete temp; + return nullptr; + } + if (temp == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + + // Note: ptr starts as nullptr; during compare_exchange, + // it is set to what is actually stored in the atomic + // if another thread beat us to computing the formatter object. + auto* nonConstThis = const_cast(this); +#ifndef __wasi__ + if (!nonConstThis->fAtomicFormatter.compare_exchange_strong(ptr, temp)) { + // Another thread beat us to computing the formatter + delete temp; + return ptr; + } else { + // Our copy of the formatter got stored in the atomic + return temp; + } +#else + nonConstThis->fAtomicFormatter = temp; + return temp; +#endif + +} + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numrange_impl.cpp b/intl/icu/source/i18n/numrange_impl.cpp new file mode 100644 index 0000000000..002a8b2a21 --- /dev/null +++ b/intl/icu/source/i18n/numrange_impl.cpp @@ -0,0 +1,459 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "unicode/numberrangeformatter.h" +#include "numrange_impl.h" +#include "patternprops.h" +#include "pluralranges.h" +#include "uresimp.h" +#include "util.h" + +using namespace icu; +using namespace icu::number; +using namespace icu::number::impl; + +namespace { + +// Helper function for 2-dimensional switch statement +constexpr int8_t identity2d(UNumberRangeIdentityFallback a, UNumberRangeIdentityResult b) { + return static_cast(a) | (static_cast(b) << 4); +} + + +struct NumberRangeData { + SimpleFormatter rangePattern; + // Note: approximatelyPattern is unused since ICU 69. + // SimpleFormatter approximatelyPattern; +}; + +class NumberRangeDataSink : public ResourceSink { + public: + NumberRangeDataSink(NumberRangeData& data) : fData(data) {} + + void put(const char* key, ResourceValue& value, UBool /*noFallback*/, UErrorCode& status) override { + ResourceTable miscTable = value.getTable(status); + if (U_FAILURE(status)) { return; } + for (int i = 0; miscTable.getKeyAndValue(i, key, value); i++) { + if (uprv_strcmp(key, "range") == 0) { + if (hasRangeData()) { + continue; // have already seen this pattern + } + fData.rangePattern = {value.getUnicodeString(status), status}; + } + /* + // Note: approximatelyPattern is unused since ICU 69. + else if (uprv_strcmp(key, "approximately") == 0) { + if (hasApproxData()) { + continue; // have already seen this pattern + } + fData.approximatelyPattern = {value.getUnicodeString(status), status}; + } + */ + } + } + + bool hasRangeData() { + return fData.rangePattern.getArgumentLimit() != 0; + } + + /* + // Note: approximatelyPattern is unused since ICU 69. + bool hasApproxData() { + return fData.approximatelyPattern.getArgumentLimit() != 0; + } + */ + + bool isComplete() { + return hasRangeData() /* && hasApproxData() */; + } + + void fillInDefaults(UErrorCode& status) { + if (!hasRangeData()) { + fData.rangePattern = {u"{0}–{1}", status}; + } + /* + if (!hasApproxData()) { + fData.approximatelyPattern = {u"~{0}", status}; + } + */ + } + + private: + NumberRangeData& fData; +}; + +void getNumberRangeData(const char* localeName, const char* nsName, NumberRangeData& data, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + LocalUResourceBundlePointer rb(ures_open(nullptr, localeName, &status)); + if (U_FAILURE(status)) { return; } + NumberRangeDataSink sink(data); + + CharString dataPath; + dataPath.append("NumberElements/", -1, status); + dataPath.append(nsName, -1, status); + dataPath.append("/miscPatterns", -1, status); + if (U_FAILURE(status)) { return; } + + UErrorCode localStatus = U_ZERO_ERROR; + ures_getAllItemsWithFallback(rb.getAlias(), dataPath.data(), sink, localStatus); + if (U_FAILURE(localStatus) && localStatus != U_MISSING_RESOURCE_ERROR) { + status = localStatus; + return; + } + + // Fall back to latn if necessary + if (!sink.isComplete()) { + ures_getAllItemsWithFallback(rb.getAlias(), "NumberElements/latn/miscPatterns", sink, status); + } + + sink.fillInDefaults(status); +} + +} // namespace + + + +NumberRangeFormatterImpl::NumberRangeFormatterImpl(const RangeMacroProps& macros, UErrorCode& status) + : formatterImpl1(macros.formatter1.fMacros, status), + formatterImpl2(macros.formatter2.fMacros, status), + fSameFormatters(macros.singleFormatter), + fCollapse(macros.collapse), + fIdentityFallback(macros.identityFallback), + fApproximatelyFormatter(status) { + + const char* nsName = formatterImpl1.getRawMicroProps().nsName; + if (!fSameFormatters && uprv_strcmp(nsName, formatterImpl2.getRawMicroProps().nsName) != 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + NumberRangeData data; + getNumberRangeData(macros.locale.getName(), nsName, data, status); + if (U_FAILURE(status)) { return; } + fRangeFormatter = data.rangePattern; + + if (fSameFormatters && ( + fIdentityFallback == UNUM_IDENTITY_FALLBACK_APPROXIMATELY || + fIdentityFallback == UNUM_IDENTITY_FALLBACK_APPROXIMATELY_OR_SINGLE_VALUE)) { + MacroProps approximatelyMacros(macros.formatter1.fMacros); + approximatelyMacros.approximately = true; + // Use in-place construction because NumberFormatterImpl has internal self-pointers + fApproximatelyFormatter.~NumberFormatterImpl(); + new (&fApproximatelyFormatter) NumberFormatterImpl(approximatelyMacros, status); + } + + // TODO: Get locale from PluralRules instead? + fPluralRanges = StandardPluralRanges::forLocale(macros.locale, status); + if (U_FAILURE(status)) { return; } +} + +void NumberRangeFormatterImpl::format(UFormattedNumberRangeData& data, bool equalBeforeRounding, UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + + MicroProps micros1; + MicroProps micros2; + formatterImpl1.preProcess(data.quantity1, micros1, status); + if (fSameFormatters) { + formatterImpl1.preProcess(data.quantity2, micros2, status); + } else { + formatterImpl2.preProcess(data.quantity2, micros2, status); + } + if (U_FAILURE(status)) { + return; + } + + // If any of the affixes are different, an identity is not possible + // and we must use formatRange(). + // TODO: Write this as MicroProps operator==() ? + // TODO: Avoid the redundancy of these equality operations with the + // ones in formatRange? + if (!micros1.modInner->semanticallyEquivalent(*micros2.modInner) + || !micros1.modMiddle->semanticallyEquivalent(*micros2.modMiddle) + || !micros1.modOuter->semanticallyEquivalent(*micros2.modOuter)) { + formatRange(data, micros1, micros2, status); + data.identityResult = UNUM_IDENTITY_RESULT_NOT_EQUAL; + return; + } + + // Check for identity + if (equalBeforeRounding) { + data.identityResult = UNUM_IDENTITY_RESULT_EQUAL_BEFORE_ROUNDING; + } else if (data.quantity1 == data.quantity2) { + data.identityResult = UNUM_IDENTITY_RESULT_EQUAL_AFTER_ROUNDING; + } else { + data.identityResult = UNUM_IDENTITY_RESULT_NOT_EQUAL; + } + + switch (identity2d(fIdentityFallback, data.identityResult)) { + case identity2d(UNUM_IDENTITY_FALLBACK_RANGE, + UNUM_IDENTITY_RESULT_NOT_EQUAL): + case identity2d(UNUM_IDENTITY_FALLBACK_RANGE, + UNUM_IDENTITY_RESULT_EQUAL_AFTER_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_RANGE, + UNUM_IDENTITY_RESULT_EQUAL_BEFORE_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY, + UNUM_IDENTITY_RESULT_NOT_EQUAL): + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY_OR_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_NOT_EQUAL): + case identity2d(UNUM_IDENTITY_FALLBACK_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_NOT_EQUAL): + formatRange(data, micros1, micros2, status); + break; + + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY, + UNUM_IDENTITY_RESULT_EQUAL_AFTER_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY, + UNUM_IDENTITY_RESULT_EQUAL_BEFORE_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY_OR_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_EQUAL_AFTER_ROUNDING): + formatApproximately(data, micros1, micros2, status); + break; + + case identity2d(UNUM_IDENTITY_FALLBACK_APPROXIMATELY_OR_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_EQUAL_BEFORE_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_EQUAL_AFTER_ROUNDING): + case identity2d(UNUM_IDENTITY_FALLBACK_SINGLE_VALUE, + UNUM_IDENTITY_RESULT_EQUAL_BEFORE_ROUNDING): + formatSingleValue(data, micros1, micros2, status); + break; + + default: + UPRV_UNREACHABLE_EXIT; + } +} + + +void NumberRangeFormatterImpl::formatSingleValue(UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const { + if (U_FAILURE(status)) { return; } + if (fSameFormatters) { + int32_t length = NumberFormatterImpl::writeNumber(micros1.simple, data.quantity1, data.getStringRef(), 0, status); + NumberFormatterImpl::writeAffixes(micros1, data.getStringRef(), 0, length, status); + } else { + formatRange(data, micros1, micros2, status); + } +} + + +void NumberRangeFormatterImpl::formatApproximately (UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const { + if (U_FAILURE(status)) { return; } + if (fSameFormatters) { + // Re-format using the approximately formatter: + MicroProps microsAppx; + data.quantity1.resetExponent(); + fApproximatelyFormatter.preProcess(data.quantity1, microsAppx, status); + int32_t length = NumberFormatterImpl::writeNumber(microsAppx.simple, data.quantity1, data.getStringRef(), 0, status); + length += microsAppx.modInner->apply(data.getStringRef(), 0, length, status); + length += microsAppx.modMiddle->apply(data.getStringRef(), 0, length, status); + microsAppx.modOuter->apply(data.getStringRef(), 0, length, status); + } else { + formatRange(data, micros1, micros2, status); + } +} + + +void NumberRangeFormatterImpl::formatRange(UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const { + if (U_FAILURE(status)) { return; } + + // modInner is always notation (scientific); collapsable in ALL. + // modOuter is always units; collapsable in ALL, AUTO, and UNIT. + // modMiddle could be either; collapsable in ALL and sometimes AUTO and UNIT. + // Never collapse an outer mod but not an inner mod. + bool collapseOuter, collapseMiddle, collapseInner; + switch (fCollapse) { + case UNUM_RANGE_COLLAPSE_ALL: + case UNUM_RANGE_COLLAPSE_AUTO: + case UNUM_RANGE_COLLAPSE_UNIT: + { + // OUTER MODIFIER + collapseOuter = micros1.modOuter->semanticallyEquivalent(*micros2.modOuter); + + if (!collapseOuter) { + // Never collapse inner mods if outer mods are not collapsable + collapseMiddle = false; + collapseInner = false; + break; + } + + // MIDDLE MODIFIER + collapseMiddle = micros1.modMiddle->semanticallyEquivalent(*micros2.modMiddle); + + if (!collapseMiddle) { + // Never collapse inner mods if outer mods are not collapsable + collapseInner = false; + break; + } + + // MIDDLE MODIFIER HEURISTICS + // (could disable collapsing of the middle modifier) + // The modifiers are equal by this point, so we can look at just one of them. + const Modifier* mm = micros1.modMiddle; + if (fCollapse == UNUM_RANGE_COLLAPSE_UNIT) { + // Only collapse if the modifier is a unit. + // TODO: Make a better way to check for a unit? + // TODO: Handle case where the modifier has both notation and unit (compact currency)? + if (!mm->containsField({UFIELD_CATEGORY_NUMBER, UNUM_CURRENCY_FIELD}) + && !mm->containsField({UFIELD_CATEGORY_NUMBER, UNUM_PERCENT_FIELD})) { + collapseMiddle = false; + } + } else if (fCollapse == UNUM_RANGE_COLLAPSE_AUTO) { + // Heuristic as of ICU 63: collapse only if the modifier is more than one code point. + if (mm->getCodePointCount() <= 1) { + collapseMiddle = false; + } + } + + if (!collapseMiddle || fCollapse != UNUM_RANGE_COLLAPSE_ALL) { + collapseInner = false; + break; + } + + // INNER MODIFIER + collapseInner = micros1.modInner->semanticallyEquivalent(*micros2.modInner); + + // All done checking for collapsibility. + break; + } + + default: + collapseOuter = false; + collapseMiddle = false; + collapseInner = false; + break; + } + + FormattedStringBuilder& string = data.getStringRef(); + int32_t lengthPrefix = 0; + int32_t length1 = 0; + int32_t lengthInfix = 0; + int32_t length2 = 0; + int32_t lengthSuffix = 0; + + // Use #define so that these are evaluated at the call site. + #define UPRV_INDEX_0 (lengthPrefix) + #define UPRV_INDEX_1 (lengthPrefix + length1) + #define UPRV_INDEX_2 (lengthPrefix + length1 + lengthInfix) + #define UPRV_INDEX_3 (lengthPrefix + length1 + lengthInfix + length2) + #define UPRV_INDEX_4 (lengthPrefix + length1 + lengthInfix + length2 + lengthSuffix) + + int32_t lengthRange = SimpleModifier::formatTwoArgPattern( + fRangeFormatter, + string, + 0, + &lengthPrefix, + &lengthSuffix, + kUndefinedField, + status); + if (U_FAILURE(status)) { return; } + lengthInfix = lengthRange - lengthPrefix - lengthSuffix; + U_ASSERT(lengthInfix > 0); + + // SPACING HEURISTIC + // Add spacing unless all modifiers are collapsed. + // TODO: add API to control this? + // TODO: Use a data-driven heuristic like currency spacing? + // TODO: Use Unicode [:whitespace:] instead of PatternProps whitespace? (consider speed implications) + { + bool repeatInner = !collapseInner && micros1.modInner->getCodePointCount() > 0; + bool repeatMiddle = !collapseMiddle && micros1.modMiddle->getCodePointCount() > 0; + bool repeatOuter = !collapseOuter && micros1.modOuter->getCodePointCount() > 0; + if (repeatInner || repeatMiddle || repeatOuter) { + // Add spacing if there is not already spacing + if (!PatternProps::isWhiteSpace(string.charAt(UPRV_INDEX_1))) { + lengthInfix += string.insertCodePoint(UPRV_INDEX_1, u'\u0020', kUndefinedField, status); + } + if (!PatternProps::isWhiteSpace(string.charAt(UPRV_INDEX_2 - 1))) { + lengthInfix += string.insertCodePoint(UPRV_INDEX_2, u'\u0020', kUndefinedField, status); + } + } + } + + length1 += NumberFormatterImpl::writeNumber(micros1.simple, data.quantity1, string, UPRV_INDEX_0, status); + // ICU-21684: Write the second number to a temp string to avoid repeated insert operations + FormattedStringBuilder tempString; + NumberFormatterImpl::writeNumber(micros2.simple, data.quantity2, tempString, 0, status); + length2 += string.insert(UPRV_INDEX_2, tempString, status); + + // TODO: Support padding? + + if (collapseInner) { + const Modifier& mod = resolveModifierPlurals(*micros1.modInner, *micros2.modInner); + lengthSuffix += mod.apply(string, UPRV_INDEX_0, UPRV_INDEX_4, status); + lengthPrefix += mod.getPrefixLength(); + lengthSuffix -= mod.getPrefixLength(); + } else { + length1 += micros1.modInner->apply(string, UPRV_INDEX_0, UPRV_INDEX_1, status); + length2 += micros2.modInner->apply(string, UPRV_INDEX_2, UPRV_INDEX_4, status); + } + + if (collapseMiddle) { + const Modifier& mod = resolveModifierPlurals(*micros1.modMiddle, *micros2.modMiddle); + lengthSuffix += mod.apply(string, UPRV_INDEX_0, UPRV_INDEX_4, status); + lengthPrefix += mod.getPrefixLength(); + lengthSuffix -= mod.getPrefixLength(); + } else { + length1 += micros1.modMiddle->apply(string, UPRV_INDEX_0, UPRV_INDEX_1, status); + length2 += micros2.modMiddle->apply(string, UPRV_INDEX_2, UPRV_INDEX_4, status); + } + + if (collapseOuter) { + const Modifier& mod = resolveModifierPlurals(*micros1.modOuter, *micros2.modOuter); + lengthSuffix += mod.apply(string, UPRV_INDEX_0, UPRV_INDEX_4, status); + lengthPrefix += mod.getPrefixLength(); + lengthSuffix -= mod.getPrefixLength(); + } else { + length1 += micros1.modOuter->apply(string, UPRV_INDEX_0, UPRV_INDEX_1, status); + length2 += micros2.modOuter->apply(string, UPRV_INDEX_2, UPRV_INDEX_4, status); + } + + // Now that all pieces are added, save the span info. + data.appendSpanInfo(UFIELD_CATEGORY_NUMBER_RANGE_SPAN, 0, UPRV_INDEX_0, length1, status); + data.appendSpanInfo(UFIELD_CATEGORY_NUMBER_RANGE_SPAN, 1, UPRV_INDEX_2, length2, status); +} + + +const Modifier& +NumberRangeFormatterImpl::resolveModifierPlurals(const Modifier& first, const Modifier& second) const { + Modifier::Parameters parameters; + first.getParameters(parameters); + if (parameters.obj == nullptr) { + // No plural form; return a fallback (e.g., the first) + return first; + } + StandardPlural::Form firstPlural = parameters.plural; + + second.getParameters(parameters); + if (parameters.obj == nullptr) { + // No plural form; return a fallback (e.g., the first) + return first; + } + StandardPlural::Form secondPlural = parameters.plural; + + // Get the required plural form from data + StandardPlural::Form resultPlural = fPluralRanges.resolve(firstPlural, secondPlural); + + // Get and return the new Modifier + const Modifier* mod = parameters.obj->getModifier(parameters.signum, resultPlural); + U_ASSERT(mod != nullptr); + return *mod; +} + + + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numrange_impl.h b/intl/icu/source/i18n/numrange_impl.h new file mode 100644 index 0000000000..ac1d8a5897 --- /dev/null +++ b/intl/icu/source/i18n/numrange_impl.h @@ -0,0 +1,89 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING +#ifndef __SOURCE_NUMRANGE_TYPES_H__ +#define __SOURCE_NUMRANGE_TYPES_H__ + +#include "unicode/numberformatter.h" +#include "unicode/numberrangeformatter.h" +#include "unicode/simpleformatter.h" +#include "number_types.h" +#include "number_decimalquantity.h" +#include "number_formatimpl.h" +#include "formatted_string_builder.h" +#include "formattedval_impl.h" +#include "pluralranges.h" + +U_NAMESPACE_BEGIN namespace number { +namespace impl { + + +/** + * Class similar to UFormattedNumberData. + * + * Has incomplete magic number logic that will need to be finished + * if this is to be exposed as C API in the future. + * + * Possible magic number: 0x46445200 + * Reads in ASCII as "FDR" (FormatteDnumberRange with room at the end) + */ +class UFormattedNumberRangeData : public FormattedValueStringBuilderImpl { +public: + UFormattedNumberRangeData() : FormattedValueStringBuilderImpl(kUndefinedField) {} + virtual ~UFormattedNumberRangeData(); + + DecimalQuantity quantity1; + DecimalQuantity quantity2; + UNumberRangeIdentityResult identityResult = UNUM_IDENTITY_RESULT_COUNT; +}; + + +class NumberRangeFormatterImpl : public UMemory { + public: + NumberRangeFormatterImpl(const RangeMacroProps& macros, UErrorCode& status); + + void format(UFormattedNumberRangeData& data, bool equalBeforeRounding, UErrorCode& status) const; + + private: + NumberFormatterImpl formatterImpl1; + NumberFormatterImpl formatterImpl2; + bool fSameFormatters; + + UNumberRangeCollapse fCollapse; + UNumberRangeIdentityFallback fIdentityFallback; + + SimpleFormatter fRangeFormatter; + NumberFormatterImpl fApproximatelyFormatter; + + StandardPluralRanges fPluralRanges; + + void formatSingleValue(UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const; + + void formatApproximately(UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const; + + void formatRange(UFormattedNumberRangeData& data, + MicroProps& micros1, MicroProps& micros2, + UErrorCode& status) const; + + const Modifier& resolveModifierPlurals(const Modifier& first, const Modifier& second) const; +}; + + +/** Helper function used in upluralrules.cpp */ +const UFormattedNumberRangeData* validateUFormattedNumberRange( + const UFormattedNumberRange* uresult, UErrorCode& status); + + +} // namespace impl +} // namespace number +U_NAMESPACE_END + +#endif //__SOURCE_NUMRANGE_TYPES_H__ +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/numsys.cpp b/intl/icu/source/i18n/numsys.cpp new file mode 100644 index 0000000000..0d5c43e4b2 --- /dev/null +++ b/intl/icu/source/i18n/numsys.cpp @@ -0,0 +1,362 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2010-2015, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* +* File NUMSYS.CPP +* +* Modification History:* +* Date Name Description +* +******************************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/localpointer.h" +#include "unicode/uchar.h" +#include "unicode/unistr.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "unicode/uloc.h" +#include "unicode/schriter.h" +#include "unicode/numsys.h" +#include "cstring.h" +#include "uassert.h" +#include "ucln_in.h" +#include "umutex.h" +#include "uresimp.h" +#include "numsys_impl.h" + +#if !UCONFIG_NO_FORMATTING + +U_NAMESPACE_BEGIN + +// Useful constants + +#define DEFAULT_DIGITS UNICODE_STRING_SIMPLE("0123456789") +static const char gNumberingSystems[] = "numberingSystems"; +static const char gNumberElements[] = "NumberElements"; +static const char gDefault[] = "default"; +static const char gNative[] = "native"; +static const char gTraditional[] = "traditional"; +static const char gFinance[] = "finance"; +static const char gDesc[] = "desc"; +static const char gRadix[] = "radix"; +static const char gAlgorithmic[] = "algorithmic"; +static const char gLatn[] = "latn"; + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NumberingSystem) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NumsysNameEnumeration) + + /** + * Default Constructor. + * + * @draft ICU 4.2 + */ + +NumberingSystem::NumberingSystem() { + radix = 10; + algorithmic = false; + UnicodeString defaultDigits = DEFAULT_DIGITS; + desc.setTo(defaultDigits); + uprv_strcpy(name,gLatn); +} + + /** + * Copy constructor. + * @draft ICU 4.2 + */ + +NumberingSystem::NumberingSystem(const NumberingSystem& other) +: UObject(other) { + *this=other; +} + +NumberingSystem* U_EXPORT2 +NumberingSystem::createInstance(int32_t radix_in, UBool isAlgorithmic_in, const UnicodeString & desc_in, UErrorCode &status) { + + if (U_FAILURE(status)) { + return nullptr; + } + + if ( radix_in < 2 ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + if ( !isAlgorithmic_in ) { + if ( desc_in.countChar32() != radix_in ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + } + + LocalPointer ns(new NumberingSystem(), status); + if (U_FAILURE(status)) { + return nullptr; + } + + ns->setRadix(radix_in); + ns->setDesc(desc_in); + ns->setAlgorithmic(isAlgorithmic_in); + ns->setName(nullptr); + + return ns.orphan(); +} + +NumberingSystem* U_EXPORT2 +NumberingSystem::createInstance(const Locale & inLocale, UErrorCode& status) { + + if (U_FAILURE(status)) { + return nullptr; + } + + UBool nsResolved = true; + UBool usingFallback = false; + char buffer[ULOC_KEYWORDS_CAPACITY] = ""; + int32_t count = inLocale.getKeywordValue("numbers", buffer, sizeof(buffer), status); + if (U_FAILURE(status) || status == U_STRING_NOT_TERMINATED_WARNING) { + // the "numbers" keyword exceeds ULOC_KEYWORDS_CAPACITY; ignore and use default. + count = 0; + status = U_ZERO_ERROR; + } + if ( count > 0 ) { // @numbers keyword was specified in the locale + U_ASSERT(count < ULOC_KEYWORDS_CAPACITY); + buffer[count] = '\0'; // Make sure it is null terminated. + if ( !uprv_strcmp(buffer,gDefault) || !uprv_strcmp(buffer,gNative) || + !uprv_strcmp(buffer,gTraditional) || !uprv_strcmp(buffer,gFinance)) { + nsResolved = false; + } + } else { + uprv_strcpy(buffer, gDefault); + nsResolved = false; + } + + if (!nsResolved) { // Resolve the numbering system ( default, native, traditional or finance ) into a "real" numbering system + UErrorCode localStatus = U_ZERO_ERROR; + LocalUResourceBundlePointer resource(ures_open(nullptr, inLocale.getName(), &localStatus)); + LocalUResourceBundlePointer numberElementsRes(ures_getByKey(resource.getAlias(), gNumberElements, nullptr, &localStatus)); + // Don't stomp on the catastrophic failure of OOM. + if (localStatus == U_MEMORY_ALLOCATION_ERROR) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + while (!nsResolved) { + localStatus = U_ZERO_ERROR; + count = 0; + const char16_t *nsName = ures_getStringByKeyWithFallback(numberElementsRes.getAlias(), buffer, &count, &localStatus); + // Don't stomp on the catastrophic failure of OOM. + if (localStatus == U_MEMORY_ALLOCATION_ERROR) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if ( count > 0 && count < ULOC_KEYWORDS_CAPACITY ) { // numbering system found + u_UCharsToChars(nsName, buffer, count); + buffer[count] = '\0'; // Make sure it is null terminated. + nsResolved = true; + } + + if (!nsResolved) { // Fallback behavior per TR35 - traditional falls back to native, finance and native fall back to default + if (!uprv_strcmp(buffer,gNative) || !uprv_strcmp(buffer,gFinance)) { + uprv_strcpy(buffer,gDefault); + } else if (!uprv_strcmp(buffer,gTraditional)) { + uprv_strcpy(buffer,gNative); + } else { // If we get here we couldn't find even the default numbering system + usingFallback = true; + nsResolved = true; + } + } + } + } + + if (usingFallback) { + status = U_USING_FALLBACK_WARNING; + NumberingSystem *ns = new NumberingSystem(); + if (ns == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return ns; + } else { + return NumberingSystem::createInstanceByName(buffer, status); + } + } + +NumberingSystem* U_EXPORT2 +NumberingSystem::createInstance(UErrorCode& status) { + return NumberingSystem::createInstance(Locale::getDefault(), status); +} + +NumberingSystem* U_EXPORT2 +NumberingSystem::createInstanceByName(const char *name, UErrorCode& status) { + int32_t radix = 10; + int32_t algorithmic = 0; + + LocalUResourceBundlePointer numberingSystemsInfo(ures_openDirect(nullptr, gNumberingSystems, &status)); + LocalUResourceBundlePointer nsCurrent(ures_getByKey(numberingSystemsInfo.getAlias(), gNumberingSystems, nullptr, &status)); + LocalUResourceBundlePointer nsTop(ures_getByKey(nsCurrent.getAlias(), name, nullptr, &status)); + + UnicodeString nsd = ures_getUnicodeStringByKey(nsTop.getAlias(), gDesc, &status); + + ures_getByKey(nsTop.getAlias(), gRadix, nsCurrent.getAlias(), &status); + radix = ures_getInt(nsCurrent.getAlias(), &status); + + ures_getByKey(nsTop.getAlias(), gAlgorithmic, nsCurrent.getAlias(), &status); + algorithmic = ures_getInt(nsCurrent.getAlias(), &status); + + UBool isAlgorithmic = ( algorithmic == 1 ); + + if (U_FAILURE(status)) { + // Don't stomp on the catastrophic failure of OOM. + if (status != U_MEMORY_ALLOCATION_ERROR) { + status = U_UNSUPPORTED_ERROR; + } + return nullptr; + } + + LocalPointer ns(NumberingSystem::createInstance(radix, isAlgorithmic, nsd, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + ns->setName(name); + return ns.orphan(); +} + + /** + * Destructor. + * @draft ICU 4.2 + */ +NumberingSystem::~NumberingSystem() { +} + +int32_t NumberingSystem::getRadix() const { + return radix; +} + +UnicodeString NumberingSystem::getDescription() const { + return desc; +} + +const char * NumberingSystem::getName() const { + return name; +} + +void NumberingSystem::setRadix(int32_t r) { + radix = r; +} + +void NumberingSystem::setAlgorithmic(UBool c) { + algorithmic = c; +} + +void NumberingSystem::setDesc(const UnicodeString &d) { + desc.setTo(d); +} +void NumberingSystem::setName(const char *n) { + if ( n == nullptr ) { + name[0] = (char) 0; + } else { + uprv_strncpy(name,n,kInternalNumSysNameCapacity); + name[kInternalNumSysNameCapacity] = '\0'; // Make sure it is null terminated. + } +} +UBool NumberingSystem::isAlgorithmic() const { + return ( algorithmic ); +} + +namespace { + +UVector* gNumsysNames = nullptr; +UInitOnce gNumSysInitOnce {}; + +U_CFUNC UBool U_CALLCONV numSysCleanup() { + delete gNumsysNames; + gNumsysNames = nullptr; + gNumSysInitOnce.reset(); + return true; +} + +U_CFUNC void initNumsysNames(UErrorCode &status) { + U_ASSERT(gNumsysNames == nullptr); + ucln_i18n_registerCleanup(UCLN_I18N_NUMSYS, numSysCleanup); + + // TODO: Simple array of UnicodeString objects, based on length of table resource? + LocalPointer numsysNames(new UVector(uprv_deleteUObject, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + + UErrorCode rbstatus = U_ZERO_ERROR; + UResourceBundle *numberingSystemsInfo = ures_openDirect(nullptr, "numberingSystems", &rbstatus); + numberingSystemsInfo = + ures_getByKey(numberingSystemsInfo, "numberingSystems", numberingSystemsInfo, &rbstatus); + if (U_FAILURE(rbstatus)) { + // Don't stomp on the catastrophic failure of OOM. + if (rbstatus == U_MEMORY_ALLOCATION_ERROR) { + status = rbstatus; + } else { + status = U_MISSING_RESOURCE_ERROR; + } + ures_close(numberingSystemsInfo); + return; + } + + while ( ures_hasNext(numberingSystemsInfo) && U_SUCCESS(status) ) { + LocalUResourceBundlePointer nsCurrent(ures_getNextResource(numberingSystemsInfo, nullptr, &rbstatus)); + if (rbstatus == U_MEMORY_ALLOCATION_ERROR) { + status = rbstatus; // we want to report OOM failure back to the caller. + break; + } + const char *nsName = ures_getKey(nsCurrent.getAlias()); + LocalPointer newElem(new UnicodeString(nsName, -1, US_INV), status); + numsysNames->adoptElement(newElem.orphan(), status); + } + + ures_close(numberingSystemsInfo); + if (U_SUCCESS(status)) { + gNumsysNames = numsysNames.orphan(); + } + return; +} + +} // end anonymous namespace + +StringEnumeration* NumberingSystem::getAvailableNames(UErrorCode &status) { + umtx_initOnce(gNumSysInitOnce, &initNumsysNames, status); + LocalPointer result(new NumsysNameEnumeration(status), status); + return result.orphan(); +} + +NumsysNameEnumeration::NumsysNameEnumeration(UErrorCode& status) : pos(0) { + (void)status; +} + +const UnicodeString* +NumsysNameEnumeration::snext(UErrorCode& status) { + if (U_SUCCESS(status) && (gNumsysNames != nullptr) && (pos < gNumsysNames->size())) { + return (const UnicodeString*)gNumsysNames->elementAt(pos++); + } + return nullptr; +} + +void +NumsysNameEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +NumsysNameEnumeration::count(UErrorCode& /*status*/) const { + return (gNumsysNames==nullptr) ? 0 : gNumsysNames->size(); +} + +NumsysNameEnumeration::~NumsysNameEnumeration() { +} +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/numsys_impl.h b/intl/icu/source/i18n/numsys_impl.h new file mode 100644 index 0000000000..42cf102a13 --- /dev/null +++ b/intl/icu/source/i18n/numsys_impl.h @@ -0,0 +1,45 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2015, International Business Machines Corporation and +* others. All Rights Reserved. * +******************************************************************************* +* +* File NUMSYS_IMPL.H +* +******************************************************************************* +*/ + +#ifndef __NUMSYS_IMPL_H__ +#define __NUMSYS_IMPL_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/numsys.h" +#include "uvector.h" +#include "unicode/strenum.h" + +U_NAMESPACE_BEGIN + +class NumsysNameEnumeration : public StringEnumeration { +public: + NumsysNameEnumeration(UErrorCode& status); + + virtual ~NumsysNameEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; +private: + int32_t pos; +}; + +U_NAMESPACE_END + +#endif + +#endif diff --git a/intl/icu/source/i18n/olsontz.cpp b/intl/icu/source/i18n/olsontz.cpp new file mode 100644 index 0000000000..260e345d79 --- /dev/null +++ b/intl/icu/source/i18n/olsontz.cpp @@ -0,0 +1,1082 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2003-2013, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: July 21 2003 +* Since: ICU 2.8 +********************************************************************** +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "olsontz.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/ures.h" +#include "unicode/simpletz.h" +#include "unicode/gregocal.h" +#include "gregoimp.h" +#include "cmemory.h" +#include "uassert.h" +#include "uvector.h" +#include // DBL_MAX +#include "uresimp.h" +#include "zonemeta.h" +#include "umutex.h" + +#ifdef U_DEBUG_TZ +# include +# include "uresimp.h" // for debugging + +static void debug_tz_loc(const char *f, int32_t l) +{ + fprintf(stderr, "%s:%d: ", f, l); +} + +static void debug_tz_msg(const char *pat, ...) +{ + va_list ap; + va_start(ap, pat); + vfprintf(stderr, pat, ap); + fflush(stderr); +} +// must use double parens, i.e.: U_DEBUG_TZ_MSG(("four is: %d",4)); +#define U_DEBUG_TZ_MSG(x) {debug_tz_loc(__FILE__,__LINE__);debug_tz_msg x;} +#else +#define U_DEBUG_TZ_MSG(x) +#endif + +static UBool arrayEqual(const void *a1, const void *a2, int32_t size) { + if (a1 == nullptr && a2 == nullptr) { + return true; + } + if ((a1 != nullptr && a2 == nullptr) || (a1 == nullptr && a2 != nullptr)) { + return false; + } + if (a1 == a2) { + return true; + } + + return (uprv_memcmp(a1, a2, size) == 0); +} + +U_NAMESPACE_BEGIN + +#define kTRANS "trans" +#define kTRANSPRE32 "transPre32" +#define kTRANSPOST32 "transPost32" +#define kTYPEOFFSETS "typeOffsets" +#define kTYPEMAP "typeMap" +#define kLINKS "links" +#define kFINALRULE "finalRule" +#define kFINALRAW "finalRaw" +#define kFINALYEAR "finalYear" + +#define SECONDS_PER_DAY (24*60*60) + +static const int32_t ZEROS[] = {0,0}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(OlsonTimeZone) + +/** + * Default constructor. Creates a time zone with an empty ID and + * a fixed GMT offset of zero. + */ +/*OlsonTimeZone::OlsonTimeZone() : finalYear(INT32_MAX), finalMillis(DBL_MAX), finalZone(0), transitionRulesInitialized(false) { + clearTransitionRules(); + constructEmpty(); +}*/ + +/** + * Construct a GMT+0 zone with no transitions. This is done when a + * constructor fails so the resultant object is well-behaved. + */ +void OlsonTimeZone::constructEmpty() { + canonicalID = nullptr; + + transitionCountPre32 = transitionCount32 = transitionCountPost32 = 0; + transitionTimesPre32 = transitionTimes32 = transitionTimesPost32 = nullptr; + + typeMapData = nullptr; + + typeCount = 1; + typeOffsets = ZEROS; + + finalZone = nullptr; +} + +/** + * Construct from a resource bundle + * @param top the top-level zoneinfo resource bundle. This is used + * to lookup the rule that `res' may refer to, if there is one. + * @param res the resource bundle of the zone to be constructed + * @param ec input-output error code + */ +OlsonTimeZone::OlsonTimeZone(const UResourceBundle* top, + const UResourceBundle* res, + const UnicodeString& tzid, + UErrorCode& ec) : + BasicTimeZone(tzid), finalZone(nullptr) +{ + clearTransitionRules(); + U_DEBUG_TZ_MSG(("OlsonTimeZone(%s)\n", ures_getKey((UResourceBundle*)res))); + if ((top == nullptr || res == nullptr) && U_SUCCESS(ec)) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + } + if (U_SUCCESS(ec)) { + // TODO -- clean up -- Doesn't work if res points to an alias + // // TODO remove nonconst casts below when ures_* API is fixed + // setID(ures_getKey((UResourceBundle*) res)); // cast away const + + int32_t len; + StackUResourceBundle r; + + // Pre-32bit second transitions + ures_getByKey(res, kTRANSPRE32, r.getAlias(), &ec); + transitionTimesPre32 = ures_getIntVector(r.getAlias(), &len, &ec); + transitionCountPre32 = static_cast(len >> 1); + if (ec == U_MISSING_RESOURCE_ERROR) { + // No pre-32bit transitions + transitionTimesPre32 = nullptr; + transitionCountPre32 = 0; + ec = U_ZERO_ERROR; + } else if (U_SUCCESS(ec) && (len < 0 || len > 0x7FFF || (len & 1) != 0) /* len must be even */) { + ec = U_INVALID_FORMAT_ERROR; + } + + // 32bit second transitions + ures_getByKey(res, kTRANS, r.getAlias(), &ec); + transitionTimes32 = ures_getIntVector(r.getAlias(), &len, &ec); + transitionCount32 = static_cast(len); + if (ec == U_MISSING_RESOURCE_ERROR) { + // No 32bit transitions + transitionTimes32 = nullptr; + transitionCount32 = 0; + ec = U_ZERO_ERROR; + } else if (U_SUCCESS(ec) && (len < 0 || len > 0x7FFF)) { + ec = U_INVALID_FORMAT_ERROR; + } + + // Post-32bit second transitions + ures_getByKey(res, kTRANSPOST32, r.getAlias(), &ec); + transitionTimesPost32 = ures_getIntVector(r.getAlias(), &len, &ec); + transitionCountPost32 = static_cast(len >> 1); + if (ec == U_MISSING_RESOURCE_ERROR) { + // No pre-32bit transitions + transitionTimesPost32 = nullptr; + transitionCountPost32 = 0; + ec = U_ZERO_ERROR; + } else if (U_SUCCESS(ec) && (len < 0 || len > 0x7FFF || (len & 1) != 0) /* len must be even */) { + ec = U_INVALID_FORMAT_ERROR; + } + + // Type offsets list must be of even size, with size >= 2 + ures_getByKey(res, kTYPEOFFSETS, r.getAlias(), &ec); + typeOffsets = ures_getIntVector(r.getAlias(), &len, &ec); + if (U_SUCCESS(ec) && (len < 2 || len > 0x7FFE || (len & 1) != 0)) { + ec = U_INVALID_FORMAT_ERROR; + } + typeCount = (int16_t) len >> 1; + + // Type map data must be of the same size as the transition count + typeMapData = nullptr; + if (transitionCount() > 0) { + ures_getByKey(res, kTYPEMAP, r.getAlias(), &ec); + typeMapData = ures_getBinary(r.getAlias(), &len, &ec); + if (ec == U_MISSING_RESOURCE_ERROR) { + // no type mapping data + ec = U_INVALID_FORMAT_ERROR; + } else if (U_SUCCESS(ec) && len != transitionCount()) { + ec = U_INVALID_FORMAT_ERROR; + } + } + + // Process final rule and data, if any + if (U_SUCCESS(ec)) { + const char16_t *ruleIdUStr = ures_getStringByKey(res, kFINALRULE, &len, &ec); + ures_getByKey(res, kFINALRAW, r.getAlias(), &ec); + int32_t ruleRaw = ures_getInt(r.getAlias(), &ec); + ures_getByKey(res, kFINALYEAR, r.getAlias(), &ec); + int32_t ruleYear = ures_getInt(r.getAlias(), &ec); + if (U_SUCCESS(ec)) { + UnicodeString ruleID(true, ruleIdUStr, len); + UResourceBundle *rule = TimeZone::loadRule(top, ruleID, nullptr, ec); + const int32_t *ruleData = ures_getIntVector(rule, &len, &ec); + if (U_SUCCESS(ec) && len == 11) { + UnicodeString emptyStr; + finalZone = new SimpleTimeZone( + ruleRaw * U_MILLIS_PER_SECOND, + emptyStr, + (int8_t)ruleData[0], (int8_t)ruleData[1], (int8_t)ruleData[2], + ruleData[3] * U_MILLIS_PER_SECOND, + (SimpleTimeZone::TimeMode) ruleData[4], + (int8_t)ruleData[5], (int8_t)ruleData[6], (int8_t)ruleData[7], + ruleData[8] * U_MILLIS_PER_SECOND, + (SimpleTimeZone::TimeMode) ruleData[9], + ruleData[10] * U_MILLIS_PER_SECOND, ec); + if (finalZone == nullptr) { + ec = U_MEMORY_ALLOCATION_ERROR; + } else { + finalStartYear = ruleYear; + + // Note: Setting finalStartYear to the finalZone is problematic. When a date is around + // year boundary, SimpleTimeZone may return false result when DST is observed at the + // beginning of year. We could apply safe margin (day or two), but when one of recurrent + // rules falls around year boundary, it could return false result. Without setting the + // start year, finalZone works fine around the year boundary of the start year. + + // finalZone->setStartYear(finalStartYear); + + + // Compute the millis for Jan 1, 0:00 GMT of the finalYear + + // Note: finalStartMillis is used for detecting either if + // historic transition data or finalZone to be used. In an + // extreme edge case - for example, two transitions fall into + // small windows of time around the year boundary, this may + // result incorrect offset computation. But I think it will + // never happen practically. Yoshito - Feb 20, 2010 + finalStartMillis = Grego::fieldsToDay(finalStartYear, 0, 1) * U_MILLIS_PER_DAY; + } + } else { + ec = U_INVALID_FORMAT_ERROR; + } + ures_close(rule); + } else if (ec == U_MISSING_RESOURCE_ERROR) { + // No final zone + ec = U_ZERO_ERROR; + } + } + + // initialize canonical ID + canonicalID = ZoneMeta::getCanonicalCLDRID(tzid, ec); + } + + if (U_FAILURE(ec)) { + constructEmpty(); + } +} + +/** + * Copy constructor + */ +OlsonTimeZone::OlsonTimeZone(const OlsonTimeZone& other) : + BasicTimeZone(other), finalZone(0) { + *this = other; +} + +/** + * Assignment operator + */ +OlsonTimeZone& OlsonTimeZone::operator=(const OlsonTimeZone& other) { + if (this == &other) { return *this; } // self-assignment: no-op + canonicalID = other.canonicalID; + + transitionTimesPre32 = other.transitionTimesPre32; + transitionTimes32 = other.transitionTimes32; + transitionTimesPost32 = other.transitionTimesPost32; + + transitionCountPre32 = other.transitionCountPre32; + transitionCount32 = other.transitionCount32; + transitionCountPost32 = other.transitionCountPost32; + + typeCount = other.typeCount; + typeOffsets = other.typeOffsets; + typeMapData = other.typeMapData; + + delete finalZone; + finalZone = (other.finalZone != 0) ? other.finalZone->clone() : 0; + + finalStartYear = other.finalStartYear; + finalStartMillis = other.finalStartMillis; + + clearTransitionRules(); + + return *this; +} + +/** + * Destructor + */ +OlsonTimeZone::~OlsonTimeZone() { + deleteTransitionRules(); + delete finalZone; +} + +/** + * Returns true if the two TimeZone objects are equal. + */ +bool OlsonTimeZone::operator==(const TimeZone& other) const { + return ((this == &other) || + (typeid(*this) == typeid(other) && + TimeZone::operator==(other) && + hasSameRules(other))); +} + +/** + * TimeZone API. + */ +OlsonTimeZone* OlsonTimeZone::clone() const { + return new OlsonTimeZone(*this); +} + +/** + * TimeZone API. + */ +int32_t OlsonTimeZone::getOffset(uint8_t era, int32_t year, int32_t month, + int32_t dom, uint8_t dow, + int32_t millis, UErrorCode& ec) const { + if (month < UCAL_JANUARY || month > UCAL_DECEMBER) { + if (U_SUCCESS(ec)) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + } + return 0; + } else { + return getOffset(era, year, month, dom, dow, millis, + Grego::monthLength(year, month), + ec); + } +} + +/** + * TimeZone API. + */ +int32_t OlsonTimeZone::getOffset(uint8_t era, int32_t year, int32_t month, + int32_t dom, uint8_t dow, + int32_t millis, int32_t monthLength, + UErrorCode& ec) const { + if (U_FAILURE(ec)) { + return 0; + } + + if ((era != GregorianCalendar::AD && era != GregorianCalendar::BC) + || month < UCAL_JANUARY + || month > UCAL_DECEMBER + || dom < 1 + || dom > monthLength + || dow < UCAL_SUNDAY + || dow > UCAL_SATURDAY + || millis < 0 + || millis >= U_MILLIS_PER_DAY + || monthLength < 28 + || monthLength > 31) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if (era == GregorianCalendar::BC) { + year = -year; + } + + if (finalZone != nullptr && year >= finalStartYear) { + return finalZone->getOffset(era, year, month, dom, dow, + millis, monthLength, ec); + } + + // Compute local epoch millis from input fields + UDate date = (UDate)(Grego::fieldsToDay(year, month, dom) * U_MILLIS_PER_DAY + millis); + int32_t rawoff, dstoff; + getHistoricalOffset(date, true, kDaylight, kStandard, rawoff, dstoff); + return rawoff + dstoff; +} + +/** + * TimeZone API. + */ +void OlsonTimeZone::getOffset(UDate date, UBool local, int32_t& rawoff, + int32_t& dstoff, UErrorCode& ec) const { + if (U_FAILURE(ec)) { + return; + } + if (finalZone != nullptr && date >= finalStartMillis) { + finalZone->getOffset(date, local, rawoff, dstoff, ec); + } else { + getHistoricalOffset(date, local, kFormer, kLatter, rawoff, dstoff); + } +} + +void OlsonTimeZone::getOffsetFromLocal(UDate date, UTimeZoneLocalOption nonExistingTimeOpt, + UTimeZoneLocalOption duplicatedTimeOpt, + int32_t& rawoff, int32_t& dstoff, UErrorCode& ec) const { + if (U_FAILURE(ec)) { + return; + } + if (finalZone != nullptr && date >= finalStartMillis) { + finalZone->getOffsetFromLocal(date, nonExistingTimeOpt, duplicatedTimeOpt, rawoff, dstoff, ec); + } else { + getHistoricalOffset(date, true, nonExistingTimeOpt, duplicatedTimeOpt, rawoff, dstoff); + } +} + + +/** + * TimeZone API. + */ +void OlsonTimeZone::setRawOffset(int32_t /*offsetMillis*/) { + // We don't support this operation, since OlsonTimeZones are + // immutable (except for the ID, which is in the base class). + + // Nothing to do! +} + +/** + * TimeZone API. + */ +int32_t OlsonTimeZone::getRawOffset() const { + UErrorCode ec = U_ZERO_ERROR; + int32_t raw, dst; + getOffset(uprv_getUTCtime(), false, raw, dst, ec); + return raw; +} + +#if defined U_DEBUG_TZ +void printTime(double ms) { + int32_t year, month, dom, dow; + double millis=0; + double days = ClockMath::floorDivide(((double)ms), (double)U_MILLIS_PER_DAY, millis); + + Grego::dayToFields(days, year, month, dom, dow); + U_DEBUG_TZ_MSG((" getHistoricalOffset: time %.1f (%04d.%02d.%02d+%.1fh)\n", ms, + year, month+1, dom, (millis/kOneHour))); + } +#endif + +int64_t +OlsonTimeZone::transitionTimeInSeconds(int16_t transIdx) const { + U_ASSERT(transIdx >= 0 && transIdx < transitionCount()); + + if (transIdx < transitionCountPre32) { + return (((int64_t)((uint32_t)transitionTimesPre32[transIdx << 1])) << 32) + | ((int64_t)((uint32_t)transitionTimesPre32[(transIdx << 1) + 1])); + } + + transIdx -= transitionCountPre32; + if (transIdx < transitionCount32) { + return (int64_t)transitionTimes32[transIdx]; + } + + transIdx -= transitionCount32; + return (((int64_t)((uint32_t)transitionTimesPost32[transIdx << 1])) << 32) + | ((int64_t)((uint32_t)transitionTimesPost32[(transIdx << 1) + 1])); +} + +// Maximum absolute offset in seconds (86400 seconds = 1 day) +// getHistoricalOffset uses this constant as safety margin of +// quick zone transition checking. +#define MAX_OFFSET_SECONDS 86400 + +void +OlsonTimeZone::getHistoricalOffset(UDate date, UBool local, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt, + int32_t& rawoff, int32_t& dstoff) const { + U_DEBUG_TZ_MSG(("getHistoricalOffset(%.1f, %s, %d, %d, raw, dst)\n", + date, local?"T":"F", NonExistingTimeOpt, DuplicatedTimeOpt)); +#if defined U_DEBUG_TZ + printTime(date*1000.0); +#endif + int16_t transCount = transitionCount(); + + if (transCount > 0) { + double sec = uprv_floor(date / U_MILLIS_PER_SECOND); + if (!local && sec < transitionTimeInSeconds(0)) { + // Before the first transition time + rawoff = initialRawOffset() * U_MILLIS_PER_SECOND; + dstoff = initialDstOffset() * U_MILLIS_PER_SECOND; + } else { + // Linear search from the end is the fastest approach, since + // most lookups will happen at/near the end. + int16_t transIdx; + for (transIdx = transCount - 1; transIdx >= 0; transIdx--) { + int64_t transition = transitionTimeInSeconds(transIdx); + + if (local && (sec >= (transition - MAX_OFFSET_SECONDS))) { + int32_t offsetBefore = zoneOffsetAt(transIdx - 1); + UBool dstBefore = dstOffsetAt(transIdx - 1) != 0; + + int32_t offsetAfter = zoneOffsetAt(transIdx); + UBool dstAfter = dstOffsetAt(transIdx) != 0; + + UBool dstToStd = dstBefore && !dstAfter; + UBool stdToDst = !dstBefore && dstAfter; + + if (offsetAfter - offsetBefore >= 0) { + // Positive transition, which makes a non-existing local time range + if (((NonExistingTimeOpt & kStdDstMask) == kStandard && dstToStd) + || ((NonExistingTimeOpt & kStdDstMask) == kDaylight && stdToDst)) { + transition += offsetBefore; + } else if (((NonExistingTimeOpt & kStdDstMask) == kStandard && stdToDst) + || ((NonExistingTimeOpt & kStdDstMask) == kDaylight && dstToStd)) { + transition += offsetAfter; + } else if ((NonExistingTimeOpt & kFormerLatterMask) == kLatter) { + transition += offsetBefore; + } else { + // Interprets the time with rule before the transition, + // default for non-existing time range + transition += offsetAfter; + } + } else { + // Negative transition, which makes a duplicated local time range + if (((DuplicatedTimeOpt & kStdDstMask) == kStandard && dstToStd) + || ((DuplicatedTimeOpt & kStdDstMask) == kDaylight && stdToDst)) { + transition += offsetAfter; + } else if (((DuplicatedTimeOpt & kStdDstMask) == kStandard && stdToDst) + || ((DuplicatedTimeOpt & kStdDstMask) == kDaylight && dstToStd)) { + transition += offsetBefore; + } else if ((DuplicatedTimeOpt & kFormerLatterMask) == kFormer) { + transition += offsetBefore; + } else { + // Interprets the time with rule after the transition, + // default for duplicated local time range + transition += offsetAfter; + } + } + } + if (sec >= transition) { + break; + } + } + // transIdx could be -1 when local=true + rawoff = rawOffsetAt(transIdx) * U_MILLIS_PER_SECOND; + dstoff = dstOffsetAt(transIdx) * U_MILLIS_PER_SECOND; + } + } else { + // No transitions, single pair of offsets only + rawoff = initialRawOffset() * U_MILLIS_PER_SECOND; + dstoff = initialDstOffset() * U_MILLIS_PER_SECOND; + } + U_DEBUG_TZ_MSG(("getHistoricalOffset(%.1f, %s, %d, %d, raw, dst) - raw=%d, dst=%d\n", + date, local?"T":"F", NonExistingTimeOpt, DuplicatedTimeOpt, rawoff, dstoff)); +} + +/** + * TimeZone API. + */ +UBool OlsonTimeZone::useDaylightTime() const { + // If DST was observed in 1942 (for example) but has never been + // observed from 1943 to the present, most clients will expect + // this method to return false. This method determines whether + // DST is in use in the current year (at any point in the year) + // and returns true if so. + + UDate current = uprv_getUTCtime(); + if (finalZone != nullptr && current >= finalStartMillis) { + return finalZone->useDaylightTime(); + } + + int32_t year, month, dom, dow, doy, mid; + Grego::timeToFields(current, year, month, dom, dow, doy, mid); + + // Find start of this year, and start of next year + double start = Grego::fieldsToDay(year, 0, 1) * SECONDS_PER_DAY; + double limit = Grego::fieldsToDay(year+1, 0, 1) * SECONDS_PER_DAY; + + // Return true if DST is observed at any time during the current + // year. + for (int16_t i = 0; i < transitionCount(); ++i) { + double transition = (double)transitionTimeInSeconds(i); + if (transition >= limit) { + break; + } + if ((transition >= start && dstOffsetAt(i) != 0) + || (transition > start && dstOffsetAt(i - 1) != 0)) { + return true; + } + } + return false; +} +int32_t +OlsonTimeZone::getDSTSavings() const{ + if (finalZone != nullptr){ + return finalZone->getDSTSavings(); + } + return TimeZone::getDSTSavings(); +} +/** + * TimeZone API. + */ +UBool OlsonTimeZone::inDaylightTime(UDate date, UErrorCode& ec) const { + int32_t raw, dst; + getOffset(date, false, raw, dst, ec); + return dst != 0; +} + +UBool +OlsonTimeZone::hasSameRules(const TimeZone &other) const { + if (this == &other) { + return true; + } + const OlsonTimeZone* z = dynamic_cast(&other); + if (z == nullptr) { + return false; + } + + // [sic] pointer comparison: typeMapData points into + // memory-mapped or DLL space, so if two zones have the same + // pointer, they are equal. + if (typeMapData == z->typeMapData) { + return true; + } + + // If the pointers are not equal, the zones may still + // be equal if their rules and transitions are equal + if ((finalZone == nullptr && z->finalZone != nullptr) + || (finalZone != nullptr && z->finalZone == nullptr) + || (finalZone != nullptr && z->finalZone != nullptr && *finalZone != *z->finalZone)) { + return false; + } + + if (finalZone != nullptr) { + if (finalStartYear != z->finalStartYear || finalStartMillis != z->finalStartMillis) { + return false; + } + } + if (typeCount != z->typeCount + || transitionCountPre32 != z->transitionCountPre32 + || transitionCount32 != z->transitionCount32 + || transitionCountPost32 != z->transitionCountPost32) { + return false; + } + + return + arrayEqual(transitionTimesPre32, z->transitionTimesPre32, sizeof(transitionTimesPre32[0]) * transitionCountPre32 << 1) + && arrayEqual(transitionTimes32, z->transitionTimes32, sizeof(transitionTimes32[0]) * transitionCount32) + && arrayEqual(transitionTimesPost32, z->transitionTimesPost32, sizeof(transitionTimesPost32[0]) * transitionCountPost32 << 1) + && arrayEqual(typeOffsets, z->typeOffsets, sizeof(typeOffsets[0]) * typeCount << 1) + && arrayEqual(typeMapData, z->typeMapData, sizeof(typeMapData[0]) * transitionCount()); +} + +void +OlsonTimeZone::clearTransitionRules() { + initialRule = nullptr; + firstTZTransition = nullptr; + firstFinalTZTransition = nullptr; + historicRules = nullptr; + historicRuleCount = 0; + finalZoneWithStartYear = nullptr; + firstTZTransitionIdx = 0; + transitionRulesInitOnce.reset(); +} + +void +OlsonTimeZone::deleteTransitionRules() { + if (initialRule != nullptr) { + delete initialRule; + } + if (firstTZTransition != nullptr) { + delete firstTZTransition; + } + if (firstFinalTZTransition != nullptr) { + delete firstFinalTZTransition; + } + if (finalZoneWithStartYear != nullptr) { + delete finalZoneWithStartYear; + } + if (historicRules != nullptr) { + for (int i = 0; i < historicRuleCount; i++) { + if (historicRules[i] != nullptr) { + delete historicRules[i]; + } + } + uprv_free(historicRules); + } + clearTransitionRules(); +} + +/* + * Lazy transition rules initializer + */ + +static void U_CALLCONV initRules(OlsonTimeZone *This, UErrorCode &status) { + This->initTransitionRules(status); +} + +void +OlsonTimeZone::checkTransitionRules(UErrorCode& status) const { + OlsonTimeZone *ncThis = const_cast(this); + umtx_initOnce(ncThis->transitionRulesInitOnce, &initRules, ncThis, status); +} + +void +OlsonTimeZone::initTransitionRules(UErrorCode& status) { + if(U_FAILURE(status)) { + return; + } + deleteTransitionRules(); + UnicodeString tzid; + getID(tzid); + + UnicodeString stdName = tzid + UNICODE_STRING_SIMPLE("(STD)"); + UnicodeString dstName = tzid + UNICODE_STRING_SIMPLE("(DST)"); + + int32_t raw, dst; + + // Create initial rule + raw = initialRawOffset() * U_MILLIS_PER_SECOND; + dst = initialDstOffset() * U_MILLIS_PER_SECOND; + initialRule = new InitialTimeZoneRule((dst == 0 ? stdName : dstName), raw, dst); + // Check to make sure initialRule was created + if (initialRule == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + + int32_t transCount = transitionCount(); + if (transCount > 0) { + int16_t transitionIdx, typeIdx; + + // We probably no longer need to check the first "real" transition + // here, because the new tzcode remove such transitions already. + // For now, keeping this code for just in case. Feb 19, 2010 Yoshito + firstTZTransitionIdx = 0; + for (transitionIdx = 0; transitionIdx < transCount; transitionIdx++) { + if (typeMapData[transitionIdx] != 0) { // type 0 is the initial type + break; + } + firstTZTransitionIdx++; + } + if (transitionIdx == transCount) { + // Actually no transitions... + } else { + // Build historic rule array + UDate* times = (UDate*)uprv_malloc(sizeof(UDate)*transCount); /* large enough to store all transition times */ + if (times == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + for (typeIdx = 0; typeIdx < typeCount; typeIdx++) { + // Gather all start times for each pair of offsets + int32_t nTimes = 0; + for (transitionIdx = firstTZTransitionIdx; transitionIdx < transCount; transitionIdx++) { + if (typeIdx == (int16_t)typeMapData[transitionIdx]) { + UDate tt = (UDate)transitionTime(transitionIdx); + if (finalZone == nullptr || tt <= finalStartMillis) { + // Exclude transitions after finalMillis + times[nTimes++] = tt; + } + } + } + if (nTimes > 0) { + // Create a TimeArrayTimeZoneRule + raw = typeOffsets[typeIdx << 1] * U_MILLIS_PER_SECOND; + dst = typeOffsets[(typeIdx << 1) + 1] * U_MILLIS_PER_SECOND; + if (historicRules == nullptr) { + historicRuleCount = typeCount; + historicRules = (TimeArrayTimeZoneRule**)uprv_malloc(sizeof(TimeArrayTimeZoneRule*)*historicRuleCount); + if (historicRules == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + uprv_free(times); + return; + } + for (int i = 0; i < historicRuleCount; i++) { + // Initialize TimeArrayTimeZoneRule pointers as nullptr + historicRules[i] = nullptr; + } + } + historicRules[typeIdx] = new TimeArrayTimeZoneRule((dst == 0 ? stdName : dstName), + raw, dst, times, nTimes, DateTimeRule::UTC_TIME); + // Check for memory allocation error + if (historicRules[typeIdx] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + } + } + uprv_free(times); + + // Create initial transition + typeIdx = (int16_t)typeMapData[firstTZTransitionIdx]; + firstTZTransition = new TimeZoneTransition((UDate)transitionTime(firstTZTransitionIdx), + *initialRule, *historicRules[typeIdx]); + // Check to make sure firstTZTransition was created. + if (firstTZTransition == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + } + } + if (finalZone != nullptr) { + // Get the first occurrence of final rule starts + UDate startTime = (UDate)finalStartMillis; + TimeZoneRule *firstFinalRule = nullptr; + + if (finalZone->useDaylightTime()) { + /* + * Note: When an OlsonTimeZone is constructed, we should set the final year + * as the start year of finalZone. However, the boundary condition used for + * getting offset from finalZone has some problems. + * For now, we do not set the valid start year when the construction time + * and create a clone and set the start year when extracting rules. + */ + finalZoneWithStartYear = finalZone->clone(); + // Check to make sure finalZone was actually cloned. + if (finalZoneWithStartYear == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + finalZoneWithStartYear->setStartYear(finalStartYear); + + TimeZoneTransition tzt; + finalZoneWithStartYear->getNextTransition(startTime, false, tzt); + firstFinalRule = tzt.getTo()->clone(); + // Check to make sure firstFinalRule received proper clone. + if (firstFinalRule == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + startTime = tzt.getTime(); + } else { + // final rule with no transitions + finalZoneWithStartYear = finalZone->clone(); + // Check to make sure finalZone was actually cloned. + if (finalZoneWithStartYear == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + finalZone->getID(tzid); + firstFinalRule = new TimeArrayTimeZoneRule(tzid, + finalZone->getRawOffset(), 0, &startTime, 1, DateTimeRule::UTC_TIME); + // Check firstFinalRule was properly created. + if (firstFinalRule == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + } + TimeZoneRule *prevRule = nullptr; + if (transCount > 0) { + prevRule = historicRules[typeMapData[transCount - 1]]; + } + if (prevRule == nullptr) { + // No historic transitions, but only finalZone available + prevRule = initialRule; + } + firstFinalTZTransition = new TimeZoneTransition(); + // Check to make sure firstFinalTZTransition was created before dereferencing + if (firstFinalTZTransition == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + deleteTransitionRules(); + return; + } + firstFinalTZTransition->setTime(startTime); + firstFinalTZTransition->adoptFrom(prevRule->clone()); + firstFinalTZTransition->adoptTo(firstFinalRule); + } +} + +UBool +OlsonTimeZone::getNextTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const { + UErrorCode status = U_ZERO_ERROR; + checkTransitionRules(status); + if (U_FAILURE(status)) { + return false; + } + + if (finalZone != nullptr) { + if (inclusive && base == firstFinalTZTransition->getTime()) { + result = *firstFinalTZTransition; + return true; + } else if (base >= firstFinalTZTransition->getTime()) { + if (finalZone->useDaylightTime()) { + //return finalZone->getNextTransition(base, inclusive, result); + return finalZoneWithStartYear->getNextTransition(base, inclusive, result); + } else { + // No more transitions + return false; + } + } + } + if (historicRules != nullptr) { + // Find a historical transition + int16_t transCount = transitionCount(); + int16_t ttidx = transCount - 1; + for (; ttidx >= firstTZTransitionIdx; ttidx--) { + UDate t = (UDate)transitionTime(ttidx); + if (base > t || (!inclusive && base == t)) { + break; + } + } + if (ttidx == transCount - 1) { + if (firstFinalTZTransition != nullptr) { + result = *firstFinalTZTransition; + return true; + } else { + return false; + } + } else if (ttidx < firstTZTransitionIdx) { + result = *firstTZTransition; + return true; + } else { + // Create a TimeZoneTransition + TimeZoneRule *to = historicRules[typeMapData[ttidx + 1]]; + TimeZoneRule *from = historicRules[typeMapData[ttidx]]; + UDate startTime = (UDate)transitionTime(ttidx+1); + + // The transitions loaded from zoneinfo.res may contain non-transition data + UnicodeString fromName, toName; + from->getName(fromName); + to->getName(toName); + if (fromName == toName && from->getRawOffset() == to->getRawOffset() + && from->getDSTSavings() == to->getDSTSavings()) { + return getNextTransition(startTime, false, result); + } + result.setTime(startTime); + result.adoptFrom(from->clone()); + result.adoptTo(to->clone()); + return true; + } + } + return false; +} + +UBool +OlsonTimeZone::getPreviousTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const { + UErrorCode status = U_ZERO_ERROR; + checkTransitionRules(status); + if (U_FAILURE(status)) { + return false; + } + + if (finalZone != nullptr) { + if (inclusive && base == firstFinalTZTransition->getTime()) { + result = *firstFinalTZTransition; + return true; + } else if (base > firstFinalTZTransition->getTime()) { + if (finalZone->useDaylightTime()) { + //return finalZone->getPreviousTransition(base, inclusive, result); + return finalZoneWithStartYear->getPreviousTransition(base, inclusive, result); + } else { + result = *firstFinalTZTransition; + return true; + } + } + } + + if (historicRules != nullptr) { + // Find a historical transition + int16_t ttidx = transitionCount() - 1; + for (; ttidx >= firstTZTransitionIdx; ttidx--) { + UDate t = (UDate)transitionTime(ttidx); + if (base > t || (inclusive && base == t)) { + break; + } + } + if (ttidx < firstTZTransitionIdx) { + // No more transitions + return false; + } else if (ttidx == firstTZTransitionIdx) { + result = *firstTZTransition; + return true; + } else { + // Create a TimeZoneTransition + TimeZoneRule *to = historicRules[typeMapData[ttidx]]; + TimeZoneRule *from = historicRules[typeMapData[ttidx-1]]; + UDate startTime = (UDate)transitionTime(ttidx); + + // The transitions loaded from zoneinfo.res may contain non-transition data + UnicodeString fromName, toName; + from->getName(fromName); + to->getName(toName); + if (fromName == toName && from->getRawOffset() == to->getRawOffset() + && from->getDSTSavings() == to->getDSTSavings()) { + return getPreviousTransition(startTime, false, result); + } + result.setTime(startTime); + result.adoptFrom(from->clone()); + result.adoptTo(to->clone()); + return true; + } + } + return false; +} + +int32_t +OlsonTimeZone::countTransitionRules(UErrorCode& status) const { + if (U_FAILURE(status)) { + return 0; + } + checkTransitionRules(status); + if (U_FAILURE(status)) { + return 0; + } + + int32_t count = 0; + if (historicRules != nullptr) { + // historicRules may contain null entries when original zoneinfo data + // includes non transition data. + for (int32_t i = 0; i < historicRuleCount; i++) { + if (historicRules[i] != nullptr) { + count++; + } + } + } + if (finalZone != nullptr) { + if (finalZone->useDaylightTime()) { + count += 2; + } else { + count++; + } + } + return count; +} + +void +OlsonTimeZone::getTimeZoneRules(const InitialTimeZoneRule*& initial, + const TimeZoneRule* trsrules[], + int32_t& trscount, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + checkTransitionRules(status); + if (U_FAILURE(status)) { + return; + } + + // Initial rule + initial = initialRule; + + // Transition rules + int32_t cnt = 0; + if (historicRules != nullptr && trscount > cnt) { + // historicRules may contain null entries when original zoneinfo data + // includes non transition data. + for (int32_t i = 0; i < historicRuleCount; i++) { + if (historicRules[i] != nullptr) { + trsrules[cnt++] = historicRules[i]; + if (cnt >= trscount) { + break; + } + } + } + } + if (finalZoneWithStartYear != nullptr && trscount > cnt) { + const InitialTimeZoneRule *tmpini; + int32_t tmpcnt = trscount - cnt; + finalZoneWithStartYear->getTimeZoneRules(tmpini, &trsrules[cnt], tmpcnt, status); + if (U_FAILURE(status)) { + return; + } + cnt += tmpcnt; + } + // Set the result length + trscount = cnt; +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING + +//eof diff --git a/intl/icu/source/i18n/olsontz.h b/intl/icu/source/i18n/olsontz.h new file mode 100644 index 0000000000..9fe0d5dfed --- /dev/null +++ b/intl/icu/source/i18n/olsontz.h @@ -0,0 +1,455 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2003-2013, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Author: Alan Liu +* Created: July 21 2003 +* Since: ICU 2.8 +********************************************************************** +*/ +#ifndef OLSONTZ_H +#define OLSONTZ_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/basictz.h" +#include "umutex.h" + +struct UResourceBundle; + +U_NAMESPACE_BEGIN + +class SimpleTimeZone; + +/** + * A time zone based on the Olson tz database. Olson time zones change + * behavior over time. The raw offset, rules, presence or absence of + * daylight savings time, and even the daylight savings amount can all + * vary. + * + * This class uses a resource bundle named "zoneinfo". Zoneinfo is a + * table containing different kinds of resources. In several places, + * zones are referred to using integers. A zone's integer is a number + * from 0..n-1, where n is the number of zones, with the zones sorted + * in lexicographic order. + * + * 1. Zones. These have keys corresponding to the Olson IDs, e.g., + * "Asia/Shanghai". Each resource describes the behavior of the given + * zone. Zones come in two different formats. + * + * a. Zone (table). A zone is a table resource contains several + * type of resources below: + * + * - typeOffsets:intvector (Required) + * + * Sets of UTC raw/dst offset pairs in seconds. Entries at + * 2n represents raw offset and 2n+1 represents dst offset + * paired with the raw offset at 2n. The very first pair represents + * the initial zone offset (before the first transition) always. + * + * - trans:intvector (Optional) + * + * List of transition times represented by 32bit seconds from the + * epoch (1970-01-01T00:00Z) in ascending order. + * + * - transPre32/transPost32:intvector (Optional) + * + * List of transition times before/after 32bit minimum seconds. + * Each time is represented by a pair of 32bit integer. + * + * - typeMap:bin (Optional) + * + * Array of bytes representing the mapping between each transition + * time (transPre32/trans/transPost32) and its corresponding offset + * data (typeOffsets). + * + * - finalRule:string (Optional) + * + * If a recurrent transition rule is applicable to a zone forever + * after the final transition time, finalRule represents the rule + * in Rules data. + * + * - finalRaw:int (Optional) + * + * When finalRule is available, finalRaw is required and specifies + * the raw (base) offset of the rule. + * + * - finalYear:int (Optional) + * + * When finalRule is available, finalYear is required and specifies + * the start year of the rule. + * + * - links:intvector (Optional) + * + * When this zone data is shared with other zones, links specifies + * all zones including the zone itself. Each zone is referenced by + * integer index. + * + * b. Link (int, length 1). A link zone is an int resource. The + * integer is the zone number of the target zone. The key of this + * resource is an alternate name for the target zone. This data + * is corresponding to Link data in the tz database. + * + * + * 2. Rules. These have keys corresponding to the Olson rule IDs, + * with an underscore prepended, e.g., "_EU". Each resource describes + * the behavior of the given rule using an intvector, containing the + * onset list, the cessation list, and the DST savings. The onset and + * cessation lists consist of the month, dowim, dow, time, and time + * mode. The end result is that the 11 integers describing the rule + * can be passed directly into the SimpleTimeZone 13-argument + * constructor (the other two arguments will be the raw offset, taken + * from the complex zone element 5, and the ID string, which is not + * used), with the times and the DST savings multiplied by 1000 to + * scale from seconds to milliseconds. + * + * 3. Regions. An array specifies mapping between zones and regions. + * Each item is either a 2-letter ISO country code or "001" + * (UN M.49 - World). This data is generated from "zone.tab" + * in the tz database. + */ +class U_I18N_API OlsonTimeZone: public BasicTimeZone { + public: + /** + * Construct from a resource bundle. + * @param top the top-level zoneinfo resource bundle. This is used + * to lookup the rule that `res' may refer to, if there is one. + * @param res the resource bundle of the zone to be constructed + * @param tzid the time zone ID + * @param ec input-output error code + */ + OlsonTimeZone(const UResourceBundle* top, + const UResourceBundle* res, + const UnicodeString& tzid, + UErrorCode& ec); + + /** + * Copy constructor + */ + OlsonTimeZone(const OlsonTimeZone& other); + + /** + * Destructor + */ + virtual ~OlsonTimeZone(); + + /** + * Assignment operator + */ + OlsonTimeZone& operator=(const OlsonTimeZone& other); + + /** + * Returns true if the two TimeZone objects are equal. + */ + virtual bool operator==(const TimeZone& other) const override; + + /** + * TimeZone API. + */ + virtual OlsonTimeZone* clone() const override; + + /** + * TimeZone API. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + /** + * TimeZone API. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * TimeZone API. Do not call this; prefer getOffset(UDate,...). + */ + virtual int32_t getOffset(uint8_t era, int32_t year, int32_t month, + int32_t day, uint8_t dayOfWeek, + int32_t millis, UErrorCode& ec) const override; + + /** + * TimeZone API. Do not call this; prefer getOffset(UDate,...). + */ + virtual int32_t getOffset(uint8_t era, int32_t year, int32_t month, + int32_t day, uint8_t dayOfWeek, + int32_t millis, int32_t monthLength, + UErrorCode& ec) const override; + + /** + * TimeZone API. + */ + virtual void getOffset(UDate date, UBool local, int32_t& rawOffset, + int32_t& dstOffset, UErrorCode& ec) const override; + + /** + * BasicTimeZone API. + */ + virtual void getOffsetFromLocal( + UDate date, UTimeZoneLocalOption nonExistingTimeOpt, + UTimeZoneLocalOption duplicatedTimeOpt, + int32_t& rawOffset, int32_t& dstOffset, UErrorCode& status) const override; + + /** + * TimeZone API. This method has no effect since objects of this + * class are quasi-immutable (the base class allows the ID to be + * changed). + */ + virtual void setRawOffset(int32_t offsetMillis) override; + + /** + * TimeZone API. For a historical zone, the raw offset can change + * over time, so this API is not useful. In order to approximate + * expected behavior, this method returns the raw offset for the + * current moment in time. + */ + virtual int32_t getRawOffset() const override; + + /** + * TimeZone API. For a historical zone, whether DST is used or + * not varies over time. In order to approximate expected + * behavior, this method returns true if DST is observed at any + * point in the current year. + */ + virtual UBool useDaylightTime() const override; + + /** + * TimeZone API. + */ + virtual UBool inDaylightTime(UDate date, UErrorCode& ec) const override; + + /** + * TimeZone API. + */ + virtual int32_t getDSTSavings() const override; + + /** + * TimeZone API. Also comare historic transitions. + */ + virtual UBool hasSameRules(const TimeZone& other) const override; + + /** + * BasicTimeZone API. + * Gets the first time zone transition after the base time. + * @param base The base time. + * @param inclusive Whether the base time is inclusive or not. + * @param result Receives the first transition after the base time. + * @return true if the transition is found. + */ + virtual UBool getNextTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const override; + + /** + * BasicTimeZone API. + * Gets the most recent time zone transition before the base time. + * @param base The base time. + * @param inclusive Whether the base time is inclusive or not. + * @param result Receives the most recent transition before the base time. + * @return true if the transition is found. + */ + virtual UBool getPreviousTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const override; + + /** + * BasicTimeZone API. + * Returns the number of TimeZoneRules which represents time transitions, + * for this time zone, that is, all TimeZoneRules for this time zone except + * InitialTimeZoneRule. The return value range is 0 or any positive value. + * @param status Receives error status code. + * @return The number of TimeZoneRules representing time transitions. + */ + virtual int32_t countTransitionRules(UErrorCode& status) const override; + + /** + * Gets the InitialTimeZoneRule and the set of TimeZoneRule + * which represent time transitions for this time zone. On successful return, + * the argument initial points to non-nullptr InitialTimeZoneRule and + * the array trsrules is filled with 0 or multiple TimeZoneRule + * instances up to the size specified by trscount. The results are referencing the + * rule instance held by this time zone instance. Therefore, after this time zone + * is destructed, they are no longer available. + * @param initial Receives the initial timezone rule + * @param trsrules Receives the timezone transition rules + * @param trscount On input, specify the size of the array 'transitions' receiving + * the timezone transition rules. On output, actual number of + * rules filled in the array will be set. + * @param status Receives error status code. + */ + virtual void getTimeZoneRules(const InitialTimeZoneRule*& initial, + const TimeZoneRule* trsrules[], int32_t& trscount, UErrorCode& status) const override; + + /** + * Internal API returning the canonical ID of this zone. + * This ID won't be affected by setID(). + */ + const char16_t *getCanonicalID() const; + +private: + /** + * Default constructor. Creates a time zone with an empty ID and + * a fixed GMT offset of zero. + */ + OlsonTimeZone(); + +private: + + void constructEmpty(); + + void getHistoricalOffset(UDate date, UBool local, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt, + int32_t& rawoff, int32_t& dstoff) const; + + int16_t transitionCount() const; + + int64_t transitionTimeInSeconds(int16_t transIdx) const; + double transitionTime(int16_t transIdx) const; + + /* + * Following 3 methods return an offset at the given transition time index. + * When the index is negative, return the initial offset. + */ + int32_t zoneOffsetAt(int16_t transIdx) const; + int32_t rawOffsetAt(int16_t transIdx) const; + int32_t dstOffsetAt(int16_t transIdx) const; + + /* + * Following methods return the initial offset. + */ + int32_t initialRawOffset() const; + int32_t initialDstOffset() const; + + /** + * Number of transitions in each time range + */ + int16_t transitionCountPre32; + int16_t transitionCount32; + int16_t transitionCountPost32; + + /** + * Time of each transition in seconds from 1970 epoch before 32bit second range (<= 1900). + * Each transition in this range is represented by a pair of int32_t. + * Length is transitionCount int32_t's. nullptr if no transitions in this range. + */ + const int32_t *transitionTimesPre32; // alias into res; do not delete + + /** + * Time of each transition in seconds from 1970 epoch in 32bit second range. + * Length is transitionCount int32_t's. nullptr if no transitions in this range. + */ + const int32_t *transitionTimes32; // alias into res; do not delete + + /** + * Time of each transition in seconds from 1970 epoch after 32bit second range (>= 2038). + * Each transition in this range is represented by a pair of int32_t. + * Length is transitionCount int32_t's. nullptr if no transitions in this range. + */ + const int32_t *transitionTimesPost32; // alias into res; do not delete + + /** + * Number of types, 1..255 + */ + int16_t typeCount; + + /** + * Offset from GMT in seconds for each type. + * Length is typeCount int32_t's. At least one type (a pair of int32_t) + * is required. + */ + const int32_t *typeOffsets; // alias into res; do not delete + + /** + * Type description data, consisting of transitionCount uint8_t + * type indices (from 0..typeCount-1). + * Length is transitionCount int16_t's. nullptr if no transitions. + */ + const uint8_t *typeMapData; // alias into res; do not delete + + /** + * A SimpleTimeZone that governs the behavior for date >= finalMillis. + */ + SimpleTimeZone *finalZone; // owned, may be nullptr + + /** + * For date >= finalMillis, the finalZone will be used. + */ + double finalStartMillis; + + /** + * For year >= finalYear, the finalZone will be used. + */ + int32_t finalStartYear; + + /* + * Canonical (CLDR) ID of this zone + */ + const char16_t *canonicalID; + + /* BasicTimeZone support */ + void clearTransitionRules(); + void deleteTransitionRules(); + void checkTransitionRules(UErrorCode& status) const; + + public: // Internal, for access from plain C code + void initTransitionRules(UErrorCode& status); + private: + + InitialTimeZoneRule *initialRule; + TimeZoneTransition *firstTZTransition; + int16_t firstTZTransitionIdx; + TimeZoneTransition *firstFinalTZTransition; + TimeArrayTimeZoneRule **historicRules; + int16_t historicRuleCount; + SimpleTimeZone *finalZoneWithStartYear; // hack + UInitOnce transitionRulesInitOnce {}; +}; + +inline int16_t +OlsonTimeZone::transitionCount() const { + return transitionCountPre32 + transitionCount32 + transitionCountPost32; +} + +inline double +OlsonTimeZone::transitionTime(int16_t transIdx) const { + return (double)transitionTimeInSeconds(transIdx) * U_MILLIS_PER_SECOND; +} + +inline int32_t +OlsonTimeZone::zoneOffsetAt(int16_t transIdx) const { + int16_t typeIdx = (transIdx >= 0 ? typeMapData[transIdx] : 0) << 1; + return typeOffsets[typeIdx] + typeOffsets[typeIdx + 1]; +} + +inline int32_t +OlsonTimeZone::rawOffsetAt(int16_t transIdx) const { + int16_t typeIdx = (transIdx >= 0 ? typeMapData[transIdx] : 0) << 1; + return typeOffsets[typeIdx]; +} + +inline int32_t +OlsonTimeZone::dstOffsetAt(int16_t transIdx) const { + int16_t typeIdx = (transIdx >= 0 ? typeMapData[transIdx] : 0) << 1; + return typeOffsets[typeIdx + 1]; +} + +inline int32_t +OlsonTimeZone::initialRawOffset() const { + return typeOffsets[0]; +} + +inline int32_t +OlsonTimeZone::initialDstOffset() const { + return typeOffsets[1]; +} + +inline const char16_t* +OlsonTimeZone::getCanonicalID() const { + return canonicalID; +} + + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_FORMATTING +#endif // OLSONTZ_H + +//eof diff --git a/intl/icu/source/i18n/persncal.cpp b/intl/icu/source/i18n/persncal.cpp new file mode 100644 index 0000000000..ab13f43434 --- /dev/null +++ b/intl/icu/source/i18n/persncal.cpp @@ -0,0 +1,301 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ****************************************************************************** + * Copyright (C) 2003-2013, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File PERSNCAL.CPP + * + * Modification History: + * + * Date Name Description + * 9/23/2003 mehran posted to icu-design + * 10/1/2012 roozbeh Fixed algorithm and heavily refactored and rewrote + * based on the implementation of Gregorian + ***************************************************************************** + */ + +#include "persncal.h" + +#if !UCONFIG_NO_FORMATTING + +#include "umutex.h" +#include "gregoimp.h" // Math +#include + +static const int16_t kPersianNumDays[] += {0,31,62,93,124,155,186,216,246,276,306,336}; // 0-based, for day-in-year +static const int8_t kPersianMonthLength[] += {31,31,31,31,31,31,30,30,30,30,30,29}; // 0-based +static const int8_t kPersianLeapMonthLength[] += {31,31,31,31,31,31,30,30,30,30,30,30}; // 0-based + +static const int32_t kPersianCalendarLimits[UCAL_FIELD_COUNT][4] = { + // Minimum Greatest Least Maximum + // Minimum Maximum + { 0, 0, 0, 0}, // ERA + { -5000000, -5000000, 5000000, 5000000}, // YEAR + { 0, 0, 11, 11}, // MONTH + { 1, 1, 52, 53}, // WEEK_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH + { 1, 1, 29, 31}, // DAY_OF_MONTH + { 1, 1, 365, 366}, // DAY_OF_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK + { 1, 1, 5, 5}, // DAY_OF_WEEK_IN_MONTH + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET + { -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL + { -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY + {/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH + { 0, 0, 11, 11}, // ORDINAL_MONTH +}; + +U_NAMESPACE_BEGIN + +static const int32_t PERSIAN_EPOCH = 1948320; + +// Implementation of the PersianCalendar class + +//------------------------------------------------------------------------- +// Constructors... +//------------------------------------------------------------------------- + +const char *PersianCalendar::getType() const { + return "persian"; +} + +PersianCalendar* PersianCalendar::clone() const { + return new PersianCalendar(*this); +} + +PersianCalendar::PersianCalendar(const Locale& aLocale, UErrorCode& success) + : Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success) +{ + setTimeInMillis(getNow(), success); // Call this again now that the vtable is set up properly. +} + +PersianCalendar::PersianCalendar(const PersianCalendar& other) : Calendar(other) { +} + +PersianCalendar::~PersianCalendar() +{ +} + +//------------------------------------------------------------------------- +// Minimum / Maximum access functions +//------------------------------------------------------------------------- + + +int32_t PersianCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const { + return kPersianCalendarLimits[field][limitType]; +} + +//------------------------------------------------------------------------- +// Assorted calculation utilities +// + +/** + * Determine whether a year is a leap year in the Persian calendar + */ +UBool PersianCalendar::isLeapYear(int32_t year) +{ + int32_t remainder; + ClockMath::floorDivide(25 * year + 11, 33, &remainder); + return (remainder < 8); +} + +/** + * Return the day # on which the given year starts. Days are counted + * from the Persian epoch, origin 0. + */ +int32_t PersianCalendar::yearStart(int32_t year) { + return handleComputeMonthStart(year,0,false); +} + +/** + * Return the day # on which the given month starts. Days are counted + * from the Persian epoch, origin 0. + * + * @param year The Persian year + * @param year The Persian month, 0-based + */ +int32_t PersianCalendar::monthStart(int32_t year, int32_t month) const { + return handleComputeMonthStart(year,month,true); +} + +//---------------------------------------------------------------------- +// Calendar framework +//---------------------------------------------------------------------- + +/** + * Return the length (in days) of the given month. + * + * @param year The Persian year + * @param year The Persian month, 0-based + */ +int32_t PersianCalendar::handleGetMonthLength(int32_t extendedYear, int32_t month) const { + // If the month is out of range, adjust it into range, and + // modify the extended year value accordingly. + if (month < 0 || month > 11) { + extendedYear += ClockMath::floorDivide(month, 12, &month); + } + + return isLeapYear(extendedYear) ? kPersianLeapMonthLength[month] : kPersianMonthLength[month]; +} + +/** + * Return the number of days in the given Persian year + */ +int32_t PersianCalendar::handleGetYearLength(int32_t extendedYear) const { + return isLeapYear(extendedYear) ? 366 : 365; +} + +//------------------------------------------------------------------------- +// Functions for converting from field values to milliseconds.... +//------------------------------------------------------------------------- + +// Return JD of start of given month/year +int32_t PersianCalendar::handleComputeMonthStart(int32_t eyear, int32_t month, UBool /*useMonth*/) const { + // If the month is out of range, adjust it into range, and + // modify the extended year value accordingly. + if (month < 0 || month > 11) { + eyear += ClockMath::floorDivide(month, 12, &month); + } + + int32_t julianDay = PERSIAN_EPOCH - 1 + 365 * (eyear - 1) + ClockMath::floorDivide(8 * eyear + 21, 33); + + if (month != 0) { + julianDay += kPersianNumDays[month]; + } + + return julianDay; +} + +//------------------------------------------------------------------------- +// Functions for converting from milliseconds to field values +//------------------------------------------------------------------------- + +int32_t PersianCalendar::handleGetExtendedYear() { + int32_t year; + if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) { + year = internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1 + } else { + year = internalGet(UCAL_YEAR, 1); // Default to year 1 + } + return year; +} + +/** + * Override Calendar to compute several fields specific to the Persian + * calendar system. These are: + * + *

    • ERA + *
    • YEAR + *
    • MONTH + *
    • DAY_OF_MONTH + *
    • DAY_OF_YEAR + *
    • EXTENDED_YEAR
    + * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. + */ +void PersianCalendar::handleComputeFields(int32_t julianDay, UErrorCode &/*status*/) { + int32_t year, month, dayOfMonth, dayOfYear; + + int32_t daysSinceEpoch = julianDay - PERSIAN_EPOCH; + year = 1 + (int32_t)ClockMath::floorDivide(33 * (int64_t)daysSinceEpoch + 3, (int64_t)12053); + + int32_t farvardin1 = 365 * (year - 1) + ClockMath::floorDivide(8 * year + 21, 33); + dayOfYear = (daysSinceEpoch - farvardin1); // 0-based + if (dayOfYear < 216) { // Compute 0-based month + month = dayOfYear / 31; + } else { + month = (dayOfYear - 6) / 30; + } + dayOfMonth = dayOfYear - kPersianNumDays[month] + 1; + ++dayOfYear; // Make it 1-based now + + internalSet(UCAL_ERA, 0); + internalSet(UCAL_YEAR, year); + internalSet(UCAL_EXTENDED_YEAR, year); + internalSet(UCAL_MONTH, month); + internalSet(UCAL_ORDINAL_MONTH, month); + internalSet(UCAL_DAY_OF_MONTH, dayOfMonth); + internalSet(UCAL_DAY_OF_YEAR, dayOfYear); +} + +constexpr uint32_t kPersianRelatedYearDiff = 622; + +int32_t PersianCalendar::getRelatedYear(UErrorCode &status) const +{ + int32_t year = get(UCAL_EXTENDED_YEAR, status); + if (U_FAILURE(status)) { + return 0; + } + return year + kPersianRelatedYearDiff; +} + +void PersianCalendar::setRelatedYear(int32_t year) +{ + // set extended year + set(UCAL_EXTENDED_YEAR, year - kPersianRelatedYearDiff); +} + +// default century + +static UDate gSystemDefaultCenturyStart = DBL_MIN; +static int32_t gSystemDefaultCenturyStartYear = -1; +static icu::UInitOnce gSystemDefaultCenturyInit {}; + +UBool PersianCalendar::haveDefaultCentury() const +{ + return true; +} + +static void U_CALLCONV initializeSystemDefaultCentury() { + // initialize systemDefaultCentury and systemDefaultCenturyYear based + // on the current time. They'll be set to 80 years before + // the current time. + UErrorCode status = U_ZERO_ERROR; + PersianCalendar calendar(Locale("@calendar=persian"),status); + if (U_SUCCESS(status)) + { + calendar.setTime(Calendar::getNow(), status); + calendar.add(UCAL_YEAR, -80, status); + + gSystemDefaultCenturyStart = calendar.getTime(status); + gSystemDefaultCenturyStartYear = calendar.get(UCAL_YEAR, status); + } + // We have no recourse upon failure unless we want to propagate the failure + // out. +} + +UDate PersianCalendar::defaultCenturyStart() const { + // lazy-evaluate systemDefaultCenturyStart + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStart; +} + +int32_t PersianCalendar::defaultCenturyStartYear() const { + // lazy-evaluate systemDefaultCenturyStartYear + umtx_initOnce(gSystemDefaultCenturyInit, &initializeSystemDefaultCentury); + return gSystemDefaultCenturyStartYear; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(PersianCalendar) + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/persncal.h b/intl/icu/source/i18n/persncal.h new file mode 100644 index 0000000000..b943321a54 --- /dev/null +++ b/intl/icu/source/i18n/persncal.h @@ -0,0 +1,325 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ****************************************************************************** + * Copyright (C) 2003-2013, International Business Machines Corporation + * and others. All Rights Reserved. + ****************************************************************************** + * + * File PERSNCAL.H + * + * Modification History: + * + * Date Name Description + * 9/23/2003 mehran posted to icu-design + ***************************************************************************** + */ + +#ifndef PERSNCAL_H +#define PERSNCAL_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/calendar.h" + +U_NAMESPACE_BEGIN + +/** + * PersianCalendar is a subclass of Calendar + * that implements the Persian calendar. It is used as the official + * calendar in Iran. This calendar is also known as the "Hijri Shamsi" + * calendar, since it starts at the time of Mohammed's emigration (or + * "hijra") to Medinah on Thursday, July 15, 622 AD (Julian) and is a + * solar calendar system (or "shamsi"). + *

    + * The Persian calendar is strictly solar, and thus a Persian year has twelve + * solar months. A Persian year is about 365 days long, except in leap years + * which is 366 days long. + *

    + * The six first months of Persian Calendar are 31 days long. The next five + * months are 30 days long. The last month is 29 days long in normal years, + * and 30 days long in leap years. + * + * @see GregorianCalendar + * + * @author Mehran Mehr + * @internal + */ +class PersianCalendar : public Calendar { + public: + //------------------------------------------------------------------------- + // Constants... + //------------------------------------------------------------------------- + /** + * Constants for the months + * @internal + */ + enum EMonths { + /** + * Constant for Farvardin, the 1st month of the Persian year. + * @internal + */ + FARVARDIN = 0, + + /** + * Constant for Ordibehesht, the 2nd month of the Persian year. + * @internal + */ + ORDIBEHESHT = 1, + + /** + * Constant for Khordad, the 3rd month of the Persian year. + * @internal + */ + KHORDAD = 2, + + /** + * Constant for Tir, the 4th month of the Persian year. + * @internal + */ + TIR = 3, + + /** + * Constant for Mordad, the 5th month of the Persian year. + * @internal + */ + MORDAD = 4, + + /** + * Constant for Shahrivar, the 6th month of the Persian year. + * @internal + */ + SHAHRIVAR = 5, + + /** + * Constant for Mehr, the 7th month of the Persian year. + * @internal + */ + MEHR = 6, + + /** + * Constant for Aban, the 8th month of the Persian year. + * @internal + */ + ABAN = 7, + + /** + * Constant for Azar, the 9th month of the Persian year. + * @internal + */ + AZAR = 8, + + /** + * Constant for Dei, the 10th month of the Persian year. + * @internal + */ + DEI = 9, + + /** + * Constant for Bahman, the 11th month of the Persian year. + * @internal + */ + BAHMAN = 10, + + /** + * Constant for Esfand, the 12th month of the Persian year. + * @internal + */ + ESFAND = 11, + + PERSIAN_MONTH_MAX + }; + + + + //------------------------------------------------------------------------- + // Constructors... + //------------------------------------------------------------------------- + + /** + * Constructs a PersianCalendar based on the current time in the default time zone + * with the given locale. + * + * @param aLocale The given locale. + * @param success Indicates the status of PersianCalendar object construction. + * Returns U_ZERO_ERROR if constructed successfully. + * @internal + */ + PersianCalendar(const Locale& aLocale, UErrorCode &success); + + /** + * Copy Constructor + * @internal + */ + PersianCalendar(const PersianCalendar& other); + + /** + * Destructor. + * @internal + */ + virtual ~PersianCalendar(); + + // TODO: copy c'tor, etc + + // clone + virtual PersianCalendar* clone() const override; + + private: + /** + * Determine whether a year is a leap year in the Persian calendar + */ + static UBool isLeapYear(int32_t year); + + /** + * Return the day # on which the given year starts. Days are counted + * from the Hijri epoch, origin 0. + */ + int32_t yearStart(int32_t year); + + /** + * Return the day # on which the given month starts. Days are counted + * from the Hijri epoch, origin 0. + * + * @param year The hijri shamsi year + * @param year The hijri shamsi month, 0-based + */ + int32_t monthStart(int32_t year, int32_t month) const; + + //---------------------------------------------------------------------- + // Calendar framework + //---------------------------------------------------------------------- + protected: + /** + * @internal + */ + virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override; + + /** + * Return the length (in days) of the given month. + * + * @param year The hijri shamsi year + * @param year The hijri shamsi month, 0-based + * @internal + */ + virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month) const override; + + /** + * Return the number of days in the given Persian year + * @internal + */ + virtual int32_t handleGetYearLength(int32_t extendedYear) const override; + + //------------------------------------------------------------------------- + // Functions for converting from field values to milliseconds.... + //------------------------------------------------------------------------- + + // Return JD of start of given month/year + /** + * @internal + */ + virtual int32_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth) const override; + + //------------------------------------------------------------------------- + // Functions for converting from milliseconds to field values + //------------------------------------------------------------------------- + + /** + * @internal + */ + virtual int32_t handleGetExtendedYear() override; + + /** + * Override Calendar to compute several fields specific to the Persian + * calendar system. These are: + * + *

    • ERA + *
    • YEAR + *
    • MONTH + *
    • DAY_OF_MONTH + *
    • DAY_OF_YEAR + *
    • EXTENDED_YEAR
    + * + * The DAY_OF_WEEK and DOW_LOCAL fields are already set when this + * method is called. The getGregorianXxx() methods return Gregorian + * calendar equivalents for the given Julian day. + * @internal + */ + virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override; + + // UObject stuff + public: + /** + * @return The class ID for this object. All objects of a given class have the + * same class ID. Objects of other classes have different class IDs. + * @internal + */ + virtual UClassID getDynamicClassID() const override; + + /** + * Return the class ID for this class. This is useful only for comparing to a return + * value from getDynamicClassID(). For example: + * + * Base* polymorphic_pointer = createPolymorphicObject(); + * if (polymorphic_pointer->getDynamicClassID() == + * Derived::getStaticClassID()) ... + * + * @return The class ID for all objects of this class. + * @internal + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * return the calendar type, "persian". + * + * @return calendar type + * @internal + */ + virtual const char * getType() const override; + + /** + * @return The related Gregorian year; will be obtained by modifying the value + * obtained by get from UCAL_EXTENDED_YEAR field + * @internal + */ + virtual int32_t getRelatedYear(UErrorCode &status) const override; + + /** + * @param year The related Gregorian year to set; will be modified as necessary then + * set in UCAL_EXTENDED_YEAR field + * @internal + */ + virtual void setRelatedYear(int32_t year) override; + + private: + PersianCalendar(); // default constructor not implemented + + protected: + /** + * Returns true because the Persian Calendar does have a default century + * @internal + */ + virtual UBool haveDefaultCentury() const override; + + /** + * Returns the date of the start of the default century + * @return start of century - in milliseconds since epoch, 1970 + * @internal + */ + virtual UDate defaultCenturyStart() const override; + + /** + * Returns the year in which the default century begins + * @internal + */ + virtual int32_t defaultCenturyStartYear() const override; +}; + +U_NAMESPACE_END + +#endif +#endif + + + diff --git a/intl/icu/source/i18n/pluralranges.cpp b/intl/icu/source/i18n/pluralranges.cpp new file mode 100644 index 0000000000..403836f627 --- /dev/null +++ b/intl/icu/source/i18n/pluralranges.cpp @@ -0,0 +1,144 @@ +// © 2018 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +// Allow implicit conversion from char16_t* to UnicodeString for this file: +// Helpful in toString methods and elsewhere. +#define UNISTR_FROM_STRING_EXPLICIT + +#include "unicode/numberrangeformatter.h" +#include "pluralranges.h" +#include "uresimp.h" +#include "charstr.h" +#include "uassert.h" +#include "util.h" +#include "numrange_impl.h" + +U_NAMESPACE_BEGIN + + +namespace { + +class PluralRangesDataSink : public ResourceSink { + public: + PluralRangesDataSink(StandardPluralRanges& output) : fOutput(output) {} + + void put(const char* /*key*/, ResourceValue& value, UBool /*noFallback*/, UErrorCode& status) override { + ResourceArray entriesArray = value.getArray(status); + if (U_FAILURE(status)) { return; } + fOutput.setCapacity(entriesArray.getSize(), status); + if (U_FAILURE(status)) { return; } + for (int i = 0; entriesArray.getValue(i, value); i++) { + ResourceArray pluralFormsArray = value.getArray(status); + if (U_FAILURE(status)) { return; } + if (pluralFormsArray.getSize() != 3) { + status = U_RESOURCE_TYPE_MISMATCH; + return; + } + pluralFormsArray.getValue(0, value); + StandardPlural::Form first = StandardPlural::fromString(value.getUnicodeString(status), status); + if (U_FAILURE(status)) { return; } + pluralFormsArray.getValue(1, value); + StandardPlural::Form second = StandardPlural::fromString(value.getUnicodeString(status), status); + if (U_FAILURE(status)) { return; } + pluralFormsArray.getValue(2, value); + StandardPlural::Form result = StandardPlural::fromString(value.getUnicodeString(status), status); + if (U_FAILURE(status)) { return; } + fOutput.addPluralRange(first, second, result); + } + } + + private: + StandardPluralRanges& fOutput; +}; + +void getPluralRangesData(const Locale& locale, StandardPluralRanges& output, UErrorCode& status) { + LocalUResourceBundlePointer rb(ures_openDirect(nullptr, "pluralRanges", &status)); + if (U_FAILURE(status)) { return; } + + CharString dataPath; + dataPath.append("locales/", -1, status); + dataPath.append(locale.getLanguage(), -1, status); + if (U_FAILURE(status)) { return; } + int32_t setLen; + // Not all languages are covered: fail gracefully + UErrorCode internalStatus = U_ZERO_ERROR; + const char16_t* set = ures_getStringByKeyWithFallback(rb.getAlias(), dataPath.data(), &setLen, &internalStatus); + if (U_FAILURE(internalStatus)) { return; } + + dataPath.clear(); + dataPath.append("rules/", -1, status); + dataPath.appendInvariantChars(set, setLen, status); + if (U_FAILURE(status)) { return; } + PluralRangesDataSink sink(output); + ures_getAllItemsWithFallback(rb.getAlias(), dataPath.data(), sink, status); +} + +} // namespace + + +StandardPluralRanges +StandardPluralRanges::forLocale(const Locale& locale, UErrorCode& status) { + StandardPluralRanges result; + getPluralRangesData(locale, result, status); + return result; +} + +StandardPluralRanges +StandardPluralRanges::copy(UErrorCode& status) const { + StandardPluralRanges result; + if (fTriplesLen > result.fTriples.getCapacity()) { + if (result.fTriples.resize(fTriplesLen) == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return result; + } + } + uprv_memcpy(result.fTriples.getAlias(), + fTriples.getAlias(), + fTriplesLen * sizeof(fTriples[0])); + result.fTriplesLen = fTriplesLen; + return result; +} + +LocalPointer +StandardPluralRanges::toPointer(UErrorCode& status) && noexcept { + return LocalPointer(new StandardPluralRanges(std::move(*this)), status); +} + +void StandardPluralRanges::addPluralRange( + StandardPlural::Form first, + StandardPlural::Form second, + StandardPlural::Form result) { + U_ASSERT(fTriplesLen < fTriples.getCapacity()); + fTriples[fTriplesLen] = {first, second, result}; + fTriplesLen++; +} + +void StandardPluralRanges::setCapacity(int32_t length, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + if (length > fTriples.getCapacity()) { + if (fTriples.resize(length, 0) == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + } +} + +StandardPlural::Form +StandardPluralRanges::resolve(StandardPlural::Form first, StandardPlural::Form second) const { + for (int32_t i=0; i toPointer(UErrorCode& status) && noexcept; + + /** Select rule based on the first and second forms */ + StandardPlural::Form resolve(StandardPlural::Form first, StandardPlural::Form second) const; + + /** Used for data loading. */ + void addPluralRange( + StandardPlural::Form first, + StandardPlural::Form second, + StandardPlural::Form result); + + /** Used for data loading. */ + void setCapacity(int32_t length, UErrorCode& status); + + private: + struct StandardPluralRangeTriple { + StandardPlural::Form first; + StandardPlural::Form second; + StandardPlural::Form result; + }; + + // TODO: An array is simple here, but it results in linear lookup time. + // Certain locales have 20-30 entries in this list. + // Consider changing to a smarter data structure. + typedef MaybeStackArray PluralRangeTriples; + PluralRangeTriples fTriples; + int32_t fTriplesLen = 0; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ +#endif //__PLURALRANGES_H__ diff --git a/intl/icu/source/i18n/plurfmt.cpp b/intl/icu/source/i18n/plurfmt.cpp new file mode 100644 index 0000000000..33a539cd19 --- /dev/null +++ b/intl/icu/source/i18n/plurfmt.cpp @@ -0,0 +1,607 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2009-2015, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File PLURFMT.CPP +******************************************************************************* +*/ + +#include "unicode/decimfmt.h" +#include "unicode/messagepattern.h" +#include "unicode/plurfmt.h" +#include "unicode/plurrule.h" +#include "unicode/utypes.h" +#include "cmemory.h" +#include "messageimpl.h" +#include "nfrule.h" +#include "plurrule_impl.h" +#include "uassert.h" +#include "uhash.h" +#include "number_decimalquantity.h" +#include "number_utils.h" +#include "number_utypes.h" + +#if !UCONFIG_NO_FORMATTING + +U_NAMESPACE_BEGIN + +using number::impl::DecimalQuantity; + +static const char16_t OTHER_STRING[] = { + 0x6F, 0x74, 0x68, 0x65, 0x72, 0 // "other" +}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(PluralFormat) + +PluralFormat::PluralFormat(UErrorCode& status) + : locale(Locale::getDefault()), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, UPLURAL_TYPE_CARDINAL, status); +} + +PluralFormat::PluralFormat(const Locale& loc, UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, UPLURAL_TYPE_CARDINAL, status); +} + +PluralFormat::PluralFormat(const PluralRules& rules, UErrorCode& status) + : locale(Locale::getDefault()), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(&rules, UPLURAL_TYPE_COUNT, status); +} + +PluralFormat::PluralFormat(const Locale& loc, + const PluralRules& rules, + UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(&rules, UPLURAL_TYPE_COUNT, status); +} + +PluralFormat::PluralFormat(const Locale& loc, + UPluralType type, + UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, type, status); +} + +PluralFormat::PluralFormat(const UnicodeString& pat, + UErrorCode& status) + : locale(Locale::getDefault()), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, UPLURAL_TYPE_CARDINAL, status); + applyPattern(pat, status); +} + +PluralFormat::PluralFormat(const Locale& loc, + const UnicodeString& pat, + UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, UPLURAL_TYPE_CARDINAL, status); + applyPattern(pat, status); +} + +PluralFormat::PluralFormat(const PluralRules& rules, + const UnicodeString& pat, + UErrorCode& status) + : locale(Locale::getDefault()), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(&rules, UPLURAL_TYPE_COUNT, status); + applyPattern(pat, status); +} + +PluralFormat::PluralFormat(const Locale& loc, + const PluralRules& rules, + const UnicodeString& pat, + UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(&rules, UPLURAL_TYPE_COUNT, status); + applyPattern(pat, status); +} + +PluralFormat::PluralFormat(const Locale& loc, + UPluralType type, + const UnicodeString& pat, + UErrorCode& status) + : locale(loc), + msgPattern(status), + numberFormat(nullptr), + offset(0) { + init(nullptr, type, status); + applyPattern(pat, status); +} + +PluralFormat::PluralFormat(const PluralFormat& other) + : Format(other), + locale(other.locale), + msgPattern(other.msgPattern), + numberFormat(nullptr), + offset(other.offset) { + copyObjects(other); +} + +void +PluralFormat::copyObjects(const PluralFormat& other) { + UErrorCode status = U_ZERO_ERROR; + if (numberFormat != nullptr) { + delete numberFormat; + } + if (pluralRulesWrapper.pluralRules != nullptr) { + delete pluralRulesWrapper.pluralRules; + } + + if (other.numberFormat == nullptr) { + numberFormat = NumberFormat::createInstance(locale, status); + } else { + numberFormat = other.numberFormat->clone(); + } + if (other.pluralRulesWrapper.pluralRules == nullptr) { + pluralRulesWrapper.pluralRules = PluralRules::forLocale(locale, status); + } else { + pluralRulesWrapper.pluralRules = other.pluralRulesWrapper.pluralRules->clone(); + } +} + + +PluralFormat::~PluralFormat() { + delete numberFormat; +} + +void +PluralFormat::init(const PluralRules* rules, UPluralType type, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + + if (rules==nullptr) { + pluralRulesWrapper.pluralRules = PluralRules::forLocale(locale, type, status); + } else { + pluralRulesWrapper.pluralRules = rules->clone(); + if (pluralRulesWrapper.pluralRules == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + numberFormat= NumberFormat::createInstance(locale, status); +} + +void +PluralFormat::applyPattern(const UnicodeString& newPattern, UErrorCode& status) { + msgPattern.parsePluralStyle(newPattern, nullptr, status); + if (U_FAILURE(status)) { + msgPattern.clear(); + offset = 0; + return; + } + offset = msgPattern.getPluralOffset(0); +} + +UnicodeString& +PluralFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const +{ + if (U_FAILURE(status)) return appendTo; + + if (obj.isNumeric()) { + return format(obj, obj.getDouble(), appendTo, pos, status); + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return appendTo; + } +} + +UnicodeString +PluralFormat::format(int32_t number, UErrorCode& status) const { + FieldPosition fpos(FieldPosition::DONT_CARE); + UnicodeString result; + return format(Formattable(number), number, result, fpos, status); +} + +UnicodeString +PluralFormat::format(double number, UErrorCode& status) const { + FieldPosition fpos(FieldPosition::DONT_CARE); + UnicodeString result; + return format(Formattable(number), number, result, fpos, status); +} + + +UnicodeString& +PluralFormat::format(int32_t number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const { + return format(Formattable(number), (double)number, appendTo, pos, status); +} + +UnicodeString& +PluralFormat::format(double number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const { + return format(Formattable(number), (double)number, appendTo, pos, status); +} + +UnicodeString& +PluralFormat::format(const Formattable& numberObject, double number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return appendTo; + } + if (msgPattern.countParts() == 0) { + return numberFormat->format(numberObject, appendTo, pos, status); + } + + // Get the appropriate sub-message. + // Select it based on the formatted number-offset. + double numberMinusOffset = number - offset; + // Call NumberFormatter to get both the DecimalQuantity and the string. + // This call site needs to use more internal APIs than the Java equivalent. + number::impl::UFormattedNumberData data; + if (offset == 0) { + // could be BigDecimal etc. + numberObject.populateDecimalQuantity(data.quantity, status); + } else { + data.quantity.setToDouble(numberMinusOffset); + } + UnicodeString numberString; + auto *decFmt = dynamic_cast(numberFormat); + if(decFmt != nullptr) { + const number::LocalizedNumberFormatter* lnf = decFmt->toNumberFormatter(status); + if (U_FAILURE(status)) { + return appendTo; + } + lnf->formatImpl(&data, status); // mutates &data + if (U_FAILURE(status)) { + return appendTo; + } + numberString = data.getStringRef().toUnicodeString(); + } else { + if (offset == 0) { + numberFormat->format(numberObject, numberString, status); + } else { + numberFormat->format(numberMinusOffset, numberString, status); + } + } + + int32_t partIndex = findSubMessage(msgPattern, 0, pluralRulesWrapper, &data.quantity, number, status); + if (U_FAILURE(status)) { return appendTo; } + // Replace syntactic # signs in the top level of this sub-message + // (not in nested arguments) with the formatted number-offset. + const UnicodeString& pattern = msgPattern.getPatternString(); + int32_t prevIndex = msgPattern.getPart(partIndex).getLimit(); + for (;;) { + const MessagePattern::Part& part = msgPattern.getPart(++partIndex); + const UMessagePatternPartType type = part.getType(); + int32_t index = part.getIndex(); + if (type == UMSGPAT_PART_TYPE_MSG_LIMIT) { + return appendTo.append(pattern, prevIndex, index - prevIndex); + } else if ((type == UMSGPAT_PART_TYPE_REPLACE_NUMBER) || + (type == UMSGPAT_PART_TYPE_SKIP_SYNTAX && MessageImpl::jdkAposMode(msgPattern))) { + appendTo.append(pattern, prevIndex, index - prevIndex); + if (type == UMSGPAT_PART_TYPE_REPLACE_NUMBER) { + appendTo.append(numberString); + } + prevIndex = part.getLimit(); + } else if (type == UMSGPAT_PART_TYPE_ARG_START) { + appendTo.append(pattern, prevIndex, index - prevIndex); + prevIndex = index; + partIndex = msgPattern.getLimitPartIndex(partIndex); + index = msgPattern.getPart(partIndex).getLimit(); + MessageImpl::appendReducedApostrophes(pattern, prevIndex, index, appendTo); + prevIndex = index; + } + } +} + +UnicodeString& +PluralFormat::toPattern(UnicodeString& appendTo) { + if (0 == msgPattern.countParts()) { + appendTo.setToBogus(); + } else { + appendTo.append(msgPattern.getPatternString()); + } + return appendTo; +} + +void +PluralFormat::setLocale(const Locale& loc, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + locale = loc; + msgPattern.clear(); + delete numberFormat; + offset = 0; + numberFormat = nullptr; + pluralRulesWrapper.reset(); + init(nullptr, UPLURAL_TYPE_CARDINAL, status); +} + +void +PluralFormat::setNumberFormat(const NumberFormat* format, UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + NumberFormat* nf = format->clone(); + if (nf != nullptr) { + delete numberFormat; + numberFormat = nf; + } else { + status = U_MEMORY_ALLOCATION_ERROR; + } +} + +PluralFormat* +PluralFormat::clone() const +{ + return new PluralFormat(*this); +} + + +PluralFormat& +PluralFormat::operator=(const PluralFormat& other) { + if (this != &other) { + locale = other.locale; + msgPattern = other.msgPattern; + offset = other.offset; + copyObjects(other); + } + + return *this; +} + +bool +PluralFormat::operator==(const Format& other) const { + if (this == &other) { + return true; + } + if (!Format::operator==(other)) { + return false; + } + const PluralFormat& o = (const PluralFormat&)other; + return + locale == o.locale && + msgPattern == o.msgPattern && // implies same offset + (numberFormat == nullptr) == (o.numberFormat == nullptr) && + (numberFormat == nullptr || *numberFormat == *o.numberFormat) && + (pluralRulesWrapper.pluralRules == nullptr) == (o.pluralRulesWrapper.pluralRules == nullptr) && + (pluralRulesWrapper.pluralRules == nullptr || + *pluralRulesWrapper.pluralRules == *o.pluralRulesWrapper.pluralRules); +} + +bool +PluralFormat::operator!=(const Format& other) const { + return !operator==(other); +} + +void +PluralFormat::parseObject(const UnicodeString& /*source*/, + Formattable& /*result*/, + ParsePosition& pos) const +{ + // Parsing not supported. + pos.setErrorIndex(pos.getIndex()); +} + +int32_t PluralFormat::findSubMessage(const MessagePattern& pattern, int32_t partIndex, + const PluralSelector& selector, void *context, + double number, UErrorCode& ec) { + if (U_FAILURE(ec)) { + return 0; + } + int32_t count=pattern.countParts(); + double offset; + const MessagePattern::Part* part=&pattern.getPart(partIndex); + if (MessagePattern::Part::hasNumericValue(part->getType())) { + offset=pattern.getNumericValue(*part); + ++partIndex; + } else { + offset=0; + } + // The keyword is empty until we need to match against a non-explicit, not-"other" value. + // Then we get the keyword from the selector. + // (In other words, we never call the selector if we match against an explicit value, + // or if the only non-explicit keyword is "other".) + UnicodeString keyword; + UnicodeString other(false, OTHER_STRING, 5); + // When we find a match, we set msgStart>0 and also set this boolean to true + // to avoid matching the keyword again (duplicates are allowed) + // while we continue to look for an explicit-value match. + UBool haveKeywordMatch=false; + // msgStart is 0 until we find any appropriate sub-message. + // We remember the first "other" sub-message if we have not seen any + // appropriate sub-message before. + // We remember the first matching-keyword sub-message if we have not seen + // one of those before. + // (The parser allows [does not check for] duplicate keywords. + // We just have to make sure to take the first one.) + // We avoid matching the keyword twice by also setting haveKeywordMatch=true + // at the first keyword match. + // We keep going until we find an explicit-value match or reach the end of the plural style. + int32_t msgStart=0; + // Iterate over (ARG_SELECTOR [ARG_INT|ARG_DOUBLE] message) tuples + // until ARG_LIMIT or end of plural-only pattern. + do { + part=&pattern.getPart(partIndex++); + const UMessagePatternPartType type = part->getType(); + if(type==UMSGPAT_PART_TYPE_ARG_LIMIT) { + break; + } + U_ASSERT (type==UMSGPAT_PART_TYPE_ARG_SELECTOR); + // part is an ARG_SELECTOR followed by an optional explicit value, and then a message + if(MessagePattern::Part::hasNumericValue(pattern.getPartType(partIndex))) { + // explicit value like "=2" + part=&pattern.getPart(partIndex++); + if(number==pattern.getNumericValue(*part)) { + // matches explicit value + return partIndex; + } + } else if(!haveKeywordMatch) { + // plural keyword like "few" or "other" + // Compare "other" first and call the selector if this is not "other". + if(pattern.partSubstringMatches(*part, other)) { + if(msgStart==0) { + msgStart=partIndex; + if(0 == keyword.compare(other)) { + // This is the first "other" sub-message, + // and the selected keyword is also "other". + // Do not match "other" again. + haveKeywordMatch=true; + } + } + } else { + if(keyword.isEmpty()) { + keyword=selector.select(context, number-offset, ec); + if(msgStart!=0 && (0 == keyword.compare(other))) { + // We have already seen an "other" sub-message. + // Do not match "other" again. + haveKeywordMatch=true; + // Skip keyword matching but do getLimitPartIndex(). + } + } + if(!haveKeywordMatch && pattern.partSubstringMatches(*part, keyword)) { + // keyword matches + msgStart=partIndex; + // Do not match this keyword again. + haveKeywordMatch=true; + } + } + } + partIndex=pattern.getLimitPartIndex(partIndex); + } while(++partIndexgetType() != UMSGPAT_PART_TYPE_ARG_SELECTOR) { + // Bad format + continue; + } + + const MessagePattern::Part* partStart = &msgPattern.getPart(partIndex++); + if (partStart->getType() != UMSGPAT_PART_TYPE_MSG_START) { + // Bad format + continue; + } + + const MessagePattern::Part* partLimit = &msgPattern.getPart(partIndex++); + if (partLimit->getType() != UMSGPAT_PART_TYPE_MSG_LIMIT) { + // Bad format + continue; + } + + UnicodeString currArg = pattern.tempSubString(partStart->getLimit(), partLimit->getIndex() - partStart->getLimit()); + if (rbnfLenientScanner != nullptr) { + // Check if non-lenient rule finds the text before call lenient parsing + int32_t tempIndex = source.indexOf(currArg, startingAt); + if (tempIndex >= 0) { + currMatchIndex = tempIndex; + } else { + // If lenient parsing is turned ON, we've got some time consuming parsing ahead of us. + int32_t length = -1; + currMatchIndex = rbnfLenientScanner->findTextLenient(source, currArg, startingAt, &length); + } + } + else { + currMatchIndex = source.indexOf(currArg, startingAt); + } + if (currMatchIndex >= 0 && currMatchIndex >= matchedIndex && currArg.length() > matchedWord.length()) { + matchedIndex = currMatchIndex; + matchedWord = currArg; + keyword = pattern.tempSubString(partStart->getLimit(), partLimit->getIndex() - partStart->getLimit()); + } + } + if (matchedIndex >= 0) { + pos.setBeginIndex(matchedIndex); + pos.setEndIndex(matchedIndex + matchedWord.length()); + result.setString(keyword); + return; + } + + // Not found! + pos.setBeginIndex(-1); + pos.setEndIndex(-1); +} + +PluralFormat::PluralSelector::~PluralSelector() {} + +PluralFormat::PluralSelectorAdapter::~PluralSelectorAdapter() { + delete pluralRules; +} + +UnicodeString PluralFormat::PluralSelectorAdapter::select(void *context, double number, + UErrorCode& /*ec*/) const { + (void)number; // unused except in the assertion + IFixedDecimal *dec=static_cast(context); + return pluralRules->select(*dec); +} + +void PluralFormat::PluralSelectorAdapter::reset() { + delete pluralRules; + pluralRules = nullptr; +} + + +U_NAMESPACE_END + + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/plurrule.cpp b/intl/icu/source/i18n/plurrule.cpp new file mode 100644 index 0000000000..9c37b09e25 --- /dev/null +++ b/intl/icu/source/i18n/plurrule.cpp @@ -0,0 +1,2006 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File plurrule.cpp +*/ + +#include +#include + +#include "unicode/utypes.h" +#include "unicode/localpointer.h" +#include "unicode/plurrule.h" +#include "unicode/upluralrules.h" +#include "unicode/ures.h" +#include "unicode/numfmt.h" +#include "unicode/decimfmt.h" +#include "unicode/numberrangeformatter.h" +#include "charstr.h" +#include "cmemory.h" +#include "cstring.h" +#include "hash.h" +#include "locutil.h" +#include "mutex.h" +#include "number_decnum.h" +#include "patternprops.h" +#include "plurrule_impl.h" +#include "putilimp.h" +#include "ucln_in.h" +#include "ustrfmt.h" +#include "uassert.h" +#include "uvectr32.h" +#include "sharedpluralrules.h" +#include "unifiedcache.h" +#include "number_decimalquantity.h" +#include "util.h" +#include "pluralranges.h" +#include "numrange_impl.h" + +#if !UCONFIG_NO_FORMATTING + +U_NAMESPACE_BEGIN + +using namespace icu::pluralimpl; +using icu::number::impl::DecNum; +using icu::number::impl::DecimalQuantity; +using icu::number::impl::RoundingMode; + +static const char16_t PLURAL_KEYWORD_OTHER[]={LOW_O,LOW_T,LOW_H,LOW_E,LOW_R,0}; +static const char16_t PLURAL_DEFAULT_RULE[]={LOW_O,LOW_T,LOW_H,LOW_E,LOW_R,COLON,SPACE,LOW_N,0}; +static const char16_t PK_IN[]={LOW_I,LOW_N,0}; +static const char16_t PK_NOT[]={LOW_N,LOW_O,LOW_T,0}; +static const char16_t PK_IS[]={LOW_I,LOW_S,0}; +static const char16_t PK_MOD[]={LOW_M,LOW_O,LOW_D,0}; +static const char16_t PK_AND[]={LOW_A,LOW_N,LOW_D,0}; +static const char16_t PK_OR[]={LOW_O,LOW_R,0}; +static const char16_t PK_VAR_N[]={LOW_N,0}; +static const char16_t PK_VAR_I[]={LOW_I,0}; +static const char16_t PK_VAR_F[]={LOW_F,0}; +static const char16_t PK_VAR_T[]={LOW_T,0}; +static const char16_t PK_VAR_E[]={LOW_E,0}; +static const char16_t PK_VAR_C[]={LOW_C,0}; +static const char16_t PK_VAR_V[]={LOW_V,0}; +static const char16_t PK_WITHIN[]={LOW_W,LOW_I,LOW_T,LOW_H,LOW_I,LOW_N,0}; +static const char16_t PK_DECIMAL[]={LOW_D,LOW_E,LOW_C,LOW_I,LOW_M,LOW_A,LOW_L,0}; +static const char16_t PK_INTEGER[]={LOW_I,LOW_N,LOW_T,LOW_E,LOW_G,LOW_E,LOW_R,0}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(PluralRules) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(PluralKeywordEnumeration) + +PluralRules::PluralRules(UErrorCode& /*status*/) +: UObject(), + mRules(nullptr), + mStandardPluralRanges(nullptr), + mInternalStatus(U_ZERO_ERROR) +{ +} + +PluralRules::PluralRules(const PluralRules& other) +: UObject(other), + mRules(nullptr), + mStandardPluralRanges(nullptr), + mInternalStatus(U_ZERO_ERROR) +{ + *this=other; +} + +PluralRules::~PluralRules() { + delete mRules; + delete mStandardPluralRanges; +} + +SharedPluralRules::~SharedPluralRules() { + delete ptr; +} + +PluralRules* +PluralRules::clone() const { + // Since clone doesn't have a 'status' parameter, the best we can do is return nullptr if + // the newly created object was not fully constructed properly (an error occurred). + UErrorCode localStatus = U_ZERO_ERROR; + return clone(localStatus); +} + +PluralRules* +PluralRules::clone(UErrorCode& status) const { + LocalPointer newObj(new PluralRules(*this), status); + if (U_SUCCESS(status) && U_FAILURE(newObj->mInternalStatus)) { + status = newObj->mInternalStatus; + newObj.adoptInstead(nullptr); + } + return newObj.orphan(); +} + +PluralRules& +PluralRules::operator=(const PluralRules& other) { + if (this != &other) { + delete mRules; + mRules = nullptr; + delete mStandardPluralRanges; + mStandardPluralRanges = nullptr; + mInternalStatus = other.mInternalStatus; + if (U_FAILURE(mInternalStatus)) { + // bail out early if the object we were copying from was already 'invalid'. + return *this; + } + if (other.mRules != nullptr) { + mRules = new RuleChain(*other.mRules); + if (mRules == nullptr) { + mInternalStatus = U_MEMORY_ALLOCATION_ERROR; + } + else if (U_FAILURE(mRules->fInternalStatus)) { + // If the RuleChain wasn't fully copied, then set our status to failure as well. + mInternalStatus = mRules->fInternalStatus; + } + } + if (other.mStandardPluralRanges != nullptr) { + mStandardPluralRanges = other.mStandardPluralRanges->copy(mInternalStatus) + .toPointer(mInternalStatus) + .orphan(); + } + } + return *this; +} + +StringEnumeration* PluralRules::getAvailableLocales(UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result(new PluralAvailableLocalesEnumeration(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + return result.orphan(); +} + + +PluralRules* U_EXPORT2 +PluralRules::createRules(const UnicodeString& description, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + PluralRuleParser parser; + LocalPointer newRules(new PluralRules(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + parser.parse(description, newRules.getAlias(), status); + if (U_FAILURE(status)) { + newRules.adoptInstead(nullptr); + } + return newRules.orphan(); +} + + +PluralRules* U_EXPORT2 +PluralRules::createDefaultRules(UErrorCode& status) { + return createRules(UnicodeString(true, PLURAL_DEFAULT_RULE, -1), status); +} + +/******************************************************************************/ +/* Create PluralRules cache */ + +template<> U_I18N_API +const SharedPluralRules *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); + LocalPointer pr(PluralRules::internalForLocale(localeId, UPLURAL_TYPE_CARDINAL, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result(new SharedPluralRules(pr.getAlias()), status); + if (U_FAILURE(status)) { + return nullptr; + } + pr.orphan(); // result was successfully created so it nows pr. + result->addRef(); + return result.orphan(); +} + +/* end plural rules cache */ +/******************************************************************************/ + +const SharedPluralRules* U_EXPORT2 +PluralRules::createSharedInstance( + const Locale& locale, UPluralType type, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + if (type != UPLURAL_TYPE_CARDINAL) { + status = U_UNSUPPORTED_ERROR; + return nullptr; + } + const SharedPluralRules *result = nullptr; + UnifiedCache::getByLocale(locale, result, status); + return result; +} + +PluralRules* U_EXPORT2 +PluralRules::forLocale(const Locale& locale, UErrorCode& status) { + return forLocale(locale, UPLURAL_TYPE_CARDINAL, status); +} + +PluralRules* U_EXPORT2 +PluralRules::forLocale(const Locale& locale, UPluralType type, UErrorCode& status) { + if (type != UPLURAL_TYPE_CARDINAL) { + return internalForLocale(locale, type, status); + } + const SharedPluralRules *shared = createSharedInstance( + locale, type, status); + if (U_FAILURE(status)) { + return nullptr; + } + PluralRules *result = (*shared)->clone(status); + shared->removeRef(); + return result; +} + +PluralRules* U_EXPORT2 +PluralRules::internalForLocale(const Locale& locale, UPluralType type, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + if (type >= UPLURAL_TYPE_COUNT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + LocalPointer newObj(new PluralRules(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + UnicodeString locRule = newObj->getRuleFromResource(locale, type, status); + // TODO: which other errors, if any, should be returned? + if (locRule.length() == 0) { + // If an out-of-memory error occurred, then stop and report the failure. + if (status == U_MEMORY_ALLOCATION_ERROR) { + return nullptr; + } + // Locales with no specific rules (all numbers have the "other" category + // will return a U_MISSING_RESOURCE_ERROR at this point. This is not + // an error. + locRule = UnicodeString(PLURAL_DEFAULT_RULE); + status = U_ZERO_ERROR; + } + PluralRuleParser parser; + parser.parse(locRule, newObj.getAlias(), status); + // TODO: should rule parse errors be returned, or + // should we silently use default rules? + // Original impl used default rules. + // Ask the question to ICU Core. + + newObj->mStandardPluralRanges = StandardPluralRanges::forLocale(locale, status) + .toPointer(status) + .orphan(); + + return newObj.orphan(); +} + +UnicodeString +PluralRules::select(int32_t number) const { + return select(FixedDecimal(number)); +} + +UnicodeString +PluralRules::select(double number) const { + return select(FixedDecimal(number)); +} + +UnicodeString +PluralRules::select(const number::FormattedNumber& number, UErrorCode& status) const { + DecimalQuantity dq; + number.getDecimalQuantity(dq, status); + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + if (U_FAILURE(mInternalStatus)) { + status = mInternalStatus; + return ICU_Utility::makeBogusString(); + } + return select(dq); +} + +UnicodeString +PluralRules::select(const IFixedDecimal &number) const { + if (mRules == nullptr) { + return UnicodeString(true, PLURAL_DEFAULT_RULE, -1); + } + else { + return mRules->select(number); + } +} + +UnicodeString +PluralRules::select(const number::FormattedNumberRange& range, UErrorCode& status) const { + return select(range.getData(status), status); +} + +UnicodeString +PluralRules::select(const number::impl::UFormattedNumberRangeData* impl, UErrorCode& status) const { + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + if (U_FAILURE(mInternalStatus)) { + status = mInternalStatus; + return ICU_Utility::makeBogusString(); + } + if (mStandardPluralRanges == nullptr) { + // Happens if PluralRules was constructed via createRules() + status = U_UNSUPPORTED_ERROR; + return ICU_Utility::makeBogusString(); + } + auto form1 = StandardPlural::fromString(select(impl->quantity1), status); + auto form2 = StandardPlural::fromString(select(impl->quantity2), status); + if (U_FAILURE(status)) { + return ICU_Utility::makeBogusString(); + } + auto result = mStandardPluralRanges->resolve(form1, form2); + return UnicodeString(StandardPlural::getKeyword(result), -1, US_INV); +} + + +StringEnumeration* +PluralRules::getKeywords(UErrorCode& status) const { + if (U_FAILURE(status)) { + return nullptr; + } + if (U_FAILURE(mInternalStatus)) { + status = mInternalStatus; + return nullptr; + } + LocalPointer nameEnumerator(new PluralKeywordEnumeration(mRules, status), status); + if (U_FAILURE(status)) { + return nullptr; + } + return nameEnumerator.orphan(); +} + +double +PluralRules::getUniqueKeywordValue(const UnicodeString& /* keyword */) { + // Not Implemented. + return UPLRULES_NO_UNIQUE_VALUE; +} + +int32_t +PluralRules::getAllKeywordValues(const UnicodeString & /* keyword */, double * /* dest */, + int32_t /* destCapacity */, UErrorCode& error) { + error = U_UNSUPPORTED_ERROR; + return 0; +} + +/** + * Helper method for the overrides of getSamples() for double and DecimalQuantity + * return value types. Provide only one of an allocated array of double or + * DecimalQuantity, and a nullptr for the other. + */ +static int32_t +getSamplesFromString(const UnicodeString &samples, double *destDbl, + DecimalQuantity* destDq, int32_t destCapacity, + UErrorCode& status) { + + if ((destDbl == nullptr && destDq == nullptr) + || (destDbl != nullptr && destDq != nullptr)) { + status = U_INTERNAL_PROGRAM_ERROR; + return 0; + } + + bool isDouble = destDbl != nullptr; + int32_t sampleCount = 0; + int32_t sampleStartIdx = 0; + int32_t sampleEndIdx = 0; + + //std::string ss; // TODO: debugging. + // std::cout << "PluralRules::getSamples(), samples = \"" << samples.toUTF8String(ss) << "\"\n"; + for (sampleCount = 0; sampleCount < destCapacity && sampleStartIdx < samples.length(); ) { + sampleEndIdx = samples.indexOf(COMMA, sampleStartIdx); + if (sampleEndIdx == -1) { + sampleEndIdx = samples.length(); + } + const UnicodeString &sampleRange = samples.tempSubStringBetween(sampleStartIdx, sampleEndIdx); + // ss.erase(); + // std::cout << "PluralRules::getSamples(), samplesRange = \"" << sampleRange.toUTF8String(ss) << "\"\n"; + int32_t tildeIndex = sampleRange.indexOf(TILDE); + if (tildeIndex < 0) { + DecimalQuantity dq = DecimalQuantity::fromExponentString(sampleRange, status); + if (isDouble) { + // See warning note below about lack of precision for floating point samples for numbers with + // trailing zeroes in the decimal fraction representation. + double dblValue = dq.toDouble(); + if (!(dblValue == floor(dblValue) && dq.fractionCount() > 0)) { + destDbl[sampleCount++] = dblValue; + } + } else { + destDq[sampleCount++] = dq; + } + } else { + DecimalQuantity rangeLo = + DecimalQuantity::fromExponentString(sampleRange.tempSubStringBetween(0, tildeIndex), status); + DecimalQuantity rangeHi = DecimalQuantity::fromExponentString(sampleRange.tempSubStringBetween(tildeIndex+1), status); + if (U_FAILURE(status)) { + break; + } + if (rangeHi.toDouble() < rangeLo.toDouble()) { + status = U_INVALID_FORMAT_ERROR; + break; + } + + DecimalQuantity incrementDq; + incrementDq.setToInt(1); + int32_t lowerDispMag = rangeLo.getLowerDisplayMagnitude(); + int32_t exponent = rangeLo.getExponent(); + int32_t incrementScale = lowerDispMag + exponent; + incrementDq.adjustMagnitude(incrementScale); + double incrementVal = incrementDq.toDouble(); // 10 ^ incrementScale + + + DecimalQuantity dq(rangeLo); + double dblValue = dq.toDouble(); + double end = rangeHi.toDouble(); + + while (dblValue <= end) { + if (isDouble) { + // Hack Alert: don't return any decimal samples with integer values that + // originated from a format with trailing decimals. + // This API is returning doubles, which can't distinguish having displayed + // zeros to the right of the decimal. + // This results in test failures with values mapping back to a different keyword. + if (!(dblValue == floor(dblValue) && dq.fractionCount() > 0)) { + destDbl[sampleCount++] = dblValue; + } + } else { + destDq[sampleCount++] = dq; + } + if (sampleCount >= destCapacity) { + break; + } + + // Increment dq for next iteration + + // Because DecNum and DecimalQuantity do not support + // add operations, we need to convert to/from double, + // despite precision lossiness for decimal fractions like 0.1. + dblValue += incrementVal; + DecNum newDqDecNum; + newDqDecNum.setTo(dblValue, status); + DecimalQuantity newDq; + newDq.setToDecNum(newDqDecNum, status); + newDq.setMinFraction(-lowerDispMag); + newDq.roundToMagnitude(lowerDispMag, RoundingMode::UNUM_ROUND_HALFEVEN, status); + newDq.adjustMagnitude(-exponent); + newDq.adjustExponent(exponent); + dblValue = newDq.toDouble(); + dq = newDq; + } + } + sampleStartIdx = sampleEndIdx + 1; + } + return sampleCount; +} + +int32_t +PluralRules::getSamples(const UnicodeString &keyword, double *dest, + int32_t destCapacity, UErrorCode& status) { + if (U_FAILURE(status)) { + return 0; + } + if (U_FAILURE(mInternalStatus)) { + status = mInternalStatus; + return 0; + } + if (dest != nullptr ? destCapacity < 0 : destCapacity != 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + RuleChain *rc = rulesForKeyword(keyword); + if (rc == nullptr) { + return 0; + } + int32_t numSamples = getSamplesFromString(rc->fIntegerSamples, dest, nullptr, destCapacity, status); + if (numSamples == 0) { + numSamples = getSamplesFromString(rc->fDecimalSamples, dest, nullptr, destCapacity, status); + } + return numSamples; +} + +int32_t +PluralRules::getSamples(const UnicodeString &keyword, DecimalQuantity *dest, + int32_t destCapacity, UErrorCode& status) { + if (U_FAILURE(status)) { + return 0; + } + if (U_FAILURE(mInternalStatus)) { + status = mInternalStatus; + return 0; + } + if (dest != nullptr ? destCapacity < 0 : destCapacity != 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + RuleChain *rc = rulesForKeyword(keyword); + if (rc == nullptr) { + return 0; + } + + int32_t numSamples = getSamplesFromString(rc->fIntegerSamples, nullptr, dest, destCapacity, status); + if (numSamples == 0) { + numSamples = getSamplesFromString(rc->fDecimalSamples, nullptr, dest, destCapacity, status); + } + return numSamples; +} + + +RuleChain *PluralRules::rulesForKeyword(const UnicodeString &keyword) const { + RuleChain *rc; + for (rc = mRules; rc != nullptr; rc = rc->fNext) { + if (rc->fKeyword == keyword) { + break; + } + } + return rc; +} + + +UBool +PluralRules::isKeyword(const UnicodeString& keyword) const { + if (0 == keyword.compare(PLURAL_KEYWORD_OTHER, 5)) { + return true; + } + return rulesForKeyword(keyword) != nullptr; +} + +UnicodeString +PluralRules::getKeywordOther() const { + return UnicodeString(true, PLURAL_KEYWORD_OTHER, 5); +} + +bool +PluralRules::operator==(const PluralRules& other) const { + const UnicodeString *ptrKeyword; + UErrorCode status= U_ZERO_ERROR; + + if ( this == &other ) { + return true; + } + LocalPointer myKeywordList(getKeywords(status)); + LocalPointer otherKeywordList(other.getKeywords(status)); + if (U_FAILURE(status)) { + return false; + } + + if (myKeywordList->count(status)!=otherKeywordList->count(status)) { + return false; + } + myKeywordList->reset(status); + while ((ptrKeyword=myKeywordList->snext(status))!=nullptr) { + if (!other.isKeyword(*ptrKeyword)) { + return false; + } + } + otherKeywordList->reset(status); + while ((ptrKeyword=otherKeywordList->snext(status))!=nullptr) { + if (!this->isKeyword(*ptrKeyword)) { + return false; + } + } + if (U_FAILURE(status)) { + return false; + } + + return true; +} + + +void +PluralRuleParser::parse(const UnicodeString& ruleData, PluralRules *prules, UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + U_ASSERT(ruleIndex == 0); // Parsers are good for a single use only! + ruleSrc = &ruleData; + + while (ruleIndex< ruleSrc->length()) { + getNextToken(status); + if (U_FAILURE(status)) { + return; + } + checkSyntax(status); + if (U_FAILURE(status)) { + return; + } + switch (type) { + case tAnd: + U_ASSERT(curAndConstraint != nullptr); + curAndConstraint = curAndConstraint->add(status); + break; + case tOr: + { + U_ASSERT(currentChain != nullptr); + OrConstraint *orNode=currentChain->ruleHeader; + while (orNode->next != nullptr) { + orNode = orNode->next; + } + orNode->next= new OrConstraint(); + if (orNode->next == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + orNode=orNode->next; + orNode->next=nullptr; + curAndConstraint = orNode->add(status); + } + break; + case tIs: + U_ASSERT(curAndConstraint != nullptr); + U_ASSERT(curAndConstraint->value == -1); + U_ASSERT(curAndConstraint->rangeList == nullptr); + break; + case tNot: + U_ASSERT(curAndConstraint != nullptr); + curAndConstraint->negated=true; + break; + + case tNotEqual: + curAndConstraint->negated=true; + U_FALLTHROUGH; + case tIn: + case tWithin: + case tEqual: + { + U_ASSERT(curAndConstraint != nullptr); + LocalPointer newRangeList(new UVector32(status), status); + if (U_FAILURE(status)) { + break; + } + curAndConstraint->rangeList = newRangeList.orphan(); + curAndConstraint->rangeList->addElement(-1, status); // range Low + curAndConstraint->rangeList->addElement(-1, status); // range Hi + rangeLowIdx = 0; + rangeHiIdx = 1; + curAndConstraint->value=PLURAL_RANGE_HIGH; + curAndConstraint->integerOnly = (type != tWithin); + } + break; + case tNumber: + U_ASSERT(curAndConstraint != nullptr); + if ( (curAndConstraint->op==AndConstraint::MOD)&& + (curAndConstraint->opNum == -1 ) ) { + curAndConstraint->opNum=getNumberValue(token); + } + else { + if (curAndConstraint->rangeList == nullptr) { + // this is for an 'is' rule + curAndConstraint->value = getNumberValue(token); + } else { + // this is for an 'in' or 'within' rule + if (curAndConstraint->rangeList->elementAti(rangeLowIdx) == -1) { + curAndConstraint->rangeList->setElementAt(getNumberValue(token), rangeLowIdx); + curAndConstraint->rangeList->setElementAt(getNumberValue(token), rangeHiIdx); + } + else { + curAndConstraint->rangeList->setElementAt(getNumberValue(token), rangeHiIdx); + if (curAndConstraint->rangeList->elementAti(rangeLowIdx) > + curAndConstraint->rangeList->elementAti(rangeHiIdx)) { + // Range Lower bound > Range Upper bound. + // U_UNEXPECTED_TOKEN seems a little funny, but it is consistently + // used for all plural rule parse errors. + status = U_UNEXPECTED_TOKEN; + break; + } + } + } + } + break; + case tComma: + // TODO: rule syntax checking is inadequate, can happen with badly formed rules. + // Catch cases like "n mod 10, is 1" here instead. + if (curAndConstraint == nullptr || curAndConstraint->rangeList == nullptr) { + status = U_UNEXPECTED_TOKEN; + break; + } + U_ASSERT(curAndConstraint->rangeList->size() >= 2); + rangeLowIdx = curAndConstraint->rangeList->size(); + curAndConstraint->rangeList->addElement(-1, status); // range Low + rangeHiIdx = curAndConstraint->rangeList->size(); + curAndConstraint->rangeList->addElement(-1, status); // range Hi + break; + case tMod: + U_ASSERT(curAndConstraint != nullptr); + curAndConstraint->op=AndConstraint::MOD; + break; + case tVariableN: + case tVariableI: + case tVariableF: + case tVariableT: + case tVariableE: + case tVariableC: + case tVariableV: + U_ASSERT(curAndConstraint != nullptr); + curAndConstraint->digitsType = type; + break; + case tKeyword: + { + RuleChain *newChain = new RuleChain; + if (newChain == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + newChain->fKeyword = token; + if (prules->mRules == nullptr) { + prules->mRules = newChain; + } else { + // The new rule chain goes at the end of the linked list of rule chains, + // unless there is an "other" keyword & chain. "other" must remain last. + RuleChain *insertAfter = prules->mRules; + while (insertAfter->fNext!=nullptr && + insertAfter->fNext->fKeyword.compare(PLURAL_KEYWORD_OTHER, 5) != 0 ){ + insertAfter=insertAfter->fNext; + } + newChain->fNext = insertAfter->fNext; + insertAfter->fNext = newChain; + } + OrConstraint *orNode = new OrConstraint(); + if (orNode == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + newChain->ruleHeader = orNode; + curAndConstraint = orNode->add(status); + currentChain = newChain; + } + break; + + case tInteger: + for (;;) { + getNextToken(status); + if (U_FAILURE(status) || type == tSemiColon || type == tEOF || type == tAt) { + break; + } + if (type == tEllipsis) { + currentChain->fIntegerSamplesUnbounded = true; + continue; + } + currentChain->fIntegerSamples.append(token); + } + break; + + case tDecimal: + for (;;) { + getNextToken(status); + if (U_FAILURE(status) || type == tSemiColon || type == tEOF || type == tAt) { + break; + } + if (type == tEllipsis) { + currentChain->fDecimalSamplesUnbounded = true; + continue; + } + currentChain->fDecimalSamples.append(token); + } + break; + + default: + break; + } + prevType=type; + if (U_FAILURE(status)) { + break; + } + } +} + +UnicodeString +PluralRules::getRuleFromResource(const Locale& locale, UPluralType type, UErrorCode& errCode) { + UnicodeString emptyStr; + + if (U_FAILURE(errCode)) { + return emptyStr; + } + LocalUResourceBundlePointer rb(ures_openDirect(nullptr, "plurals", &errCode)); + if(U_FAILURE(errCode)) { + return emptyStr; + } + const char *typeKey; + switch (type) { + case UPLURAL_TYPE_CARDINAL: + typeKey = "locales"; + break; + case UPLURAL_TYPE_ORDINAL: + typeKey = "locales_ordinals"; + break; + default: + // Must not occur: The caller should have checked for valid types. + errCode = U_ILLEGAL_ARGUMENT_ERROR; + return emptyStr; + } + LocalUResourceBundlePointer locRes(ures_getByKey(rb.getAlias(), typeKey, nullptr, &errCode)); + if(U_FAILURE(errCode)) { + return emptyStr; + } + int32_t resLen=0; + const char *curLocaleName=locale.getBaseName(); + const char16_t* s = ures_getStringByKey(locRes.getAlias(), curLocaleName, &resLen, &errCode); + + if (s == nullptr) { + // Check parent locales. + UErrorCode status = U_ZERO_ERROR; + char parentLocaleName[ULOC_FULLNAME_CAPACITY]; + const char *curLocaleName2=locale.getBaseName(); + uprv_strcpy(parentLocaleName, curLocaleName2); + + while (uloc_getParent(parentLocaleName, parentLocaleName, + ULOC_FULLNAME_CAPACITY, &status) > 0) { + resLen=0; + s = ures_getStringByKey(locRes.getAlias(), parentLocaleName, &resLen, &status); + if (s != nullptr) { + errCode = U_ZERO_ERROR; + break; + } + status = U_ZERO_ERROR; + } + } + if (s==nullptr) { + return emptyStr; + } + + char setKey[256]; + u_UCharsToChars(s, setKey, resLen + 1); + // printf("\n PluralRule: %s\n", setKey); + + LocalUResourceBundlePointer ruleRes(ures_getByKey(rb.getAlias(), "rules", nullptr, &errCode)); + if(U_FAILURE(errCode)) { + return emptyStr; + } + LocalUResourceBundlePointer setRes(ures_getByKey(ruleRes.getAlias(), setKey, nullptr, &errCode)); + if (U_FAILURE(errCode)) { + return emptyStr; + } + + int32_t numberKeys = ures_getSize(setRes.getAlias()); + UnicodeString result; + const char *key=nullptr; + for(int32_t i=0; idumpRules(rules); + } + return rules; +} + +AndConstraint::AndConstraint(const AndConstraint& other) { + this->fInternalStatus = other.fInternalStatus; + if (U_FAILURE(fInternalStatus)) { + return; // stop early if the object we are copying from is invalid. + } + this->op = other.op; + this->opNum=other.opNum; + this->value=other.value; + if (other.rangeList != nullptr) { + LocalPointer newRangeList(new UVector32(fInternalStatus), fInternalStatus); + if (U_FAILURE(fInternalStatus)) { + return; + } + this->rangeList = newRangeList.orphan(); + this->rangeList->assign(*other.rangeList, fInternalStatus); + } + this->integerOnly=other.integerOnly; + this->negated=other.negated; + this->digitsType = other.digitsType; + if (other.next != nullptr) { + this->next = new AndConstraint(*other.next); + if (this->next == nullptr) { + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + } + } +} + +AndConstraint::~AndConstraint() { + delete rangeList; + rangeList = nullptr; + delete next; + next = nullptr; +} + +UBool +AndConstraint::isFulfilled(const IFixedDecimal &number) { + UBool result = true; + if (digitsType == none) { + // An empty AndConstraint, created by a rule with a keyword but no following expression. + return true; + } + + PluralOperand operand = tokenTypeToPluralOperand(digitsType); + double n = number.getPluralOperand(operand); // pulls n | i | v | f value for the number. + // Will always be positive. + // May be non-integer (n option only) + do { + if (integerOnly && n != uprv_floor(n)) { + result = false; + break; + } + + if (op == MOD) { + n = fmod(n, opNum); + } + if (rangeList == nullptr) { + result = value == -1 || // empty rule + n == value; // 'is' rule + break; + } + result = false; // 'in' or 'within' rule + for (int32_t r=0; rsize(); r+=2) { + if (rangeList->elementAti(r) <= n && n <= rangeList->elementAti(r+1)) { + result = true; + break; + } + } + } while (false); + + if (negated) { + result = !result; + } + return result; +} + +AndConstraint* +AndConstraint::add(UErrorCode& status) { + if (U_FAILURE(fInternalStatus)) { + status = fInternalStatus; + return nullptr; + } + this->next = new AndConstraint(); + if (this->next == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return this->next; +} + + +OrConstraint::OrConstraint(const OrConstraint& other) { + this->fInternalStatus = other.fInternalStatus; + if (U_FAILURE(fInternalStatus)) { + return; // stop early if the object we are copying from is invalid. + } + if ( other.childNode != nullptr ) { + this->childNode = new AndConstraint(*(other.childNode)); + if (this->childNode == nullptr) { + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + if (other.next != nullptr ) { + this->next = new OrConstraint(*(other.next)); + if (this->next == nullptr) { + fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (U_FAILURE(this->next->fInternalStatus)) { + this->fInternalStatus = this->next->fInternalStatus; + } + } +} + +OrConstraint::~OrConstraint() { + delete childNode; + childNode = nullptr; + delete next; + next = nullptr; +} + +AndConstraint* +OrConstraint::add(UErrorCode& status) { + if (U_FAILURE(fInternalStatus)) { + status = fInternalStatus; + return nullptr; + } + OrConstraint *curOrConstraint=this; + { + while (curOrConstraint->next!=nullptr) { + curOrConstraint = curOrConstraint->next; + } + U_ASSERT(curOrConstraint->childNode == nullptr); + curOrConstraint->childNode = new AndConstraint(); + if (curOrConstraint->childNode == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + } + return curOrConstraint->childNode; +} + +UBool +OrConstraint::isFulfilled(const IFixedDecimal &number) { + OrConstraint* orRule=this; + UBool result=false; + + while (orRule!=nullptr && !result) { + result=true; + AndConstraint* andRule = orRule->childNode; + while (andRule!=nullptr && result) { + result = andRule->isFulfilled(number); + andRule=andRule->next; + } + orRule = orRule->next; + } + + return result; +} + + +RuleChain::RuleChain(const RuleChain& other) : + fKeyword(other.fKeyword), fDecimalSamples(other.fDecimalSamples), + fIntegerSamples(other.fIntegerSamples), fDecimalSamplesUnbounded(other.fDecimalSamplesUnbounded), + fIntegerSamplesUnbounded(other.fIntegerSamplesUnbounded), fInternalStatus(other.fInternalStatus) { + if (U_FAILURE(this->fInternalStatus)) { + return; // stop early if the object we are copying from is invalid. + } + if (other.ruleHeader != nullptr) { + this->ruleHeader = new OrConstraint(*(other.ruleHeader)); + if (this->ruleHeader == nullptr) { + this->fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + } + else if (U_FAILURE(this->ruleHeader->fInternalStatus)) { + // If the OrConstraint wasn't fully copied, then set our status to failure as well. + this->fInternalStatus = this->ruleHeader->fInternalStatus; + return; // exit early. + } + } + if (other.fNext != nullptr ) { + this->fNext = new RuleChain(*other.fNext); + if (this->fNext == nullptr) { + this->fInternalStatus = U_MEMORY_ALLOCATION_ERROR; + } + else if (U_FAILURE(this->fNext->fInternalStatus)) { + // If the RuleChain wasn't fully copied, then set our status to failure as well. + this->fInternalStatus = this->fNext->fInternalStatus; + } + } +} + +RuleChain::~RuleChain() { + delete fNext; + delete ruleHeader; +} + +UnicodeString +RuleChain::select(const IFixedDecimal &number) const { + if (!number.isNaN() && !number.isInfinite()) { + for (const RuleChain *rules = this; rules != nullptr; rules = rules->fNext) { + if (rules->ruleHeader->isFulfilled(number)) { + return rules->fKeyword; + } + } + } + return UnicodeString(true, PLURAL_KEYWORD_OTHER, 5); +} + +static UnicodeString tokenString(tokenType tok) { + UnicodeString s; + switch (tok) { + case tVariableN: + s.append(LOW_N); break; + case tVariableI: + s.append(LOW_I); break; + case tVariableF: + s.append(LOW_F); break; + case tVariableV: + s.append(LOW_V); break; + case tVariableT: + s.append(LOW_T); break; + case tVariableE: + s.append(LOW_E); break; + case tVariableC: + s.append(LOW_C); break; + default: + s.append(TILDE); + } + return s; +} + +void +RuleChain::dumpRules(UnicodeString& result) { + char16_t digitString[16]; + + if ( ruleHeader != nullptr ) { + result += fKeyword; + result += COLON; + result += SPACE; + OrConstraint* orRule=ruleHeader; + while ( orRule != nullptr ) { + AndConstraint* andRule=orRule->childNode; + while ( andRule != nullptr ) { + if ((andRule->op==AndConstraint::NONE) && (andRule->rangeList==nullptr) && (andRule->value == -1)) { + // Empty Rules. + } else if ( (andRule->op==AndConstraint::NONE) && (andRule->rangeList==nullptr) ) { + result += tokenString(andRule->digitsType); + result += UNICODE_STRING_SIMPLE(" is "); + if (andRule->negated) { + result += UNICODE_STRING_SIMPLE("not "); + } + uprv_itou(digitString,16, andRule->value,10,0); + result += UnicodeString(digitString); + } + else { + result += tokenString(andRule->digitsType); + result += SPACE; + if (andRule->op==AndConstraint::MOD) { + result += UNICODE_STRING_SIMPLE("mod "); + uprv_itou(digitString,16, andRule->opNum,10,0); + result += UnicodeString(digitString); + } + if (andRule->rangeList==nullptr) { + if (andRule->negated) { + result += UNICODE_STRING_SIMPLE(" is not "); + uprv_itou(digitString,16, andRule->value,10,0); + result += UnicodeString(digitString); + } + else { + result += UNICODE_STRING_SIMPLE(" is "); + uprv_itou(digitString,16, andRule->value,10,0); + result += UnicodeString(digitString); + } + } + else { + if (andRule->negated) { + if ( andRule->integerOnly ) { + result += UNICODE_STRING_SIMPLE(" not in "); + } + else { + result += UNICODE_STRING_SIMPLE(" not within "); + } + } + else { + if ( andRule->integerOnly ) { + result += UNICODE_STRING_SIMPLE(" in "); + } + else { + result += UNICODE_STRING_SIMPLE(" within "); + } + } + for (int32_t r=0; rrangeList->size(); r+=2) { + int32_t rangeLo = andRule->rangeList->elementAti(r); + int32_t rangeHi = andRule->rangeList->elementAti(r+1); + uprv_itou(digitString,16, rangeLo, 10, 0); + result += UnicodeString(digitString); + result += UNICODE_STRING_SIMPLE(".."); + uprv_itou(digitString,16, rangeHi, 10,0); + result += UnicodeString(digitString); + if (r+2 < andRule->rangeList->size()) { + result += UNICODE_STRING_SIMPLE(", "); + } + } + } + } + if ( (andRule=andRule->next) != nullptr) { + result += UNICODE_STRING_SIMPLE(" and "); + } + } + if ( (orRule = orRule->next) != nullptr ) { + result += UNICODE_STRING_SIMPLE(" or "); + } + } + } + if ( fNext != nullptr ) { + result += UNICODE_STRING_SIMPLE("; "); + fNext->dumpRules(result); + } +} + + +UErrorCode +RuleChain::getKeywords(int32_t capacityOfKeywords, UnicodeString* keywords, int32_t& arraySize) const { + if (U_FAILURE(fInternalStatus)) { + return fInternalStatus; + } + if ( arraySize < capacityOfKeywords-1 ) { + keywords[arraySize++]=fKeyword; + } + else { + return U_BUFFER_OVERFLOW_ERROR; + } + + if ( fNext != nullptr ) { + return fNext->getKeywords(capacityOfKeywords, keywords, arraySize); + } + else { + return U_ZERO_ERROR; + } +} + +UBool +RuleChain::isKeyword(const UnicodeString& keywordParam) const { + if ( fKeyword == keywordParam ) { + return true; + } + + if ( fNext != nullptr ) { + return fNext->isKeyword(keywordParam); + } + else { + return false; + } +} + + +PluralRuleParser::PluralRuleParser() : + ruleIndex(0), token(), type(none), prevType(none), + curAndConstraint(nullptr), currentChain(nullptr), rangeLowIdx(-1), rangeHiIdx(-1) +{ +} + +PluralRuleParser::~PluralRuleParser() { +} + + +int32_t +PluralRuleParser::getNumberValue(const UnicodeString& token) { + int32_t i; + char digits[128]; + + i = token.extract(0, token.length(), digits, UPRV_LENGTHOF(digits), US_INV); + digits[i]='\0'; + + return((int32_t)atoi(digits)); +} + + +void +PluralRuleParser::checkSyntax(UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + if (!(prevType==none || prevType==tSemiColon)) { + type = getKeyType(token, type); // Switch token type from tKeyword if we scanned a reserved word, + // and we are not at the start of a rule, where a + // keyword is expected. + } + + switch(prevType) { + case none: + case tSemiColon: + if (type!=tKeyword && type != tEOF) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tVariableN: + case tVariableI: + case tVariableF: + case tVariableT: + case tVariableE: + case tVariableC: + case tVariableV: + if (type != tIs && type != tMod && type != tIn && + type != tNot && type != tWithin && type != tEqual && type != tNotEqual) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tKeyword: + if (type != tColon) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tColon: + if (!(type == tVariableN || + type == tVariableI || + type == tVariableF || + type == tVariableT || + type == tVariableE || + type == tVariableC || + type == tVariableV || + type == tAt)) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tIs: + if ( type != tNumber && type != tNot) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tNot: + if (type != tNumber && type != tIn && type != tWithin) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tMod: + case tDot2: + case tIn: + case tWithin: + case tEqual: + case tNotEqual: + if (type != tNumber) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tAnd: + case tOr: + if ( type != tVariableN && + type != tVariableI && + type != tVariableF && + type != tVariableT && + type != tVariableE && + type != tVariableC && + type != tVariableV) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tComma: + if (type != tNumber) { + status = U_UNEXPECTED_TOKEN; + } + break; + case tNumber: + if (type != tDot2 && type != tSemiColon && type != tIs && type != tNot && + type != tIn && type != tEqual && type != tNotEqual && type != tWithin && + type != tAnd && type != tOr && type != tComma && type != tAt && + type != tEOF) + { + status = U_UNEXPECTED_TOKEN; + } + // TODO: a comma following a number that is not part of a range will be allowed. + // It's not the only case of this sort of thing. Parser needs a re-write. + break; + case tAt: + if (type != tDecimal && type != tInteger) { + status = U_UNEXPECTED_TOKEN; + } + break; + default: + status = U_UNEXPECTED_TOKEN; + break; + } +} + + +/* + * Scan the next token from the input rules. + * rules and returned token type are in the parser state variables. + */ +void +PluralRuleParser::getNextToken(UErrorCode &status) +{ + if (U_FAILURE(status)) { + return; + } + + char16_t ch; + while (ruleIndex < ruleSrc->length()) { + ch = ruleSrc->charAt(ruleIndex); + type = charType(ch); + if (type != tSpace) { + break; + } + ++(ruleIndex); + } + if (ruleIndex >= ruleSrc->length()) { + type = tEOF; + return; + } + int32_t curIndex= ruleIndex; + + switch (type) { + case tColon: + case tSemiColon: + case tComma: + case tEllipsis: + case tTilde: // scanned '~' + case tAt: // scanned '@' + case tEqual: // scanned '=' + case tMod: // scanned '%' + // Single character tokens. + ++curIndex; + break; + + case tNotEqual: // scanned '!' + if (ruleSrc->charAt(curIndex+1) == EQUALS) { + curIndex += 2; + } else { + type = none; + curIndex += 1; + } + break; + + case tKeyword: + while (type == tKeyword && ++curIndex < ruleSrc->length()) { + ch = ruleSrc->charAt(curIndex); + type = charType(ch); + } + type = tKeyword; + break; + + case tNumber: + while (type == tNumber && ++curIndex < ruleSrc->length()) { + ch = ruleSrc->charAt(curIndex); + type = charType(ch); + } + type = tNumber; + break; + + case tDot: + // We could be looking at either ".." in a range, or "..." at the end of a sample. + if (curIndex+1 >= ruleSrc->length() || ruleSrc->charAt(curIndex+1) != DOT) { + ++curIndex; + break; // Single dot + } + if (curIndex+2 >= ruleSrc->length() || ruleSrc->charAt(curIndex+2) != DOT) { + curIndex += 2; + type = tDot2; + break; // double dot + } + type = tEllipsis; + curIndex += 3; + break; // triple dot + + default: + status = U_UNEXPECTED_TOKEN; + ++curIndex; + break; + } + + U_ASSERT(ruleIndex <= ruleSrc->length()); + U_ASSERT(curIndex <= ruleSrc->length()); + token=UnicodeString(*ruleSrc, ruleIndex, curIndex-ruleIndex); + ruleIndex = curIndex; +} + +tokenType +PluralRuleParser::charType(char16_t ch) { + if ((ch>=U_ZERO) && (ch<=U_NINE)) { + return tNumber; + } + if (ch>=LOW_A && ch<=LOW_Z) { + return tKeyword; + } + switch (ch) { + case COLON: + return tColon; + case SPACE: + return tSpace; + case SEMI_COLON: + return tSemiColon; + case DOT: + return tDot; + case COMMA: + return tComma; + case EXCLAMATION: + return tNotEqual; + case EQUALS: + return tEqual; + case PERCENT_SIGN: + return tMod; + case AT: + return tAt; + case ELLIPSIS: + return tEllipsis; + case TILDE: + return tTilde; + default : + return none; + } +} + + +// Set token type for reserved words in the Plural Rule syntax. + +tokenType +PluralRuleParser::getKeyType(const UnicodeString &token, tokenType keyType) +{ + if (keyType != tKeyword) { + return keyType; + } + + if (0 == token.compare(PK_VAR_N, 1)) { + keyType = tVariableN; + } else if (0 == token.compare(PK_VAR_I, 1)) { + keyType = tVariableI; + } else if (0 == token.compare(PK_VAR_F, 1)) { + keyType = tVariableF; + } else if (0 == token.compare(PK_VAR_T, 1)) { + keyType = tVariableT; + } else if (0 == token.compare(PK_VAR_E, 1)) { + keyType = tVariableE; + } else if (0 == token.compare(PK_VAR_C, 1)) { + keyType = tVariableC; + } else if (0 == token.compare(PK_VAR_V, 1)) { + keyType = tVariableV; + } else if (0 == token.compare(PK_IS, 2)) { + keyType = tIs; + } else if (0 == token.compare(PK_AND, 3)) { + keyType = tAnd; + } else if (0 == token.compare(PK_IN, 2)) { + keyType = tIn; + } else if (0 == token.compare(PK_WITHIN, 6)) { + keyType = tWithin; + } else if (0 == token.compare(PK_NOT, 3)) { + keyType = tNot; + } else if (0 == token.compare(PK_MOD, 3)) { + keyType = tMod; + } else if (0 == token.compare(PK_OR, 2)) { + keyType = tOr; + } else if (0 == token.compare(PK_DECIMAL, 7)) { + keyType = tDecimal; + } else if (0 == token.compare(PK_INTEGER, 7)) { + keyType = tInteger; + } + return keyType; +} + + +PluralKeywordEnumeration::PluralKeywordEnumeration(RuleChain *header, UErrorCode& status) + : pos(0), fKeywordNames(status) { + if (U_FAILURE(status)) { + return; + } + fKeywordNames.setDeleter(uprv_deleteUObject); + UBool addKeywordOther = true; + RuleChain *node = header; + while (node != nullptr) { + LocalPointer newElem(node->fKeyword.clone(), status); + fKeywordNames.adoptElement(newElem.orphan(), status); + if (U_FAILURE(status)) { + return; + } + if (0 == node->fKeyword.compare(PLURAL_KEYWORD_OTHER, 5)) { + addKeywordOther = false; + } + node = node->fNext; + } + + if (addKeywordOther) { + LocalPointer newElem(new UnicodeString(PLURAL_KEYWORD_OTHER), status); + fKeywordNames.adoptElement(newElem.orphan(), status); + if (U_FAILURE(status)) { + return; + } + } +} + +const UnicodeString* +PluralKeywordEnumeration::snext(UErrorCode& status) { + if (U_SUCCESS(status) && pos < fKeywordNames.size()) { + return (const UnicodeString*)fKeywordNames.elementAt(pos++); + } + return nullptr; +} + +void +PluralKeywordEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +PluralKeywordEnumeration::count(UErrorCode& /*status*/) const { + return fKeywordNames.size(); +} + +PluralKeywordEnumeration::~PluralKeywordEnumeration() { +} + +PluralOperand tokenTypeToPluralOperand(tokenType tt) { + switch(tt) { + case tVariableN: + return PLURAL_OPERAND_N; + case tVariableI: + return PLURAL_OPERAND_I; + case tVariableF: + return PLURAL_OPERAND_F; + case tVariableV: + return PLURAL_OPERAND_V; + case tVariableT: + return PLURAL_OPERAND_T; + case tVariableE: + return PLURAL_OPERAND_E; + case tVariableC: + return PLURAL_OPERAND_E; + default: + UPRV_UNREACHABLE_EXIT; // unexpected. + } +} + +FixedDecimal::FixedDecimal(double n, int32_t v, int64_t f, int32_t e, int32_t c) { + init(n, v, f, e, c); +} + +FixedDecimal::FixedDecimal(double n, int32_t v, int64_t f, int32_t e) { + init(n, v, f, e); + // check values. TODO make into unit test. + // + // long visiblePower = (int) Math.pow(10.0, v); + // if (decimalDigits > visiblePower) { + // throw new IllegalArgumentException(); + // } + // double fraction = intValue + (decimalDigits / (double) visiblePower); + // if (fraction != source) { + // double diff = Math.abs(fraction - source)/(Math.abs(fraction) + Math.abs(source)); + // if (diff > 0.00000001d) { + // throw new IllegalArgumentException(); + // } + // } +} + +FixedDecimal::FixedDecimal(double n, int32_t v, int64_t f) { + init(n, v, f); +} + +FixedDecimal::FixedDecimal(double n, int32_t v) { + // Ugly, but for samples we don't care. + init(n, v, getFractionalDigits(n, v)); +} + +FixedDecimal::FixedDecimal(double n) { + init(n); +} + +FixedDecimal::FixedDecimal() { + init(0, 0, 0); +} + + +// Create a FixedDecimal from a UnicodeString containing a number. +// Inefficient, but only used for samples, so simplicity trumps efficiency. + +FixedDecimal::FixedDecimal(const UnicodeString &num, UErrorCode &status) { + CharString cs; + int32_t parsedExponent = 0; + int32_t parsedCompactExponent = 0; + + int32_t exponentIdx = num.indexOf(u'e'); + if (exponentIdx < 0) { + exponentIdx = num.indexOf(u'E'); + } + int32_t compactExponentIdx = num.indexOf(u'c'); + if (compactExponentIdx < 0) { + compactExponentIdx = num.indexOf(u'C'); + } + + if (exponentIdx >= 0) { + cs.appendInvariantChars(num.tempSubString(0, exponentIdx), status); + int32_t expSubstrStart = exponentIdx + 1; + parsedExponent = ICU_Utility::parseAsciiInteger(num, expSubstrStart); + } + else if (compactExponentIdx >= 0) { + cs.appendInvariantChars(num.tempSubString(0, compactExponentIdx), status); + int32_t expSubstrStart = compactExponentIdx + 1; + parsedCompactExponent = ICU_Utility::parseAsciiInteger(num, expSubstrStart); + + parsedExponent = parsedCompactExponent; + exponentIdx = compactExponentIdx; + } + else { + cs.appendInvariantChars(num, status); + } + + DecimalQuantity dl; + dl.setToDecNumber(cs.toStringPiece(), status); + if (U_FAILURE(status)) { + init(0, 0, 0); + return; + } + + int32_t decimalPoint = num.indexOf(DOT); + double n = dl.toDouble(); + if (decimalPoint == -1) { + init(n, 0, 0, parsedExponent); + } else { + int32_t fractionNumLength = exponentIdx < 0 ? num.length() : cs.length(); + int32_t v = fractionNumLength - decimalPoint - 1; + init(n, v, getFractionalDigits(n, v), parsedExponent); + } +} + + +FixedDecimal::FixedDecimal(const FixedDecimal &other) { + source = other.source; + visibleDecimalDigitCount = other.visibleDecimalDigitCount; + decimalDigits = other.decimalDigits; + decimalDigitsWithoutTrailingZeros = other.decimalDigitsWithoutTrailingZeros; + intValue = other.intValue; + exponent = other.exponent; + _hasIntegerValue = other._hasIntegerValue; + isNegative = other.isNegative; + _isNaN = other._isNaN; + _isInfinite = other._isInfinite; +} + +FixedDecimal::~FixedDecimal() = default; + +FixedDecimal FixedDecimal::createWithExponent(double n, int32_t v, int32_t e) { + return FixedDecimal(n, v, getFractionalDigits(n, v), e); +} + + +void FixedDecimal::init(double n) { + int32_t numFractionDigits = decimals(n); + init(n, numFractionDigits, getFractionalDigits(n, numFractionDigits)); +} + + +void FixedDecimal::init(double n, int32_t v, int64_t f) { + int32_t exponent = 0; + init(n, v, f, exponent); +} + +void FixedDecimal::init(double n, int32_t v, int64_t f, int32_t e) { + // Currently, `c` is an alias for `e` + init(n, v, f, e, e); +} + +void FixedDecimal::init(double n, int32_t v, int64_t f, int32_t e, int32_t c) { + isNegative = n < 0.0; + source = fabs(n); + _isNaN = uprv_isNaN(source); + _isInfinite = uprv_isInfinite(source); + exponent = e; + if (exponent == 0) { + exponent = c; + } + if (_isNaN || _isInfinite) { + v = 0; + f = 0; + intValue = 0; + _hasIntegerValue = false; + } else { + intValue = (int64_t)source; + _hasIntegerValue = (source == intValue); + } + + visibleDecimalDigitCount = v; + decimalDigits = f; + if (f == 0) { + decimalDigitsWithoutTrailingZeros = 0; + } else { + int64_t fdwtz = f; + while ((fdwtz%10) == 0) { + fdwtz /= 10; + } + decimalDigitsWithoutTrailingZeros = fdwtz; + } +} + + +// Fast path only exact initialization. Return true if successful. +// Note: Do not multiply by 10 each time through loop, rounding cruft can build +// up that makes the check for an integer result fail. +// A single multiply of the original number works more reliably. +static int32_t p10[] = {1, 10, 100, 1000, 10000}; +UBool FixedDecimal::quickInit(double n) { + UBool success = false; + n = fabs(n); + int32_t numFractionDigits; + for (numFractionDigits = 0; numFractionDigits <= 3; numFractionDigits++) { + double scaledN = n * p10[numFractionDigits]; + if (scaledN == floor(scaledN)) { + success = true; + break; + } + } + if (success) { + init(n, numFractionDigits, getFractionalDigits(n, numFractionDigits)); + } + return success; +} + + + +int32_t FixedDecimal::decimals(double n) { + // Count the number of decimal digits in the fraction part of the number, excluding trailing zeros. + // fastpath the common cases, integers or fractions with 3 or fewer digits + n = fabs(n); + for (int ndigits=0; ndigits<=3; ndigits++) { + double scaledN = n * p10[ndigits]; + if (scaledN == floor(scaledN)) { + return ndigits; + } + } + + // Slow path, convert with snprintf, parse converted output. + char buf[30] = {0}; + snprintf(buf, sizeof(buf), "%1.15e", n); + // formatted number looks like this: 1.234567890123457e-01 + int exponent = atoi(buf+18); + int numFractionDigits = 15; + for (int i=16; ; --i) { + if (buf[i] != '0') { + break; + } + --numFractionDigits; + } + numFractionDigits -= exponent; // Fraction part of fixed point representation. + return numFractionDigits; +} + + +// Get the fraction digits of a double, represented as an integer. +// v is the number of visible fraction digits in the displayed form of the number. +// Example: n = 1001.234, v = 6, result = 234000 +// TODO: need to think through how this is used in the plural rule context. +// This function can easily encounter integer overflow, +// and can easily return noise digits when the precision of a double is exceeded. + +int64_t FixedDecimal::getFractionalDigits(double n, int32_t v) { + if (v == 0 || n == floor(n) || uprv_isNaN(n) || uprv_isPositiveInfinity(n)) { + return 0; + } + n = fabs(n); + double fract = n - floor(n); + switch (v) { + case 1: return (int64_t)(fract*10.0 + 0.5); + case 2: return (int64_t)(fract*100.0 + 0.5); + case 3: return (int64_t)(fract*1000.0 + 0.5); + default: + double scaled = floor(fract * pow(10.0, (double)v) + 0.5); + if (scaled >= static_cast(U_INT64_MAX)) { + // Note: a double cannot accurately represent U_INT64_MAX. Casting it to double + // will round up to the next representable value, which is U_INT64_MAX + 1. + return U_INT64_MAX; + } else { + return (int64_t)scaled; + } + } +} + + +void FixedDecimal::adjustForMinFractionDigits(int32_t minFractionDigits) { + int32_t numTrailingFractionZeros = minFractionDigits - visibleDecimalDigitCount; + if (numTrailingFractionZeros > 0) { + for (int32_t i=0; i= 100000000000000000LL) { + break; + } + decimalDigits *= 10; + } + visibleDecimalDigitCount += numTrailingFractionZeros; + } +} + + +double FixedDecimal::getPluralOperand(PluralOperand operand) const { + switch(operand) { + case PLURAL_OPERAND_N: return (exponent == 0 ? source : source * pow(10.0, exponent)); + case PLURAL_OPERAND_I: return (double) longValue(); + case PLURAL_OPERAND_F: return static_cast(decimalDigits); + case PLURAL_OPERAND_T: return static_cast(decimalDigitsWithoutTrailingZeros); + case PLURAL_OPERAND_V: return visibleDecimalDigitCount; + case PLURAL_OPERAND_E: return exponent; + case PLURAL_OPERAND_C: return exponent; + default: + UPRV_UNREACHABLE_EXIT; // unexpected. + } +} + +bool FixedDecimal::isNaN() const { + return _isNaN; +} + +bool FixedDecimal::isInfinite() const { + return _isInfinite; +} + +bool FixedDecimal::hasIntegerValue() const { + return _hasIntegerValue; +} + +bool FixedDecimal::isNanOrInfinity() const { + return _isNaN || _isInfinite; +} + +int32_t FixedDecimal::getVisibleFractionDigitCount() const { + return visibleDecimalDigitCount; +} + +bool FixedDecimal::operator==(const FixedDecimal &other) const { + return source == other.source && visibleDecimalDigitCount == other.visibleDecimalDigitCount + && decimalDigits == other.decimalDigits && exponent == other.exponent; +} + +UnicodeString FixedDecimal::toString() const { + char pattern[15]; + char buffer[20]; + if (exponent != 0) { + snprintf(pattern, sizeof(pattern), "%%.%dfe%%d", visibleDecimalDigitCount); + snprintf(buffer, sizeof(buffer), pattern, source, exponent); + } else { + snprintf(pattern, sizeof(pattern), "%%.%df", visibleDecimalDigitCount); + snprintf(buffer, sizeof(buffer), pattern, source); + } + return UnicodeString(buffer, -1, US_INV); +} + +double FixedDecimal::doubleValue() const { + return (isNegative ? -source : source) * pow(10.0, exponent); +} + +int64_t FixedDecimal::longValue() const { + if (exponent == 0) { + return intValue; + } else { + return (long) (pow(10.0, exponent) * intValue); + } +} + + +PluralAvailableLocalesEnumeration::PluralAvailableLocalesEnumeration(UErrorCode &status) { + fOpenStatus = status; + if (U_FAILURE(status)) { + return; + } + fOpenStatus = U_ZERO_ERROR; // clear any warnings. + LocalUResourceBundlePointer rb(ures_openDirect(nullptr, "plurals", &fOpenStatus)); + fLocales = ures_getByKey(rb.getAlias(), "locales", nullptr, &fOpenStatus); +} + +PluralAvailableLocalesEnumeration::~PluralAvailableLocalesEnumeration() { + ures_close(fLocales); + ures_close(fRes); + fLocales = nullptr; + fRes = nullptr; +} + +const char *PluralAvailableLocalesEnumeration::next(int32_t *resultLength, UErrorCode &status) { + if (U_FAILURE(status)) { + return nullptr; + } + if (U_FAILURE(fOpenStatus)) { + status = fOpenStatus; + return nullptr; + } + fRes = ures_getNextResource(fLocales, fRes, &status); + if (fRes == nullptr || U_FAILURE(status)) { + if (status == U_INDEX_OUTOFBOUNDS_ERROR) { + status = U_ZERO_ERROR; + } + return nullptr; + } + const char *result = ures_getKey(fRes); + if (resultLength != nullptr) { + *resultLength = static_cast(uprv_strlen(result)); + } + return result; +} + + +void PluralAvailableLocalesEnumeration::reset(UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + if (U_FAILURE(fOpenStatus)) { + status = fOpenStatus; + return; + } + ures_resetIterator(fLocales); +} + +int32_t PluralAvailableLocalesEnumeration::count(UErrorCode &status) const { + if (U_FAILURE(status)) { + return 0; + } + if (U_FAILURE(fOpenStatus)) { + status = fOpenStatus; + return 0; + } + return ures_getSize(fLocales); +} + +U_NAMESPACE_END + + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/plurrule_impl.h b/intl/icu/source/i18n/plurrule_impl.h new file mode 100644 index 0000000000..4de6d6460a --- /dev/null +++ b/intl/icu/source/i18n/plurrule_impl.h @@ -0,0 +1,442 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* File PLURRULE_IMPL.H +* +******************************************************************************* +*/ + + +#ifndef PLURRULE_IMPL +#define PLURRULE_IMPL + +// Internal definitions for the PluralRules implementation. + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/format.h" +#include "unicode/locid.h" +#include "unicode/parseerr.h" +#include "unicode/strenum.h" +#include "unicode/ures.h" +#include "uvector.h" +#include "hash.h" +#include "uassert.h" + +/** + * A FixedDecimal version of UPLRULES_NO_UNIQUE_VALUE used in PluralRulesTest + * for parsing of samples. + */ +#define UPLRULES_NO_UNIQUE_VALUE_DECIMAL(ERROR_CODE) (DecimalQuantity::fromExponentString(u"-0.00123456777", ERROR_CODE)) + +class PluralRulesTest; + +U_NAMESPACE_BEGIN + +class AndConstraint; +class RuleChain; +class DigitInterval; +class PluralRules; +class VisibleDigits; + +namespace pluralimpl { + +// TODO: Remove this and replace with u"" literals. Was for EBCDIC compatibility. + +static const char16_t DOT = ((char16_t) 0x002E); +static const char16_t SINGLE_QUOTE = ((char16_t) 0x0027); +static const char16_t SLASH = ((char16_t) 0x002F); +static const char16_t BACKSLASH = ((char16_t) 0x005C); +static const char16_t SPACE = ((char16_t) 0x0020); +static const char16_t EXCLAMATION = ((char16_t) 0x0021); +static const char16_t QUOTATION_MARK = ((char16_t) 0x0022); +static const char16_t NUMBER_SIGN = ((char16_t) 0x0023); +static const char16_t PERCENT_SIGN = ((char16_t) 0x0025); +static const char16_t ASTERISK = ((char16_t) 0x002A); +static const char16_t COMMA = ((char16_t) 0x002C); +static const char16_t HYPHEN = ((char16_t) 0x002D); +static const char16_t U_ZERO = ((char16_t) 0x0030); +static const char16_t U_ONE = ((char16_t) 0x0031); +static const char16_t U_TWO = ((char16_t) 0x0032); +static const char16_t U_THREE = ((char16_t) 0x0033); +static const char16_t U_FOUR = ((char16_t) 0x0034); +static const char16_t U_FIVE = ((char16_t) 0x0035); +static const char16_t U_SIX = ((char16_t) 0x0036); +static const char16_t U_SEVEN = ((char16_t) 0x0037); +static const char16_t U_EIGHT = ((char16_t) 0x0038); +static const char16_t U_NINE = ((char16_t) 0x0039); +static const char16_t COLON = ((char16_t) 0x003A); +static const char16_t SEMI_COLON = ((char16_t) 0x003B); +static const char16_t EQUALS = ((char16_t) 0x003D); +static const char16_t AT = ((char16_t) 0x0040); +static const char16_t CAP_A = ((char16_t) 0x0041); +static const char16_t CAP_B = ((char16_t) 0x0042); +static const char16_t CAP_R = ((char16_t) 0x0052); +static const char16_t CAP_Z = ((char16_t) 0x005A); +static const char16_t LOWLINE = ((char16_t) 0x005F); +static const char16_t LEFTBRACE = ((char16_t) 0x007B); +static const char16_t RIGHTBRACE = ((char16_t) 0x007D); +static const char16_t TILDE = ((char16_t) 0x007E); +static const char16_t ELLIPSIS = ((char16_t) 0x2026); + +static const char16_t LOW_A = ((char16_t) 0x0061); +static const char16_t LOW_B = ((char16_t) 0x0062); +static const char16_t LOW_C = ((char16_t) 0x0063); +static const char16_t LOW_D = ((char16_t) 0x0064); +static const char16_t LOW_E = ((char16_t) 0x0065); +static const char16_t LOW_F = ((char16_t) 0x0066); +static const char16_t LOW_G = ((char16_t) 0x0067); +static const char16_t LOW_H = ((char16_t) 0x0068); +static const char16_t LOW_I = ((char16_t) 0x0069); +static const char16_t LOW_J = ((char16_t) 0x006a); +static const char16_t LOW_K = ((char16_t) 0x006B); +static const char16_t LOW_L = ((char16_t) 0x006C); +static const char16_t LOW_M = ((char16_t) 0x006D); +static const char16_t LOW_N = ((char16_t) 0x006E); +static const char16_t LOW_O = ((char16_t) 0x006F); +static const char16_t LOW_P = ((char16_t) 0x0070); +static const char16_t LOW_Q = ((char16_t) 0x0071); +static const char16_t LOW_R = ((char16_t) 0x0072); +static const char16_t LOW_S = ((char16_t) 0x0073); +static const char16_t LOW_T = ((char16_t) 0x0074); +static const char16_t LOW_U = ((char16_t) 0x0075); +static const char16_t LOW_V = ((char16_t) 0x0076); +static const char16_t LOW_W = ((char16_t) 0x0077); +static const char16_t LOW_Y = ((char16_t) 0x0079); +static const char16_t LOW_Z = ((char16_t) 0x007A); + +} + + +static const int32_t PLURAL_RANGE_HIGH = 0x7fffffff; + +enum tokenType { + none, + tNumber, + tComma, + tSemiColon, + tSpace, + tColon, + tAt, // '@' + tDot, + tDot2, + tEllipsis, + tKeyword, + tAnd, + tOr, + tMod, // 'mod' or '%' + tNot, // 'not' only. + tIn, // 'in' only. + tEqual, // '=' only. + tNotEqual, // '!=' + tTilde, + tWithin, + tIs, + tVariableN, + tVariableI, + tVariableF, + tVariableV, + tVariableT, + tVariableE, + tVariableC, + tDecimal, + tInteger, + tEOF +}; + + +class PluralRuleParser: public UMemory { +public: + PluralRuleParser(); + virtual ~PluralRuleParser(); + + void parse(const UnicodeString &rules, PluralRules *dest, UErrorCode &status); + void getNextToken(UErrorCode &status); + void checkSyntax(UErrorCode &status); + static int32_t getNumberValue(const UnicodeString &token); + +private: + static tokenType getKeyType(const UnicodeString& token, tokenType type); + static tokenType charType(char16_t ch); + static UBool isValidKeyword(const UnicodeString& token); + + const UnicodeString *ruleSrc; // The rules string. + int32_t ruleIndex; // String index in the input rules, the current parse position. + UnicodeString token; // Token most recently scanned. + tokenType type; + tokenType prevType; + + // The items currently being parsed & built. + // Note: currentChain may not be the last RuleChain in the + // list because the "other" chain is forced to the end. + AndConstraint *curAndConstraint; + RuleChain *currentChain; + + int32_t rangeLowIdx; // Indices in the UVector of ranges of the + int32_t rangeHiIdx; // low and hi values currently being parsed. + + enum EParseState { + kKeyword, + kExpr, + kValue, + kRangeList, + kSamples + }; +}; + +enum PluralOperand { + /** + * The double value of the entire number. + */ + PLURAL_OPERAND_N, + + /** + * The integer value, with the fraction digits truncated off. + */ + PLURAL_OPERAND_I, + + /** + * All visible fraction digits as an integer, including trailing zeros. + */ + PLURAL_OPERAND_F, + + /** + * Visible fraction digits as an integer, not including trailing zeros. + */ + PLURAL_OPERAND_T, + + /** + * Number of visible fraction digits. + */ + PLURAL_OPERAND_V, + + /** + * Number of visible fraction digits, not including trailing zeros. + */ + PLURAL_OPERAND_W, + + /** + * Suppressed exponent for scientific notation (exponent needed in + * scientific notation to approximate i). + */ + PLURAL_OPERAND_E, + + /** + * This operand is currently treated as an alias for `PLURAL_OPERAND_E`. + * In the future, it will represent: + * + * Suppressed exponent for compact notation (exponent needed in + * compact notation to approximate i). + */ + PLURAL_OPERAND_C, + + /** + * THIS OPERAND IS DEPRECATED AND HAS BEEN REMOVED FROM THE SPEC. + * + *

    Returns the integer value, but will fail if the number has fraction digits. + * That is, using "j" instead of "i" is like implicitly adding "v is 0". + * + *

    For example, "j is 3" is equivalent to "i is 3 and v is 0": it matches + * "3" but not "3.1" or "3.0". + */ + PLURAL_OPERAND_J +}; + +/** + * Converts from the tokenType enum to PluralOperand. Asserts that the given + * tokenType can be mapped to a PluralOperand. + */ +PluralOperand tokenTypeToPluralOperand(tokenType tt); + +/** + * An interface to FixedDecimal, allowing for other implementations. + * @internal + */ +class U_I18N_API IFixedDecimal { + public: + virtual ~IFixedDecimal(); + + /** + * Returns the value corresponding to the specified operand (n, i, f, t, v, or w). + * If the operand is 'n', returns a double; otherwise, returns an integer. + */ + virtual double getPluralOperand(PluralOperand operand) const = 0; + + virtual bool isNaN() const = 0; + + virtual bool isInfinite() const = 0; + + /** Whether the number has no nonzero fraction digits. */ + virtual bool hasIntegerValue() const = 0; +}; + +/** + * class FixedDecimal serves to communicate the properties + * of a formatted number from a decimal formatter to PluralRules::select() + * + * see DecimalFormat::getFixedDecimal() + * @internal + */ +class U_I18N_API FixedDecimal: public IFixedDecimal, public UObject { + public: + /** + * @param n the number, e.g. 12.345 + * @param v The number of visible fraction digits, e.g. 3 + * @param f The fraction digits, e.g. 345 + * @param e The exponent, e.g. 7 in 1.2e7, for scientific notation + * @param c Currently: an alias for param `e`. + */ + FixedDecimal(double n, int32_t v, int64_t f, int32_t e, int32_t c); + FixedDecimal(double n, int32_t v, int64_t f, int32_t e); + FixedDecimal(double n, int32_t v, int64_t f); + FixedDecimal(double n, int32_t); + explicit FixedDecimal(double n); + FixedDecimal(); + ~FixedDecimal() override; + FixedDecimal(const UnicodeString &s, UErrorCode &ec); + FixedDecimal(const FixedDecimal &other); + + static FixedDecimal createWithExponent(double n, int32_t v, int32_t e); + + double getPluralOperand(PluralOperand operand) const override; + bool isNaN() const override; + bool isInfinite() const override; + bool hasIntegerValue() const override; + + bool isNanOrInfinity() const; // used in decimfmtimpl.cpp + + int32_t getVisibleFractionDigitCount() const; + + void init(double n, int32_t v, int64_t f, int32_t e, int32_t c); + void init(double n, int32_t v, int64_t f, int32_t e); + void init(double n, int32_t v, int64_t f); + void init(double n); + UBool quickInit(double n); // Try a fast-path only initialization, + // return true if successful. + void adjustForMinFractionDigits(int32_t min); + static int64_t getFractionalDigits(double n, int32_t v); + static int32_t decimals(double n); + + FixedDecimal& operator=(const FixedDecimal& other) = default; + bool operator==(const FixedDecimal &other) const; + + UnicodeString toString() const; + + double doubleValue() const; + int64_t longValue() const; + + double source; + int32_t visibleDecimalDigitCount; + int64_t decimalDigits; + int64_t decimalDigitsWithoutTrailingZeros; + int64_t intValue; + int32_t exponent; + UBool _hasIntegerValue; + UBool isNegative; + UBool _isNaN; + UBool _isInfinite; +}; + +class AndConstraint : public UMemory { +public: + typedef enum RuleOp { + NONE, + MOD + } RuleOp; + RuleOp op = AndConstraint::NONE; + int32_t opNum = -1; // for mod expressions, the right operand of the mod. + int32_t value = -1; // valid for 'is' rules only. + UVector32 *rangeList = nullptr; // for 'in', 'within' rules. Null otherwise. + UBool negated = false; // true for negated rules. + UBool integerOnly = false; // true for 'within' rules. + tokenType digitsType = none; // n | i | v | f constraint. + AndConstraint *next = nullptr; + // Internal error status, used for errors that occur during the copy constructor. + UErrorCode fInternalStatus = U_ZERO_ERROR; + + AndConstraint() = default; + AndConstraint(const AndConstraint& other); + virtual ~AndConstraint(); + AndConstraint* add(UErrorCode& status); + // UBool isFulfilled(double number); + UBool isFulfilled(const IFixedDecimal &number); +}; + +class OrConstraint : public UMemory { +public: + AndConstraint *childNode = nullptr; + OrConstraint *next = nullptr; + // Internal error status, used for errors that occur during the copy constructor. + UErrorCode fInternalStatus = U_ZERO_ERROR; + + OrConstraint() = default; + OrConstraint(const OrConstraint& other); + virtual ~OrConstraint(); + AndConstraint* add(UErrorCode& status); + // UBool isFulfilled(double number); + UBool isFulfilled(const IFixedDecimal &number); +}; + +class RuleChain : public UMemory { +public: + UnicodeString fKeyword; + RuleChain *fNext = nullptr; + OrConstraint *ruleHeader = nullptr; + UnicodeString fDecimalSamples; // Samples strings from rule source + UnicodeString fIntegerSamples; // without @decimal or @integer, otherwise unprocessed. + UBool fDecimalSamplesUnbounded = false; + UBool fIntegerSamplesUnbounded = false; + // Internal error status, used for errors that occur during the copy constructor. + UErrorCode fInternalStatus = U_ZERO_ERROR; + + RuleChain() = default; + RuleChain(const RuleChain& other); + virtual ~RuleChain(); + + UnicodeString select(const IFixedDecimal &number) const; + void dumpRules(UnicodeString& result); + UErrorCode getKeywords(int32_t maxArraySize, UnicodeString *keywords, int32_t& arraySize) const; + UBool isKeyword(const UnicodeString& keyword) const; +}; + +class PluralKeywordEnumeration : public StringEnumeration { +public: + PluralKeywordEnumeration(RuleChain *header, UErrorCode& status); + virtual ~PluralKeywordEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; +private: + int32_t pos; + UVector fKeywordNames; +}; + + +class U_I18N_API PluralAvailableLocalesEnumeration: public StringEnumeration { + public: + PluralAvailableLocalesEnumeration(UErrorCode &status); + virtual ~PluralAvailableLocalesEnumeration(); + virtual const char* next(int32_t *resultLength, UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; + private: + UErrorCode fOpenStatus; + UResourceBundle *fLocales = nullptr; + UResourceBundle *fRes = nullptr; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif // _PLURRULE_IMPL +//eof diff --git a/intl/icu/source/i18n/quant.cpp b/intl/icu/source/i18n/quant.cpp new file mode 100644 index 0000000000..4a45db1bf5 --- /dev/null +++ b/intl/icu/source/i18n/quant.cpp @@ -0,0 +1,151 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2001-2012, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 07/26/01 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "quant.h" +#include "unicode/unistr.h" +#include "util.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Quantifier) + +Quantifier::Quantifier(UnicodeFunctor *adoptedMatcher, + uint32_t _minCount, uint32_t _maxCount) { + // assert(adopted != 0); + // assert(minCount <= maxCount); + matcher = adoptedMatcher; + this->minCount = _minCount; + this->maxCount = _maxCount; +} + +Quantifier::Quantifier(const Quantifier& o) : + UnicodeFunctor(o), + UnicodeMatcher(o), + matcher(o.matcher->clone()), + minCount(o.minCount), + maxCount(o.maxCount) +{ +} + +Quantifier::~Quantifier() { + delete matcher; +} + +/** + * Implement UnicodeFunctor + */ +Quantifier* Quantifier::clone() const { + return new Quantifier(*this); +} + +/** + * UnicodeFunctor API. Cast 'this' to a UnicodeMatcher* pointer + * and return the pointer. + */ +UnicodeMatcher* Quantifier::toMatcher() const { + Quantifier *nonconst_this = const_cast(this); + UnicodeMatcher *nonconst_base = static_cast(nonconst_this); + + return nonconst_base; +} + +UMatchDegree Quantifier::matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental) { + int32_t start = offset; + uint32_t count = 0; + while (count < maxCount) { + int32_t pos = offset; + UMatchDegree m = matcher->toMatcher()->matches(text, offset, limit, incremental); + if (m == U_MATCH) { + ++count; + if (pos == offset) { + // If offset has not moved we have a zero-width match. + // Don't keep matching it infinitely. + break; + } + } else if (incremental && m == U_PARTIAL_MATCH) { + return U_PARTIAL_MATCH; + } else { + break; + } + } + if (incremental && offset == limit) { + return U_PARTIAL_MATCH; + } + if (count >= minCount) { + return U_MATCH; + } + offset = start; + return U_MISMATCH; +} + +/** + * Implement UnicodeMatcher + */ +UnicodeString& Quantifier::toPattern(UnicodeString& result, + UBool escapeUnprintable) const { + result.truncate(0); + matcher->toMatcher()->toPattern(result, escapeUnprintable); + if (minCount == 0) { + if (maxCount == 1) { + return result.append((char16_t)63); /*?*/ + } else if (maxCount == MAX) { + return result.append((char16_t)42); /***/ + } + // else fall through + } else if (minCount == 1 && maxCount == MAX) { + return result.append((char16_t)43); /*+*/ + } + result.append((char16_t)123); /*{*/ + ICU_Utility::appendNumber(result, minCount); + result.append((char16_t)44); /*,*/ + if (maxCount != MAX) { + ICU_Utility::appendNumber(result, maxCount); + } + result.append((char16_t)125); /*}*/ + return result; +} + +/** + * Implement UnicodeMatcher + */ +UBool Quantifier::matchesIndexValue(uint8_t v) const { + return (minCount == 0) || matcher->toMatcher()->matchesIndexValue(v); +} + +/** + * Implement UnicodeMatcher + */ +void Quantifier::addMatchSetTo(UnicodeSet& toUnionTo) const { + if (maxCount > 0) { + matcher->toMatcher()->addMatchSetTo(toUnionTo); + } +} + +/** + * Implement UnicodeFunctor + */ +void Quantifier::setData(const TransliterationRuleData* d) { + matcher->setData(d); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +//eof diff --git a/intl/icu/source/i18n/quant.h b/intl/icu/source/i18n/quant.h new file mode 100644 index 0000000000..427a6b0480 --- /dev/null +++ b/intl/icu/source/i18n/quant.h @@ -0,0 +1,126 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 2001-2011, International Business Machines Corporation + * and others. All Rights Reserved. + ********************************************************************** + * Date Name Description + * 07/26/01 aliu Creation. + ********************************************************************** + */ +#ifndef QUANT_H +#define QUANT_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unifunct.h" +#include "unicode/unimatch.h" + +U_NAMESPACE_BEGIN + +class Quantifier : public UnicodeFunctor, public UnicodeMatcher { + + public: + + enum { MAX = 0x7FFFFFFF }; + + Quantifier(UnicodeFunctor *adoptedMatcher, + uint32_t minCount, uint32_t maxCount); + + Quantifier(const Quantifier& o); + + virtual ~Quantifier(); + + /** + * UnicodeFunctor API. Cast 'this' to a UnicodeMatcher* pointer + * and return the pointer. + * @return the UnicodeMatcher pointer. + */ + virtual UnicodeMatcher* toMatcher() const override; + + /** + * Implement UnicodeFunctor + * @return a copy of the object. + */ + virtual Quantifier* clone() const override; + + /** + * Implement UnicodeMatcher + * @param text the text to be matched + * @param offset on input, the index into text at which to begin + * matching. On output, the limit of the matched text. The + * number of matched characters is the output value of offset + * minus the input value. Offset should always point to the + * HIGH SURROGATE (leading code unit) of a pair of surrogates, + * both on entry and upon return. + * @param limit the limit index of text to be matched. Greater + * than offset for a forward direction match, less than offset for + * a backward direction match. The last character to be + * considered for matching will be text.charAt(limit-1) in the + * forward direction or text.charAt(limit+1) in the backward + * direction. + * @param incremental if true, then assume further characters may + * be inserted at limit and check for partial matching. Otherwise + * assume the text as given is complete. + * @return a match degree value indicating a full match, a partial + * match, or a mismatch. If incremental is false then + * U_PARTIAL_MATCH should never be returned. + */ + virtual UMatchDegree matches(const Replaceable& text, + int32_t& offset, + int32_t limit, + UBool incremental) override; + + /** + * Implement UnicodeMatcher + * @param result Output param to receive the pattern. + * @param escapeUnprintable if True then escape the unprintable characters. + * @return A reference to 'result'. + */ + virtual UnicodeString& toPattern(UnicodeString& result, + UBool escapeUnprintable = false) const override; + + /** + * Implement UnicodeMatcher + * @param v the given index value. + * @return true if this rule matches the given index value. + */ + virtual UBool matchesIndexValue(uint8_t v) const override; + + /** + * Implement UnicodeMatcher + */ + virtual void addMatchSetTo(UnicodeSet& toUnionTo) const override; + + /** + * UnicodeFunctor API + */ + virtual void setData(const TransliterationRuleData*) override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + + private: + + UnicodeFunctor* matcher; // owned + + uint32_t minCount; + + uint32_t maxCount; +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/quantityformatter.cpp b/intl/icu/source/i18n/quantityformatter.cpp new file mode 100644 index 0000000000..0a1982e3d4 --- /dev/null +++ b/intl/icu/source/i18n/quantityformatter.cpp @@ -0,0 +1,243 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2014-2016, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* quantityformatter.cpp +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/simpleformatter.h" +#include "quantityformatter.h" +#include "uassert.h" +#include "unicode/unistr.h" +#include "unicode/decimfmt.h" +#include "cstring.h" +#include "unicode/plurrule.h" +#include "charstr.h" +#include "unicode/fmtable.h" +#include "unicode/fieldpos.h" +#include "standardplural.h" +#include "uassert.h" +#include "number_decimalquantity.h" +#include "number_utypes.h" +#include "formatted_string_builder.h" + +U_NAMESPACE_BEGIN + +QuantityFormatter::QuantityFormatter() { + for (int32_t i = 0; i < UPRV_LENGTHOF(formatters); ++i) { + formatters[i] = nullptr; + } +} + +QuantityFormatter::QuantityFormatter(const QuantityFormatter &other) { + for (int32_t i = 0; i < UPRV_LENGTHOF(formatters); ++i) { + if (other.formatters[i] == nullptr) { + formatters[i] = nullptr; + } else { + formatters[i] = new SimpleFormatter(*other.formatters[i]); + } + } +} + +QuantityFormatter &QuantityFormatter::operator=( + const QuantityFormatter& other) { + if (this == &other) { + return *this; + } + for (int32_t i = 0; i < UPRV_LENGTHOF(formatters); ++i) { + delete formatters[i]; + if (other.formatters[i] == nullptr) { + formatters[i] = nullptr; + } else { + formatters[i] = new SimpleFormatter(*other.formatters[i]); + } + } + return *this; +} + +QuantityFormatter::~QuantityFormatter() { + for (int32_t i = 0; i < UPRV_LENGTHOF(formatters); ++i) { + delete formatters[i]; + } +} + +void QuantityFormatter::reset() { + for (int32_t i = 0; i < UPRV_LENGTHOF(formatters); ++i) { + delete formatters[i]; + formatters[i] = nullptr; + } +} + +UBool QuantityFormatter::addIfAbsent( + const char *variant, + const UnicodeString &rawPattern, + UErrorCode &status) { + int32_t pluralIndex = StandardPlural::indexFromString(variant, status); + if (U_FAILURE(status)) { + return false; + } + if (formatters[pluralIndex] != nullptr) { + return true; + } + SimpleFormatter *newFmt = new SimpleFormatter(rawPattern, 0, 1, status); + if (newFmt == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return false; + } + if (U_FAILURE(status)) { + delete newFmt; + return false; + } + formatters[pluralIndex] = newFmt; + return true; +} + +UBool QuantityFormatter::isValid() const { + return formatters[StandardPlural::OTHER] != nullptr; +} + +const SimpleFormatter *QuantityFormatter::getByVariant( + const char *variant) const { + U_ASSERT(isValid()); + int32_t pluralIndex = StandardPlural::indexOrOtherIndexFromString(variant); + const SimpleFormatter *pattern = formatters[pluralIndex]; + if (pattern == nullptr) { + pattern = formatters[StandardPlural::OTHER]; + } + return pattern; +} + +UnicodeString &QuantityFormatter::format( + const Formattable &number, + const NumberFormat &fmt, + const PluralRules &rules, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const { + UnicodeString formattedNumber; + StandardPlural::Form p = selectPlural(number, fmt, rules, formattedNumber, pos, status); + if (U_FAILURE(status)) { + return appendTo; + } + const SimpleFormatter *pattern = formatters[p]; + if (pattern == nullptr) { + pattern = formatters[StandardPlural::OTHER]; + if (pattern == nullptr) { + status = U_INVALID_STATE_ERROR; + return appendTo; + } + } + return format(*pattern, formattedNumber, appendTo, pos, status); +} + +// The following methods live here so that class PluralRules does not depend on number formatting, +// and the SimpleFormatter does not depend on FieldPosition. + +StandardPlural::Form QuantityFormatter::selectPlural( + const Formattable &number, + const NumberFormat &fmt, + const PluralRules &rules, + UnicodeString &formattedNumber, + FieldPosition &pos, + UErrorCode &status) { + if (U_FAILURE(status)) { + return StandardPlural::OTHER; + } + UnicodeString pluralKeyword; + const DecimalFormat *decFmt = dynamic_cast(&fmt); + if (decFmt != nullptr) { + number::impl::DecimalQuantity dq; + decFmt->formatToDecimalQuantity(number, dq, status); + if (U_FAILURE(status)) { + return StandardPlural::OTHER; + } + pluralKeyword = rules.select(dq); + decFmt->format(number, formattedNumber, pos, status); + } else { + if (number.getType() == Formattable::kDouble) { + pluralKeyword = rules.select(number.getDouble()); + } else if (number.getType() == Formattable::kLong) { + pluralKeyword = rules.select(number.getLong()); + } else if (number.getType() == Formattable::kInt64) { + pluralKeyword = rules.select((double) number.getInt64()); + } else { + status = U_ILLEGAL_ARGUMENT_ERROR; + return StandardPlural::OTHER; + } + fmt.format(number, formattedNumber, pos, status); + } + return StandardPlural::orOtherFromString(pluralKeyword); +} + +void QuantityFormatter::formatAndSelect( + double quantity, + const NumberFormat& fmt, + const PluralRules& rules, + FormattedStringBuilder& output, + StandardPlural::Form& pluralForm, + UErrorCode& status) { + UnicodeString pluralKeyword; + const DecimalFormat* df = dynamic_cast(&fmt); + if (df != nullptr) { + number::impl::UFormattedNumberData fn; + fn.quantity.setToDouble(quantity); + const number::LocalizedNumberFormatter* lnf = df->toNumberFormatter(status); + if (U_FAILURE(status)) { + return; + } + lnf->formatImpl(&fn, status); + if (U_FAILURE(status)) { + return; + } + output = std::move(fn.getStringRef()); + pluralKeyword = rules.select(fn.quantity); + } else { + UnicodeString result; + fmt.format(quantity, result, status); + if (U_FAILURE(status)) { + return; + } + // This code path is probably RBNF. Use the generic numeric field. + output.append(result, kGeneralNumericField, status); + if (U_FAILURE(status)) { + return; + } + pluralKeyword = rules.select(quantity); + } + pluralForm = StandardPlural::orOtherFromString(pluralKeyword); +} + +UnicodeString &QuantityFormatter::format( + const SimpleFormatter &pattern, + const UnicodeString &value, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) { + if (U_FAILURE(status)) { + return appendTo; + } + const UnicodeString *param = &value; + int32_t offset; + pattern.formatAndAppend(¶m, 1, appendTo, &offset, 1, status); + if (pos.getBeginIndex() != 0 || pos.getEndIndex() != 0) { + if (offset >= 0) { + pos.setBeginIndex(pos.getBeginIndex() + offset); + pos.setEndIndex(pos.getEndIndex() + offset); + } else { + pos.setBeginIndex(0); + pos.setEndIndex(0); + } + } + return appendTo; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/quantityformatter.h b/intl/icu/source/i18n/quantityformatter.h new file mode 100644 index 0000000000..ca0c0ee371 --- /dev/null +++ b/intl/icu/source/i18n/quantityformatter.h @@ -0,0 +1,165 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2014-2016, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* quantityformatter.h +*/ + +#ifndef __QUANTITY_FORMATTER_H__ +#define __QUANTITY_FORMATTER_H__ + +#include "unicode/utypes.h" +#include "unicode/uobject.h" + +#if !UCONFIG_NO_FORMATTING + +#include "standardplural.h" + +U_NAMESPACE_BEGIN + +class SimpleFormatter; +class UnicodeString; +class PluralRules; +class NumberFormat; +class Formattable; +class FieldPosition; +class FormattedStringBuilder; + +/** + * A plural aware formatter that is good for expressing a single quantity and + * a unit. + *

    + * First use the add() methods to add a pattern for each plural variant. + * There must be a pattern for the "other" variant. + * Then use the format() method. + *

    + * Concurrent calls only to const methods on a QuantityFormatter object are + * safe, but concurrent const and non-const method calls on a QuantityFormatter + * object are not safe and require synchronization. + * + */ +class U_I18N_API QuantityFormatter : public UMemory { +public: + /** + * Default constructor. + */ + QuantityFormatter(); + + /** + * Copy constructor. + */ + QuantityFormatter(const QuantityFormatter& other); + + /** + * Assignment operator + */ + QuantityFormatter &operator=(const QuantityFormatter& other); + + /** + * Destructor. + */ + ~QuantityFormatter(); + + /** + * Removes all variants from this object including the "other" variant. + */ + void reset(); + + /** + * Adds a plural variant if there is none yet for the plural form. + * + * @param variant "zero", "one", "two", "few", "many", "other" + * @param rawPattern the pattern for the variant e.g "{0} meters" + * @param status any error returned here. + * @return true on success; false if status was set to a non zero error. + */ + UBool addIfAbsent(const char *variant, const UnicodeString &rawPattern, UErrorCode &status); + + /** + * returns true if this object has at least the "other" variant. + */ + UBool isValid() const; + + /** + * Gets the pattern formatter that would be used for a particular variant. + * If isValid() returns true, this method is guaranteed to return a + * non-nullptr value. + */ + const SimpleFormatter *getByVariant(const char *variant) const; + + /** + * Formats a number with this object appending the result to appendTo. + * At least the "other" variant must be added to this object for this + * method to work. + * + * @param number the single number. + * @param fmt formats the number + * @param rules computes the plural variant to use. + * @param appendTo result appended here. + * @param status any error returned here. + * @return appendTo + */ + UnicodeString &format( + const Formattable &number, + const NumberFormat &fmt, + const PluralRules &rules, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status) const; + + /** + * Selects the standard plural form for the number/formatter/rules. + * Used in MeasureFormat for backwards compatibility with NumberFormat. + */ + static StandardPlural::Form selectPlural( + const Formattable &number, + const NumberFormat &fmt, + const PluralRules &rules, + UnicodeString &formattedNumber, + FieldPosition &pos, + UErrorCode &status); + + /** + * Formats a quantity and selects its plural form. The output is appended + * to a FormattedStringBuilder in order to retain field information. + * + * @param quantity The number to format. + * @param fmt The formatter to use to format the number. + * @param rules The rules to use to select the plural form of the + * formatted number. + * @param output Where to append the result of the format operation. + * @param pluralForm Output variable populated with the plural form of the + * formatted number. + * @param status Set if an error occurs. + */ + static void formatAndSelect( + double quantity, + const NumberFormat& fmt, + const PluralRules& rules, + FormattedStringBuilder& output, + StandardPlural::Form& pluralForm, + UErrorCode& status); + + /** + * Formats the pattern with the value and adjusts the FieldPosition. + * TODO: Remove? + */ + static UnicodeString &format( + const SimpleFormatter &pattern, + const UnicodeString &value, + UnicodeString &appendTo, + FieldPosition &pos, + UErrorCode &status); + +private: + SimpleFormatter *formatters[StandardPlural::COUNT]; +}; + +U_NAMESPACE_END + +#endif + +#endif diff --git a/intl/icu/source/i18n/rbnf.cpp b/intl/icu/source/i18n/rbnf.cpp new file mode 100644 index 0000000000..06599b4fd1 --- /dev/null +++ b/intl/icu/source/i18n/rbnf.cpp @@ -0,0 +1,1991 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1997-2015, International Business Machines Corporation +* and others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/rbnf.h" + +#if U_HAVE_RBNF + +#include "unicode/normlzr.h" +#include "unicode/plurfmt.h" +#include "unicode/tblcoll.h" +#include "unicode/uchar.h" +#include "unicode/ucol.h" +#include "unicode/uloc.h" +#include "unicode/unum.h" +#include "unicode/ures.h" +#include "unicode/ustring.h" +#include "unicode/utf16.h" +#include "unicode/udata.h" +#include "unicode/udisplaycontext.h" +#include "unicode/brkiter.h" +#include "unicode/ucasemap.h" + +#include "cmemory.h" +#include "cstring.h" +#include "patternprops.h" +#include "uresimp.h" +#include "nfrs.h" +#include "number_decimalquantity.h" + +// debugging +// #define RBNF_DEBUG + +#ifdef RBNF_DEBUG +#include +#endif + +#define U_ICUDATA_RBNF U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "rbnf" + +static const char16_t gPercentPercent[] = +{ + 0x25, 0x25, 0 +}; /* "%%" */ + +// All urbnf objects are created through openRules, so we init all of the +// Unicode string constants required by rbnf, nfrs, or nfr here. +static const char16_t gLenientParse[] = +{ + 0x25, 0x25, 0x6C, 0x65, 0x6E, 0x69, 0x65, 0x6E, 0x74, 0x2D, 0x70, 0x61, 0x72, 0x73, 0x65, 0x3A, 0 +}; /* "%%lenient-parse:" */ +static const char16_t gSemiColon = 0x003B; +static const char16_t gSemiPercent[] = +{ + 0x3B, 0x25, 0 +}; /* ";%" */ + +#define kSomeNumberOfBitsDiv2 22 +#define kHalfMaxDouble (double)(1 << kSomeNumberOfBitsDiv2) +#define kMaxDouble (kHalfMaxDouble * kHalfMaxDouble) + +U_NAMESPACE_BEGIN + +using number::impl::DecimalQuantity; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedNumberFormat) + +/* +This is a utility class. It does not use ICU's RTTI. +If ICU's RTTI is needed again, you can uncomment the RTTI code and derive from UObject. +Please make sure that intltest passes on Windows in Release mode, +since the string pooling per compilation unit will mess up how RTTI works. +The RTTI code was also removed due to lack of code coverage. +*/ +class LocalizationInfo : public UMemory { +protected: + virtual ~LocalizationInfo(); + uint32_t refcount; + +public: + LocalizationInfo() : refcount(0) {} + + LocalizationInfo* ref() { + ++refcount; + return this; + } + + LocalizationInfo* unref() { + if (refcount && --refcount == 0) { + delete this; + } + return nullptr; + } + + virtual bool operator==(const LocalizationInfo* rhs) const; + inline bool operator!=(const LocalizationInfo* rhs) const { return !operator==(rhs); } + + virtual int32_t getNumberOfRuleSets() const = 0; + virtual const char16_t* getRuleSetName(int32_t index) const = 0; + virtual int32_t getNumberOfDisplayLocales() const = 0; + virtual const char16_t* getLocaleName(int32_t index) const = 0; + virtual const char16_t* getDisplayName(int32_t localeIndex, int32_t ruleIndex) const = 0; + + virtual int32_t indexForLocale(const char16_t* locale) const; + virtual int32_t indexForRuleSet(const char16_t* ruleset) const; + +// virtual UClassID getDynamicClassID() const = 0; +// static UClassID getStaticClassID(); +}; + +LocalizationInfo::~LocalizationInfo() {} + +//UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(LocalizationInfo) + +// if both strings are nullptr, this returns true +static UBool +streq(const char16_t* lhs, const char16_t* rhs) { + if (rhs == lhs) { + return true; + } + if (lhs && rhs) { + return u_strcmp(lhs, rhs) == 0; + } + return false; +} + +bool +LocalizationInfo::operator==(const LocalizationInfo* rhs) const { + if (rhs) { + if (this == rhs) { + return true; + } + + int32_t rsc = getNumberOfRuleSets(); + if (rsc == rhs->getNumberOfRuleSets()) { + for (int i = 0; i < rsc; ++i) { + if (!streq(getRuleSetName(i), rhs->getRuleSetName(i))) { + return false; + } + } + int32_t dlc = getNumberOfDisplayLocales(); + if (dlc == rhs->getNumberOfDisplayLocales()) { + for (int i = 0; i < dlc; ++i) { + const char16_t* locale = getLocaleName(i); + int32_t ix = rhs->indexForLocale(locale); + // if no locale, ix is -1, getLocaleName returns null, so streq returns false + if (!streq(locale, rhs->getLocaleName(ix))) { + return false; + } + for (int j = 0; j < rsc; ++j) { + if (!streq(getDisplayName(i, j), rhs->getDisplayName(ix, j))) { + return false; + } + } + } + return true; + } + } + } + return false; +} + +int32_t +LocalizationInfo::indexForLocale(const char16_t* locale) const { + for (int i = 0; i < getNumberOfDisplayLocales(); ++i) { + if (streq(locale, getLocaleName(i))) { + return i; + } + } + return -1; +} + +int32_t +LocalizationInfo::indexForRuleSet(const char16_t* ruleset) const { + if (ruleset) { + for (int i = 0; i < getNumberOfRuleSets(); ++i) { + if (streq(ruleset, getRuleSetName(i))) { + return i; + } + } + } + return -1; +} + + +typedef void (*Fn_Deleter)(void*); + +class VArray { + void** buf; + int32_t cap; + int32_t size; + Fn_Deleter deleter; +public: + VArray() : buf(nullptr), cap(0), size(0), deleter(nullptr) {} + + VArray(Fn_Deleter del) : buf(nullptr), cap(0), size(0), deleter(del) {} + + ~VArray() { + if (deleter) { + for (int i = 0; i < size; ++i) { + (*deleter)(buf[i]); + } + } + uprv_free(buf); + } + + int32_t length() { + return size; + } + + void add(void* elem, UErrorCode& status) { + if (U_SUCCESS(status)) { + if (size == cap) { + if (cap == 0) { + cap = 1; + } else if (cap < 256) { + cap *= 2; + } else { + cap += 256; + } + if (buf == nullptr) { + buf = (void**)uprv_malloc(cap * sizeof(void*)); + } else { + buf = (void**)uprv_realloc(buf, cap * sizeof(void*)); + } + if (buf == nullptr) { + // if we couldn't realloc, we leak the memory we've already allocated, but we're in deep trouble anyway + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + void* start = &buf[size]; + size_t count = (cap - size) * sizeof(void*); + uprv_memset(start, 0, count); // fill with nulls, just because + } + buf[size++] = elem; + } + } + + void** release() { + void** result = buf; + buf = nullptr; + cap = 0; + size = 0; + return result; + } +}; + +class LocDataParser; + +class StringLocalizationInfo : public LocalizationInfo { + char16_t* info; + char16_t*** data; + int32_t numRuleSets; + int32_t numLocales; + +friend class LocDataParser; + + StringLocalizationInfo(char16_t* i, char16_t*** d, int32_t numRS, int32_t numLocs) + : info(i), data(d), numRuleSets(numRS), numLocales(numLocs) + { + } + +public: + static StringLocalizationInfo* create(const UnicodeString& info, UParseError& perror, UErrorCode& status); + + virtual ~StringLocalizationInfo(); + virtual int32_t getNumberOfRuleSets() const override { return numRuleSets; } + virtual const char16_t* getRuleSetName(int32_t index) const override; + virtual int32_t getNumberOfDisplayLocales() const override { return numLocales; } + virtual const char16_t* getLocaleName(int32_t index) const override; + virtual const char16_t* getDisplayName(int32_t localeIndex, int32_t ruleIndex) const override; + +// virtual UClassID getDynamicClassID() const; +// static UClassID getStaticClassID(); + +private: + void init(UErrorCode& status) const; +}; + + +enum { + OPEN_ANGLE = 0x003c, /* '<' */ + CLOSE_ANGLE = 0x003e, /* '>' */ + COMMA = 0x002c, + TICK = 0x0027, + QUOTE = 0x0022, + SPACE = 0x0020 +}; + +/** + * Utility for parsing a localization string and returning a StringLocalizationInfo*. + */ +class LocDataParser { + char16_t* data; + const char16_t* e; + char16_t* p; + char16_t ch; + UParseError& pe; + UErrorCode& ec; + +public: + LocDataParser(UParseError& parseError, UErrorCode& status) + : data(nullptr), e(nullptr), p(nullptr), ch(0xffff), pe(parseError), ec(status) {} + ~LocDataParser() {} + + /* + * On a successful parse, return a StringLocalizationInfo*, otherwise delete locData, set perror and status, + * and return nullptr. The StringLocalizationInfo will adopt locData if it is created. + */ + StringLocalizationInfo* parse(char16_t* data, int32_t len); + +private: + + inline void inc() { + ++p; + ch = 0xffff; + } + inline UBool checkInc(char16_t c) { + if (p < e && (ch == c || *p == c)) { + inc(); + return true; + } + return false; + } + inline UBool check(char16_t c) { + return p < e && (ch == c || *p == c); + } + inline void skipWhitespace() { + while (p < e && PatternProps::isWhiteSpace(ch != 0xffff ? ch : *p)) { + inc(); + } + } + inline UBool inList(char16_t c, const char16_t* list) const { + if (*list == SPACE && PatternProps::isWhiteSpace(c)) { + return true; + } + while (*list && *list != c) { + ++list; + } + return *list == c; + } + void parseError(const char* msg); + + StringLocalizationInfo* doParse(); + + char16_t** nextArray(int32_t& requiredLength); + char16_t* nextString(); +}; + +#ifdef RBNF_DEBUG +#define ERROR(msg) UPRV_BLOCK_MACRO_BEGIN { \ + parseError(msg); \ + return nullptr; \ +} UPRV_BLOCK_MACRO_END +#define EXPLANATION_ARG explanationArg +#else +#define ERROR(msg) UPRV_BLOCK_MACRO_BEGIN { \ + parseError(nullptr); \ + return nullptr; \ +} UPRV_BLOCK_MACRO_END +#define EXPLANATION_ARG +#endif + + +static const char16_t DQUOTE_STOPLIST[] = { + QUOTE, 0 +}; + +static const char16_t SQUOTE_STOPLIST[] = { + TICK, 0 +}; + +static const char16_t NOQUOTE_STOPLIST[] = { + SPACE, COMMA, CLOSE_ANGLE, OPEN_ANGLE, TICK, QUOTE, 0 +}; + +static void +DeleteFn(void* p) { + uprv_free(p); +} + +StringLocalizationInfo* +LocDataParser::parse(char16_t* _data, int32_t len) { + if (U_FAILURE(ec)) { + if (_data) uprv_free(_data); + return nullptr; + } + + pe.line = 0; + pe.offset = -1; + pe.postContext[0] = 0; + pe.preContext[0] = 0; + + if (_data == nullptr) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + if (len <= 0) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + uprv_free(_data); + return nullptr; + } + + data = _data; + e = data + len; + p = _data; + ch = 0xffff; + + return doParse(); +} + + +StringLocalizationInfo* +LocDataParser::doParse() { + skipWhitespace(); + if (!checkInc(OPEN_ANGLE)) { + ERROR("Missing open angle"); + } else { + VArray array(DeleteFn); + UBool mightHaveNext = true; + int32_t requiredLength = -1; + while (mightHaveNext) { + mightHaveNext = false; + char16_t** elem = nextArray(requiredLength); + skipWhitespace(); + UBool haveComma = check(COMMA); + if (elem) { + array.add(elem, ec); + if (haveComma) { + inc(); + mightHaveNext = true; + } + } else if (haveComma) { + ERROR("Unexpected character"); + } + } + + skipWhitespace(); + if (!checkInc(CLOSE_ANGLE)) { + if (check(OPEN_ANGLE)) { + ERROR("Missing comma in outer array"); + } else { + ERROR("Missing close angle bracket in outer array"); + } + } + + skipWhitespace(); + if (p != e) { + ERROR("Extra text after close of localization data"); + } + + array.add(nullptr, ec); + if (U_SUCCESS(ec)) { + int32_t numLocs = array.length() - 2; // subtract first, nullptr + char16_t*** result = (char16_t***)array.release(); + + return new StringLocalizationInfo(data, result, requiredLength-2, numLocs); // subtract first, nullptr + } + } + + ERROR("Unknown error"); +} + +char16_t** +LocDataParser::nextArray(int32_t& requiredLength) { + if (U_FAILURE(ec)) { + return nullptr; + } + + skipWhitespace(); + if (!checkInc(OPEN_ANGLE)) { + ERROR("Missing open angle"); + } + + VArray array; + UBool mightHaveNext = true; + while (mightHaveNext) { + mightHaveNext = false; + char16_t* elem = nextString(); + skipWhitespace(); + UBool haveComma = check(COMMA); + if (elem) { + array.add(elem, ec); + if (haveComma) { + inc(); + mightHaveNext = true; + } + } else if (haveComma) { + ERROR("Unexpected comma"); + } + } + skipWhitespace(); + if (!checkInc(CLOSE_ANGLE)) { + if (check(OPEN_ANGLE)) { + ERROR("Missing close angle bracket in inner array"); + } else { + ERROR("Missing comma in inner array"); + } + } + + array.add(nullptr, ec); + if (U_SUCCESS(ec)) { + if (requiredLength == -1) { + requiredLength = array.length() + 1; + } else if (array.length() != requiredLength) { + ec = U_ILLEGAL_ARGUMENT_ERROR; + ERROR("Array not of required length"); + } + + return (char16_t**)array.release(); + } + ERROR("Unknown Error"); +} + +char16_t* +LocDataParser::nextString() { + char16_t* result = nullptr; + + skipWhitespace(); + if (p < e) { + const char16_t* terminators; + char16_t c = *p; + UBool haveQuote = c == QUOTE || c == TICK; + if (haveQuote) { + inc(); + terminators = c == QUOTE ? DQUOTE_STOPLIST : SQUOTE_STOPLIST; + } else { + terminators = NOQUOTE_STOPLIST; + } + char16_t* start = p; + while (p < e && !inList(*p, terminators)) ++p; + if (p == e) { + ERROR("Unexpected end of data"); + } + + char16_t x = *p; + if (p > start) { + ch = x; + *p = 0x0; // terminate by writing to data + result = start; // just point into data + } + if (haveQuote) { + if (x != c) { + ERROR("Missing matching quote"); + } else if (p == start) { + ERROR("Empty string"); + } + inc(); + } else if (x == OPEN_ANGLE || x == TICK || x == QUOTE) { + ERROR("Unexpected character in string"); + } + } + + // ok for there to be no next string + return result; +} + +void LocDataParser::parseError(const char* EXPLANATION_ARG) +{ + if (!data) { + return; + } + + const char16_t* start = p - U_PARSE_CONTEXT_LEN - 1; + if (start < data) { + start = data; + } + for (char16_t* x = p; --x >= start;) { + if (!*x) { + start = x+1; + break; + } + } + const char16_t* limit = p + U_PARSE_CONTEXT_LEN - 1; + if (limit > e) { + limit = e; + } + u_strncpy(pe.preContext, start, (int32_t)(p-start)); + pe.preContext[p-start] = 0; + u_strncpy(pe.postContext, p, (int32_t)(limit-p)); + pe.postContext[limit-p] = 0; + pe.offset = (int32_t)(p - data); + +#ifdef RBNF_DEBUG + fprintf(stderr, "%s at or near character %ld: ", EXPLANATION_ARG, p-data); + + UnicodeString msg; + msg.append(start, p - start); + msg.append((char16_t)0x002f); /* SOLIDUS/SLASH */ + msg.append(p, limit-p); + msg.append(UNICODE_STRING_SIMPLE("'")); + + char buf[128]; + int32_t len = msg.extract(0, msg.length(), buf, 128); + if (len >= 128) { + buf[127] = 0; + } else { + buf[len] = 0; + } + fprintf(stderr, "%s\n", buf); + fflush(stderr); +#endif + + uprv_free(data); + data = nullptr; + p = nullptr; + e = nullptr; + + if (U_SUCCESS(ec)) { + ec = U_PARSE_ERROR; + } +} + +//UOBJECT_DEFINE_RTTI_IMPLEMENTATION(StringLocalizationInfo) + +StringLocalizationInfo* +StringLocalizationInfo::create(const UnicodeString& info, UParseError& perror, UErrorCode& status) { + if (U_FAILURE(status)) { + return nullptr; + } + + int32_t len = info.length(); + if (len == 0) { + return nullptr; // no error; + } + + char16_t* p = (char16_t*)uprv_malloc(len * sizeof(char16_t)); + if (!p) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + info.extract(p, len, status); + if (!U_FAILURE(status)) { + status = U_ZERO_ERROR; // clear warning about non-termination + } + + LocDataParser parser(perror, status); + return parser.parse(p, len); +} + +StringLocalizationInfo::~StringLocalizationInfo() { + for (char16_t*** p = (char16_t***)data; *p; ++p) { + // remaining data is simply pointer into our unicode string data. + if (*p) uprv_free(*p); + } + if (data) uprv_free(data); + if (info) uprv_free(info); +} + + +const char16_t* +StringLocalizationInfo::getRuleSetName(int32_t index) const { + if (index >= 0 && index < getNumberOfRuleSets()) { + return data[0][index]; + } + return nullptr; +} + +const char16_t* +StringLocalizationInfo::getLocaleName(int32_t index) const { + if (index >= 0 && index < getNumberOfDisplayLocales()) { + return data[index+1][0]; + } + return nullptr; +} + +const char16_t* +StringLocalizationInfo::getDisplayName(int32_t localeIndex, int32_t ruleIndex) const { + if (localeIndex >= 0 && localeIndex < getNumberOfDisplayLocales() && + ruleIndex >= 0 && ruleIndex < getNumberOfRuleSets()) { + return data[localeIndex+1][ruleIndex+1]; + } + return nullptr; +} + +// ---------- + +RuleBasedNumberFormat::RuleBasedNumberFormat(const UnicodeString& description, + const UnicodeString& locs, + const Locale& alocale, UParseError& perror, UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(alocale) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + LocalizationInfo* locinfo = StringLocalizationInfo::create(locs, perror, status); + init(description, locinfo, perror, status); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(const UnicodeString& description, + const UnicodeString& locs, + UParseError& perror, UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(Locale::getDefault()) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + LocalizationInfo* locinfo = StringLocalizationInfo::create(locs, perror, status); + init(description, locinfo, perror, status); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(const UnicodeString& description, + LocalizationInfo* info, + const Locale& alocale, UParseError& perror, UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(alocale) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + init(description, info, perror, status); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(const UnicodeString& description, + UParseError& perror, + UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(Locale::getDefault()) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + init(description, nullptr, perror, status); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(const UnicodeString& description, + const Locale& aLocale, + UParseError& perror, + UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(aLocale) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + init(description, nullptr, perror, status); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(URBNFRuleSetTag tag, const Locale& alocale, UErrorCode& status) + : fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(alocale) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + if (U_FAILURE(status)) { + return; + } + + const char* rules_tag = "RBNFRules"; + const char* fmt_tag = ""; + switch (tag) { + case URBNF_SPELLOUT: fmt_tag = "SpelloutRules"; break; + case URBNF_ORDINAL: fmt_tag = "OrdinalRules"; break; + case URBNF_DURATION: fmt_tag = "DurationRules"; break; + case URBNF_NUMBERING_SYSTEM: fmt_tag = "NumberingSystemRules"; break; + default: status = U_ILLEGAL_ARGUMENT_ERROR; return; + } + + // TODO: read localization info from resource + LocalizationInfo* locinfo = nullptr; + + UResourceBundle* nfrb = ures_open(U_ICUDATA_RBNF, locale.getName(), &status); + if (U_SUCCESS(status)) { + setLocaleIDs(ures_getLocaleByType(nfrb, ULOC_VALID_LOCALE, &status), + ures_getLocaleByType(nfrb, ULOC_ACTUAL_LOCALE, &status)); + + UResourceBundle* rbnfRules = ures_getByKeyWithFallback(nfrb, rules_tag, nullptr, &status); + if (U_FAILURE(status)) { + ures_close(nfrb); + } + UResourceBundle* ruleSets = ures_getByKeyWithFallback(rbnfRules, fmt_tag, nullptr, &status); + if (U_FAILURE(status)) { + ures_close(rbnfRules); + ures_close(nfrb); + return; + } + + UnicodeString desc; + while (ures_hasNext(ruleSets)) { + desc.append(ures_getNextUnicodeString(ruleSets,nullptr,&status)); + } + UParseError perror; + + init(desc, locinfo, perror, status); + + ures_close(ruleSets); + ures_close(rbnfRules); + } + ures_close(nfrb); +} + +RuleBasedNumberFormat::RuleBasedNumberFormat(const RuleBasedNumberFormat& rhs) + : NumberFormat(rhs) + , fRuleSets(nullptr) + , ruleSetDescriptions(nullptr) + , numRuleSets(0) + , defaultRuleSet(nullptr) + , locale(rhs.locale) + , collator(nullptr) + , decimalFormatSymbols(nullptr) + , defaultInfinityRule(nullptr) + , defaultNaNRule(nullptr) + , fRoundingMode(DecimalFormat::ERoundingMode::kRoundUnnecessary) + , lenient(false) + , lenientParseRules(nullptr) + , localizations(nullptr) + , capitalizationInfoSet(false) + , capitalizationForUIListMenu(false) + , capitalizationForStandAlone(false) + , capitalizationBrkIter(nullptr) +{ + this->operator=(rhs); +} + +// -------- + +RuleBasedNumberFormat& +RuleBasedNumberFormat::operator=(const RuleBasedNumberFormat& rhs) +{ + if (this == &rhs) { + return *this; + } + NumberFormat::operator=(rhs); + UErrorCode status = U_ZERO_ERROR; + dispose(); + locale = rhs.locale; + lenient = rhs.lenient; + + UParseError perror; + setDecimalFormatSymbols(*rhs.getDecimalFormatSymbols()); + init(rhs.originalDescription, rhs.localizations ? rhs.localizations->ref() : nullptr, perror, status); + setDefaultRuleSet(rhs.getDefaultRuleSetName(), status); + setRoundingMode(rhs.getRoundingMode()); + + capitalizationInfoSet = rhs.capitalizationInfoSet; + capitalizationForUIListMenu = rhs.capitalizationForUIListMenu; + capitalizationForStandAlone = rhs.capitalizationForStandAlone; +#if !UCONFIG_NO_BREAK_ITERATION + capitalizationBrkIter = (rhs.capitalizationBrkIter!=nullptr)? rhs.capitalizationBrkIter->clone(): nullptr; +#endif + + return *this; +} + +RuleBasedNumberFormat::~RuleBasedNumberFormat() +{ + dispose(); +} + +RuleBasedNumberFormat* +RuleBasedNumberFormat::clone() const +{ + return new RuleBasedNumberFormat(*this); +} + +bool +RuleBasedNumberFormat::operator==(const Format& other) const +{ + if (this == &other) { + return true; + } + + if (typeid(*this) == typeid(other)) { + const RuleBasedNumberFormat& rhs = static_cast(other); + // test for capitalization info equality is adequately handled + // by the NumberFormat test for fCapitalizationContext equality; + // the info here is just derived from that. + if (locale == rhs.locale && + lenient == rhs.lenient && + (localizations == nullptr + ? rhs.localizations == nullptr + : (rhs.localizations == nullptr + ? false + : *localizations == rhs.localizations))) { + + NFRuleSet** p = fRuleSets; + NFRuleSet** q = rhs.fRuleSets; + if (p == nullptr) { + return q == nullptr; + } else if (q == nullptr) { + return false; + } + while (*p && *q && (**p == **q)) { + ++p; + ++q; + } + return *q == nullptr && *p == nullptr; + } + } + + return false; +} + +UnicodeString +RuleBasedNumberFormat::getRules() const +{ + UnicodeString result; + if (fRuleSets != nullptr) { + for (NFRuleSet** p = fRuleSets; *p; ++p) { + (*p)->appendRules(result); + } + } + return result; +} + +UnicodeString +RuleBasedNumberFormat::getRuleSetName(int32_t index) const +{ + if (localizations) { + UnicodeString string(true, localizations->getRuleSetName(index), (int32_t)-1); + return string; + } + else if (fRuleSets) { + UnicodeString result; + for (NFRuleSet** p = fRuleSets; *p; ++p) { + NFRuleSet* rs = *p; + if (rs->isPublic()) { + if (--index == -1) { + rs->getName(result); + return result; + } + } + } + } + UnicodeString empty; + return empty; +} + +int32_t +RuleBasedNumberFormat::getNumberOfRuleSetNames() const +{ + int32_t result = 0; + if (localizations) { + result = localizations->getNumberOfRuleSets(); + } + else if (fRuleSets) { + for (NFRuleSet** p = fRuleSets; *p; ++p) { + if ((**p).isPublic()) { + ++result; + } + } + } + return result; +} + +int32_t +RuleBasedNumberFormat::getNumberOfRuleSetDisplayNameLocales() const { + if (localizations) { + return localizations->getNumberOfDisplayLocales(); + } + return 0; +} + +Locale +RuleBasedNumberFormat::getRuleSetDisplayNameLocale(int32_t index, UErrorCode& status) const { + if (U_FAILURE(status)) { + return Locale(""); + } + if (localizations && index >= 0 && index < localizations->getNumberOfDisplayLocales()) { + UnicodeString name(true, localizations->getLocaleName(index), -1); + char buffer[64]; + int32_t cap = name.length() + 1; + char* bp = buffer; + if (cap > 64) { + bp = (char *)uprv_malloc(cap); + if (bp == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return Locale(""); + } + } + name.extract(0, name.length(), bp, cap, UnicodeString::kInvariant); + Locale retLocale(bp); + if (bp != buffer) { + uprv_free(bp); + } + return retLocale; + } + status = U_ILLEGAL_ARGUMENT_ERROR; + Locale retLocale; + return retLocale; +} + +UnicodeString +RuleBasedNumberFormat::getRuleSetDisplayName(int32_t index, const Locale& localeParam) { + if (localizations && index >= 0 && index < localizations->getNumberOfRuleSets()) { + UnicodeString localeName(localeParam.getBaseName(), -1, UnicodeString::kInvariant); + int32_t len = localeName.length(); + char16_t* localeStr = localeName.getBuffer(len + 1); + while (len >= 0) { + localeStr[len] = 0; + int32_t ix = localizations->indexForLocale(localeStr); + if (ix >= 0) { + UnicodeString name(true, localizations->getDisplayName(ix, index), -1); + return name; + } + + // trim trailing portion, skipping over omitted sections + do { --len;} while (len > 0 && localeStr[len] != 0x005f); // underscore + while (len > 0 && localeStr[len-1] == 0x005F) --len; + } + UnicodeString name(true, localizations->getRuleSetName(index), -1); + return name; + } + UnicodeString bogus; + bogus.setToBogus(); + return bogus; +} + +UnicodeString +RuleBasedNumberFormat::getRuleSetDisplayName(const UnicodeString& ruleSetName, const Locale& localeParam) { + if (localizations) { + UnicodeString rsn(ruleSetName); + int32_t ix = localizations->indexForRuleSet(rsn.getTerminatedBuffer()); + return getRuleSetDisplayName(ix, localeParam); + } + UnicodeString bogus; + bogus.setToBogus(); + return bogus; +} + +NFRuleSet* +RuleBasedNumberFormat::findRuleSet(const UnicodeString& name, UErrorCode& status) const +{ + if (U_SUCCESS(status) && fRuleSets) { + for (NFRuleSet** p = fRuleSets; *p; ++p) { + NFRuleSet* rs = *p; + if (rs->isNamed(name)) { + return rs; + } + } + status = U_ILLEGAL_ARGUMENT_ERROR; + } + return nullptr; +} + +UnicodeString& +RuleBasedNumberFormat::format(const DecimalQuantity &number, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode &status) const { + if (U_FAILURE(status)) { + return appendTo; + } + DecimalQuantity copy(number); + if (copy.fitsInLong()) { + format(number.toLong(), appendTo, pos, status); + } + else { + copy.roundToMagnitude(0, number::impl::RoundingMode::UNUM_ROUND_HALFEVEN, status); + if (copy.fitsInLong()) { + format(number.toDouble(), appendTo, pos, status); + } + else { + // We're outside of our normal range that this framework can handle. + // The DecimalFormat will provide more accurate results. + + // TODO this section should probably be optimized. The DecimalFormat is shared in ICU4J. + LocalPointer decimalFormat(NumberFormat::createInstance(locale, UNUM_DECIMAL, status), status); + if (decimalFormat.isNull()) { + return appendTo; + } + Formattable f; + LocalPointer decimalQuantity(new DecimalQuantity(number), status); + if (decimalQuantity.isNull()) { + return appendTo; + } + f.adoptDecimalQuantity(decimalQuantity.orphan()); // f now owns decimalQuantity. + decimalFormat->format(f, appendTo, pos, status); + } + } + return appendTo; +} + +UnicodeString& +RuleBasedNumberFormat::format(int32_t number, + UnicodeString& toAppendTo, + FieldPosition& pos) const +{ + return format((int64_t)number, toAppendTo, pos); +} + + +UnicodeString& +RuleBasedNumberFormat::format(int64_t number, + UnicodeString& toAppendTo, + FieldPosition& /* pos */) const +{ + if (defaultRuleSet) { + UErrorCode status = U_ZERO_ERROR; + format(number, defaultRuleSet, toAppendTo, status); + } + return toAppendTo; +} + + +UnicodeString& +RuleBasedNumberFormat::format(double number, + UnicodeString& toAppendTo, + FieldPosition& /* pos */) const +{ + UErrorCode status = U_ZERO_ERROR; + if (defaultRuleSet) { + format(number, *defaultRuleSet, toAppendTo, status); + } + return toAppendTo; +} + + +UnicodeString& +RuleBasedNumberFormat::format(int32_t number, + const UnicodeString& ruleSetName, + UnicodeString& toAppendTo, + FieldPosition& pos, + UErrorCode& status) const +{ + return format((int64_t)number, ruleSetName, toAppendTo, pos, status); +} + + +UnicodeString& +RuleBasedNumberFormat::format(int64_t number, + const UnicodeString& ruleSetName, + UnicodeString& toAppendTo, + FieldPosition& /* pos */, + UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + if (ruleSetName.indexOf(gPercentPercent, 2, 0) == 0) { + // throw new IllegalArgumentException("Can't use internal rule set"); + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + NFRuleSet *rs = findRuleSet(ruleSetName, status); + if (rs) { + format(number, rs, toAppendTo, status); + } + } + } + return toAppendTo; +} + + +UnicodeString& +RuleBasedNumberFormat::format(double number, + const UnicodeString& ruleSetName, + UnicodeString& toAppendTo, + FieldPosition& /* pos */, + UErrorCode& status) const +{ + if (U_SUCCESS(status)) { + if (ruleSetName.indexOf(gPercentPercent, 2, 0) == 0) { + // throw new IllegalArgumentException("Can't use internal rule set"); + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + NFRuleSet *rs = findRuleSet(ruleSetName, status); + if (rs) { + format(number, *rs, toAppendTo, status); + } + } + } + return toAppendTo; +} + +void +RuleBasedNumberFormat::format(double number, + NFRuleSet& rs, + UnicodeString& toAppendTo, + UErrorCode& status) const +{ + int32_t startPos = toAppendTo.length(); + if (getRoundingMode() != DecimalFormat::ERoundingMode::kRoundUnnecessary && !uprv_isNaN(number) && !uprv_isInfinite(number)) { + DecimalQuantity digitList; + digitList.setToDouble(number); + digitList.roundToMagnitude( + -getMaximumFractionDigits(), + static_cast(getRoundingMode()), + status); + number = digitList.toDouble(); + } + rs.format(number, toAppendTo, toAppendTo.length(), 0, status); + adjustForCapitalizationContext(startPos, toAppendTo, status); +} + +/** + * Bottleneck through which all the public format() methods + * that take a long pass. By the time we get here, we know + * which rule set we're using to do the formatting. + * @param number The number to format + * @param ruleSet The rule set to use to format the number + * @return The text that resulted from formatting the number + */ +UnicodeString& +RuleBasedNumberFormat::format(int64_t number, NFRuleSet *ruleSet, UnicodeString& toAppendTo, UErrorCode& status) const +{ + // all API format() routines that take a double vector through + // here. We have these two identical functions-- one taking a + // double and one taking a long-- the couple digits of precision + // that long has but double doesn't (both types are 8 bytes long, + // but double has to borrow some of the mantissa bits to hold + // the exponent). + // Create an empty string buffer where the result will + // be built, and pass it to the rule set (along with an insertion + // position of 0 and the number being formatted) to the rule set + // for formatting + + if (U_SUCCESS(status)) { + if (number == U_INT64_MIN) { + // We can't handle this value right now. Provide an accurate default value. + + // TODO this section should probably be optimized. The DecimalFormat is shared in ICU4J. + NumberFormat *decimalFormat = NumberFormat::createInstance(locale, UNUM_DECIMAL, status); + if (decimalFormat == nullptr) { + return toAppendTo; + } + Formattable f; + FieldPosition pos(FieldPosition::DONT_CARE); + DecimalQuantity *decimalQuantity = new DecimalQuantity(); + if (decimalQuantity == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + delete decimalFormat; + return toAppendTo; + } + decimalQuantity->setToLong(number); + f.adoptDecimalQuantity(decimalQuantity); // f now owns decimalQuantity. + decimalFormat->format(f, toAppendTo, pos, status); + delete decimalFormat; + } + else { + int32_t startPos = toAppendTo.length(); + ruleSet->format(number, toAppendTo, toAppendTo.length(), 0, status); + adjustForCapitalizationContext(startPos, toAppendTo, status); + } + } + return toAppendTo; +} + +UnicodeString& +RuleBasedNumberFormat::adjustForCapitalizationContext(int32_t startPos, + UnicodeString& currentResult, + UErrorCode& status) const +{ +#if !UCONFIG_NO_BREAK_ITERATION + UDisplayContext capitalizationContext = getContext(UDISPCTX_TYPE_CAPITALIZATION, status); + if (capitalizationContext != UDISPCTX_CAPITALIZATION_NONE && startPos == 0 && currentResult.length() > 0) { + // capitalize currentResult according to context + UChar32 ch = currentResult.char32At(0); + if (u_islower(ch) && U_SUCCESS(status) && capitalizationBrkIter != nullptr && + ( capitalizationContext == UDISPCTX_CAPITALIZATION_FOR_BEGINNING_OF_SENTENCE || + (capitalizationContext == UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU && capitalizationForUIListMenu) || + (capitalizationContext == UDISPCTX_CAPITALIZATION_FOR_STANDALONE && capitalizationForStandAlone)) ) { + // titlecase first word of currentResult, here use sentence iterator unlike current implementations + // in LocaleDisplayNamesImpl::adjustForUsageAndContext and RelativeDateFormat::format + currentResult.toTitle(capitalizationBrkIter, locale, U_TITLECASE_NO_LOWERCASE | U_TITLECASE_NO_BREAK_ADJUSTMENT); + } + } +#endif + return currentResult; +} + + +void +RuleBasedNumberFormat::parse(const UnicodeString& text, + Formattable& result, + ParsePosition& parsePosition) const +{ + if (!fRuleSets) { + parsePosition.setErrorIndex(0); + return; + } + + UnicodeString workingText(text, parsePosition.getIndex()); + ParsePosition workingPos(0); + + ParsePosition high_pp(0); + Formattable high_result; + + for (NFRuleSet** p = fRuleSets; *p; ++p) { + NFRuleSet *rp = *p; + if (rp->isPublic() && rp->isParseable()) { + ParsePosition working_pp(0); + Formattable working_result; + + rp->parse(workingText, working_pp, kMaxDouble, 0, working_result); + if (working_pp.getIndex() > high_pp.getIndex()) { + high_pp = working_pp; + high_result = working_result; + + if (high_pp.getIndex() == workingText.length()) { + break; + } + } + } + } + + int32_t startIndex = parsePosition.getIndex(); + parsePosition.setIndex(startIndex + high_pp.getIndex()); + if (high_pp.getIndex() > 0) { + parsePosition.setErrorIndex(-1); + } else { + int32_t errorIndex = (high_pp.getErrorIndex()>0)? high_pp.getErrorIndex(): 0; + parsePosition.setErrorIndex(startIndex + errorIndex); + } + result = high_result; + if (result.getType() == Formattable::kDouble) { + double d = result.getDouble(); + if (!uprv_isNaN(d) && d == uprv_trunc(d) && INT32_MIN <= d && d <= INT32_MAX) { + // Note: casting a double to an int when the double is too large or small + // to fit the destination is undefined behavior. The explicit range checks, + // above, are required. Just casting and checking the result value is undefined. + result.setLong(static_cast(d)); + } + } +} + +#if !UCONFIG_NO_COLLATION + +void +RuleBasedNumberFormat::setLenient(UBool enabled) +{ + lenient = enabled; + if (!enabled && collator) { + delete collator; + collator = nullptr; + } +} + +#endif + +void +RuleBasedNumberFormat::setDefaultRuleSet(const UnicodeString& ruleSetName, UErrorCode& status) { + if (U_SUCCESS(status)) { + if (ruleSetName.isEmpty()) { + if (localizations) { + UnicodeString name(true, localizations->getRuleSetName(0), -1); + defaultRuleSet = findRuleSet(name, status); + } else { + initDefaultRuleSet(); + } + } else if (ruleSetName.startsWith(UNICODE_STRING_SIMPLE("%%"))) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + NFRuleSet* result = findRuleSet(ruleSetName, status); + if (result != nullptr) { + defaultRuleSet = result; + } + } + } +} + +UnicodeString +RuleBasedNumberFormat::getDefaultRuleSetName() const { + UnicodeString result; + if (defaultRuleSet && defaultRuleSet->isPublic()) { + defaultRuleSet->getName(result); + } else { + result.setToBogus(); + } + return result; +} + +void +RuleBasedNumberFormat::initDefaultRuleSet() +{ + defaultRuleSet = nullptr; + if (!fRuleSets) { + return; + } + + const UnicodeString spellout(UNICODE_STRING_SIMPLE("%spellout-numbering")); + const UnicodeString ordinal(UNICODE_STRING_SIMPLE("%digits-ordinal")); + const UnicodeString duration(UNICODE_STRING_SIMPLE("%duration")); + + NFRuleSet**p = &fRuleSets[0]; + while (*p) { + if ((*p)->isNamed(spellout) || (*p)->isNamed(ordinal) || (*p)->isNamed(duration)) { + defaultRuleSet = *p; + return; + } else { + ++p; + } + } + + defaultRuleSet = *--p; + if (!defaultRuleSet->isPublic()) { + while (p != fRuleSets) { + if ((*--p)->isPublic()) { + defaultRuleSet = *p; + break; + } + } + } +} + + +void +RuleBasedNumberFormat::init(const UnicodeString& rules, LocalizationInfo* localizationInfos, + UParseError& pErr, UErrorCode& status) +{ + // TODO: implement UParseError + uprv_memset(&pErr, 0, sizeof(UParseError)); + // Note: this can leave ruleSets == nullptr, so remaining code should check + if (U_FAILURE(status)) { + return; + } + + initializeDecimalFormatSymbols(status); + initializeDefaultInfinityRule(status); + initializeDefaultNaNRule(status); + if (U_FAILURE(status)) { + return; + } + + this->localizations = localizationInfos == nullptr ? nullptr : localizationInfos->ref(); + + UnicodeString description(rules); + if (!description.length()) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + // start by stripping the trailing whitespace from all the rules + // (this is all the whitespace following each semicolon in the + // description). This allows us to look for rule-set boundaries + // by searching for ";%" without having to worry about whitespace + // between the ; and the % + stripWhitespace(description); + + // check to see if there's a set of lenient-parse rules. If there + // is, pull them out into our temporary holding place for them, + // and delete them from the description before the real desciption- + // parsing code sees them + int32_t lp = description.indexOf(gLenientParse, -1, 0); + if (lp != -1) { + // we've got to make sure we're not in the middle of a rule + // (where "%%lenient-parse" would actually get treated as + // rule text) + if (lp == 0 || description.charAt(lp - 1) == gSemiColon) { + // locate the beginning and end of the actual collation + // rules (there may be whitespace between the name and + // the first token in the description) + int lpEnd = description.indexOf(gSemiPercent, 2, lp); + + if (lpEnd == -1) { + lpEnd = description.length() - 1; + } + int lpStart = lp + u_strlen(gLenientParse); + while (PatternProps::isWhiteSpace(description.charAt(lpStart))) { + ++lpStart; + } + + // copy out the lenient-parse rules and delete them + // from the description + lenientParseRules = new UnicodeString(); + /* test for nullptr */ + if (lenientParseRules == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + lenientParseRules->setTo(description, lpStart, lpEnd - lpStart); + + description.remove(lp, lpEnd + 1 - lp); + } + } + + // pre-flight parsing the description and count the number of + // rule sets (";%" marks the end of one rule set and the beginning + // of the next) + numRuleSets = 0; + for (int32_t p = description.indexOf(gSemiPercent, 2, 0); p != -1; p = description.indexOf(gSemiPercent, 2, p)) { + ++numRuleSets; + ++p; + } + ++numRuleSets; + + // our rule list is an array of the appropriate size + fRuleSets = (NFRuleSet **)uprv_malloc((numRuleSets + 1) * sizeof(NFRuleSet *)); + /* test for nullptr */ + if (fRuleSets == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + for (int i = 0; i <= numRuleSets; ++i) { + fRuleSets[i] = nullptr; + } + + // divide up the descriptions into individual rule-set descriptions + // and store them in a temporary array. At each step, we also + // new up a rule set, but all this does is initialize its name + // and remove it from its description. We can't actually parse + // the rest of the descriptions and finish initializing everything + // because we have to know the names and locations of all the rule + // sets before we can actually set everything up + if(!numRuleSets) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + ruleSetDescriptions = new UnicodeString[numRuleSets]; + if (ruleSetDescriptions == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + { + int curRuleSet = 0; + int32_t start = 0; + for (int32_t p = description.indexOf(gSemiPercent, 2, 0); p != -1; p = description.indexOf(gSemiPercent, 2, start)) { + ruleSetDescriptions[curRuleSet].setTo(description, start, p + 1 - start); + fRuleSets[curRuleSet] = new NFRuleSet(this, ruleSetDescriptions, curRuleSet, status); + if (fRuleSets[curRuleSet] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + ++curRuleSet; + start = p + 1; + } + ruleSetDescriptions[curRuleSet].setTo(description, start, description.length() - start); + fRuleSets[curRuleSet] = new NFRuleSet(this, ruleSetDescriptions, curRuleSet, status); + if (fRuleSets[curRuleSet] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + // now we can take note of the formatter's default rule set, which + // is the last public rule set in the description (it's the last + // rather than the first so that a user can create a new formatter + // from an existing formatter and change its default behavior just + // by appending more rule sets to the end) + + // {dlf} Initialization of a fraction rule set requires the default rule + // set to be known. For purposes of initialization, this is always the + // last public rule set, no matter what the localization data says. + initDefaultRuleSet(); + + // finally, we can go back through the temporary descriptions + // list and finish setting up the substructure (and we throw + // away the temporary descriptions as we go) + { + for (int i = 0; i < numRuleSets; i++) { + fRuleSets[i]->parseRules(ruleSetDescriptions[i], status); + } + } + + // Now that the rules are initialized, the 'real' default rule + // set can be adjusted by the localization data. + + // The C code keeps the localization array as is, rather than building + // a separate array of the public rule set names, so we have less work + // to do here-- but we still need to check the names. + + if (localizationInfos) { + // confirm the names, if any aren't in the rules, that's an error + // it is ok if the rules contain public rule sets that are not in this list + for (int32_t i = 0; i < localizationInfos->getNumberOfRuleSets(); ++i) { + UnicodeString name(true, localizationInfos->getRuleSetName(i), -1); + NFRuleSet* rs = findRuleSet(name, status); + if (rs == nullptr) { + break; // error + } + if (i == 0) { + defaultRuleSet = rs; + } + } + } else { + defaultRuleSet = getDefaultRuleSet(); + } + originalDescription = rules; +} + +// override the NumberFormat implementation in order to +// lazily initialize relevant items +void +RuleBasedNumberFormat::setContext(UDisplayContext value, UErrorCode& status) +{ + NumberFormat::setContext(value, status); + if (U_SUCCESS(status)) { + if (!capitalizationInfoSet && + (value==UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU || value==UDISPCTX_CAPITALIZATION_FOR_STANDALONE)) { + initCapitalizationContextInfo(locale); + capitalizationInfoSet = true; + } +#if !UCONFIG_NO_BREAK_ITERATION + if ( capitalizationBrkIter == nullptr && (value==UDISPCTX_CAPITALIZATION_FOR_BEGINNING_OF_SENTENCE || + (value==UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU && capitalizationForUIListMenu) || + (value==UDISPCTX_CAPITALIZATION_FOR_STANDALONE && capitalizationForStandAlone)) ) { + status = U_ZERO_ERROR; + capitalizationBrkIter = BreakIterator::createSentenceInstance(locale, status); + if (U_FAILURE(status)) { + delete capitalizationBrkIter; + capitalizationBrkIter = nullptr; + } + } +#endif + } +} + +void +RuleBasedNumberFormat::initCapitalizationContextInfo(const Locale& thelocale) +{ +#if !UCONFIG_NO_BREAK_ITERATION + const char * localeID = (thelocale != nullptr)? thelocale.getBaseName(): nullptr; + UErrorCode status = U_ZERO_ERROR; + UResourceBundle *rb = ures_open(nullptr, localeID, &status); + rb = ures_getByKeyWithFallback(rb, "contextTransforms", rb, &status); + rb = ures_getByKeyWithFallback(rb, "number-spellout", rb, &status); + if (U_SUCCESS(status) && rb != nullptr) { + int32_t len = 0; + const int32_t * intVector = ures_getIntVector(rb, &len, &status); + if (U_SUCCESS(status) && intVector != nullptr && len >= 2) { + capitalizationForUIListMenu = static_cast(intVector[0]); + capitalizationForStandAlone = static_cast(intVector[1]); + } + } + ures_close(rb); +#endif +} + +void +RuleBasedNumberFormat::stripWhitespace(UnicodeString& description) +{ + // iterate through the characters... + UnicodeString result; + + int start = 0; + while (start != -1 && start < description.length()) { + // seek to the first non-whitespace character... + while (start < description.length() + && PatternProps::isWhiteSpace(description.charAt(start))) { + ++start; + } + + // locate the next semicolon in the text and copy the text from + // our current position up to that semicolon into the result + int32_t p = description.indexOf(gSemiColon, start); + if (p == -1) { + // or if we don't find a semicolon, just copy the rest of + // the string into the result + result.append(description, start, description.length() - start); + start = -1; + } + else if (p < description.length()) { + result.append(description, start, p + 1 - start); + start = p + 1; + } + + // when we get here, we've seeked off the end of the string, and + // we terminate the loop (we continue until *start* is -1 rather + // than until *p* is -1, because otherwise we'd miss the last + // rule in the description) + else { + start = -1; + } + } + + description.setTo(result); +} + + +void +RuleBasedNumberFormat::dispose() +{ + if (fRuleSets) { + for (NFRuleSet** p = fRuleSets; *p; ++p) { + delete *p; + } + uprv_free(fRuleSets); + fRuleSets = nullptr; + } + + if (ruleSetDescriptions) { + delete [] ruleSetDescriptions; + ruleSetDescriptions = nullptr; + } + +#if !UCONFIG_NO_COLLATION + delete collator; +#endif + collator = nullptr; + + delete decimalFormatSymbols; + decimalFormatSymbols = nullptr; + + delete defaultInfinityRule; + defaultInfinityRule = nullptr; + + delete defaultNaNRule; + defaultNaNRule = nullptr; + + delete lenientParseRules; + lenientParseRules = nullptr; + +#if !UCONFIG_NO_BREAK_ITERATION + delete capitalizationBrkIter; + capitalizationBrkIter = nullptr; +#endif + + if (localizations) { + localizations = localizations->unref(); + } +} + + +//----------------------------------------------------------------------- +// package-internal API +//----------------------------------------------------------------------- + +/** + * Returns the collator to use for lenient parsing. The collator is lazily created: + * this function creates it the first time it's called. + * @return The collator to use for lenient parsing, or null if lenient parsing + * is turned off. +*/ +const RuleBasedCollator* +RuleBasedNumberFormat::getCollator() const +{ +#if !UCONFIG_NO_COLLATION + if (!fRuleSets) { + return nullptr; + } + + // lazy-evaluate the collator + if (collator == nullptr && lenient) { + // create a default collator based on the formatter's locale, + // then pull out that collator's rules, append any additional + // rules specified in the description, and create a _new_ + // collator based on the combination of those rules + + UErrorCode status = U_ZERO_ERROR; + + Collator* temp = Collator::createInstance(locale, status); + RuleBasedCollator* newCollator; + if (U_SUCCESS(status) && (newCollator = dynamic_cast(temp)) != nullptr) { + if (lenientParseRules) { + UnicodeString rules(newCollator->getRules()); + rules.append(*lenientParseRules); + + newCollator = new RuleBasedCollator(rules, status); + // Exit if newCollator could not be created. + if (newCollator == nullptr) { + return nullptr; + } + } else { + temp = nullptr; + } + if (U_SUCCESS(status)) { + newCollator->setAttribute(UCOL_DECOMPOSITION_MODE, UCOL_ON, status); + // cast away const + ((RuleBasedNumberFormat*)this)->collator = newCollator; + } else { + delete newCollator; + } + } + delete temp; + } +#endif + + // if lenient-parse mode is off, this will be null + // (see setLenientParseMode()) + return collator; +} + + +DecimalFormatSymbols* +RuleBasedNumberFormat::initializeDecimalFormatSymbols(UErrorCode &status) +{ + // lazy-evaluate the DecimalFormatSymbols object. This object + // is shared by all DecimalFormat instances belonging to this + // formatter + if (decimalFormatSymbols == nullptr) { + LocalPointer temp(new DecimalFormatSymbols(locale, status), status); + if (U_SUCCESS(status)) { + decimalFormatSymbols = temp.orphan(); + } + } + return decimalFormatSymbols; +} + +/** + * Returns the DecimalFormatSymbols object that should be used by all DecimalFormat + * instances owned by this formatter. +*/ +const DecimalFormatSymbols* +RuleBasedNumberFormat::getDecimalFormatSymbols() const +{ + return decimalFormatSymbols; +} + +NFRule* +RuleBasedNumberFormat::initializeDefaultInfinityRule(UErrorCode &status) +{ + if (U_FAILURE(status)) { + return nullptr; + } + if (defaultInfinityRule == nullptr) { + UnicodeString rule(UNICODE_STRING_SIMPLE("Inf: ")); + rule.append(getDecimalFormatSymbols()->getSymbol(DecimalFormatSymbols::kInfinitySymbol)); + LocalPointer temp(new NFRule(this, rule, status), status); + if (U_SUCCESS(status)) { + defaultInfinityRule = temp.orphan(); + } + } + return defaultInfinityRule; +} + +const NFRule* +RuleBasedNumberFormat::getDefaultInfinityRule() const +{ + return defaultInfinityRule; +} + +NFRule* +RuleBasedNumberFormat::initializeDefaultNaNRule(UErrorCode &status) +{ + if (U_FAILURE(status)) { + return nullptr; + } + if (defaultNaNRule == nullptr) { + UnicodeString rule(UNICODE_STRING_SIMPLE("NaN: ")); + rule.append(getDecimalFormatSymbols()->getSymbol(DecimalFormatSymbols::kNaNSymbol)); + LocalPointer temp(new NFRule(this, rule, status), status); + if (U_SUCCESS(status)) { + defaultNaNRule = temp.orphan(); + } + } + return defaultNaNRule; +} + +const NFRule* +RuleBasedNumberFormat::getDefaultNaNRule() const +{ + return defaultNaNRule; +} + +// De-owning the current localized symbols and adopt the new symbols. +void +RuleBasedNumberFormat::adoptDecimalFormatSymbols(DecimalFormatSymbols* symbolsToAdopt) +{ + if (symbolsToAdopt == nullptr) { + return; // do not allow caller to set decimalFormatSymbols to nullptr + } + + if (decimalFormatSymbols != nullptr) { + delete decimalFormatSymbols; + } + + decimalFormatSymbols = symbolsToAdopt; + + { + // Apply the new decimalFormatSymbols by reparsing the rulesets + UErrorCode status = U_ZERO_ERROR; + + delete defaultInfinityRule; + defaultInfinityRule = nullptr; + initializeDefaultInfinityRule(status); // Reset with the new DecimalFormatSymbols + + delete defaultNaNRule; + defaultNaNRule = nullptr; + initializeDefaultNaNRule(status); // Reset with the new DecimalFormatSymbols + + if (fRuleSets) { + for (int32_t i = 0; i < numRuleSets; i++) { + fRuleSets[i]->setDecimalFormatSymbols(*symbolsToAdopt, status); + } + } + } +} + +// Setting the symbols is equivalent to adopting a newly created localized symbols. +void +RuleBasedNumberFormat::setDecimalFormatSymbols(const DecimalFormatSymbols& symbols) +{ + adoptDecimalFormatSymbols(new DecimalFormatSymbols(symbols)); +} + +PluralFormat * +RuleBasedNumberFormat::createPluralFormat(UPluralType pluralType, + const UnicodeString &pattern, + UErrorCode& status) const +{ + auto *pf = new PluralFormat(locale, pluralType, pattern, status); + if (pf == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return pf; +} + +/** + * Get the rounding mode. + * @return A rounding mode + */ +DecimalFormat::ERoundingMode RuleBasedNumberFormat::getRoundingMode() const { + return fRoundingMode; +} + +/** + * Set the rounding mode. This has no effect unless the rounding + * increment is greater than zero. + * @param roundingMode A rounding mode + */ +void RuleBasedNumberFormat::setRoundingMode(DecimalFormat::ERoundingMode roundingMode) { + fRoundingMode = roundingMode; +} + +U_NAMESPACE_END + +/* U_HAVE_RBNF */ +#endif diff --git a/intl/icu/source/i18n/rbt.cpp b/intl/icu/source/i18n/rbt.cpp new file mode 100644 index 0000000000..2176e89fdd --- /dev/null +++ b/intl/icu/source/i18n/rbt.cpp @@ -0,0 +1,307 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2015, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/rep.h" +#include "unicode/uniset.h" +#include "rbt_pars.h" +#include "rbt_data.h" +#include "rbt_rule.h" +#include "rbt.h" +#include "mutex.h" +#include "umutex.h" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedTransliterator) + +static Replaceable *gLockedText = nullptr; + +void RuleBasedTransliterator::_construct(const UnicodeString& rules, + UTransDirection direction, + UParseError& parseError, + UErrorCode& status) { + fData = 0; + isDataOwned = true; + if (U_FAILURE(status)) { + return; + } + + TransliteratorParser parser(status); + parser.parse(rules, direction, parseError, status); + if (U_FAILURE(status)) { + return; + } + + if (parser.idBlockVector.size() != 0 || + parser.compoundFilter != nullptr || + parser.dataVector.size() == 0) { + status = U_INVALID_RBT_SYNTAX; // ::ID blocks disallowed in RBT + return; + } + + fData = (TransliterationRuleData*)parser.dataVector.orphanElementAt(0); + setMaximumContextLength(fData->ruleSet.getMaximumContextLength()); +} + +/** + * Constructs a new transliterator from the given rules. + * @param id the id for the transliterator. + * @param rules rules, separated by ';' + * @param direction either FORWARD or REVERSE. + * @param adoptedFilter the filter for this transliterator. + * @param parseError Struct to receive information on position + * of error if an error is encountered + * @param status Output param set to success/failure code. + * @exception IllegalArgumentException if rules are malformed + * or direction is invalid. + */ +RuleBasedTransliterator::RuleBasedTransliterator( + const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UnicodeFilter* adoptedFilter, + UParseError& parseError, + UErrorCode& status) : + Transliterator(id, adoptedFilter) { + _construct(rules, direction,parseError,status); +} + +/** + * Constructs a new transliterator from the given rules. + * @param id the id for the transliterator. + * @param rules rules, separated by ';' + * @param direction either FORWARD or REVERSE. + * @param adoptedFilter the filter for this transliterator. + * @param status Output param set to success/failure code. + * @exception IllegalArgumentException if rules are malformed + * or direction is invalid. + */ +/*RuleBasedTransliterator::RuleBasedTransliterator( + const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UnicodeFilter* adoptedFilter, + UErrorCode& status) : + Transliterator(id, adoptedFilter) { + UParseError parseError; + _construct(rules, direction,parseError, status); +}*/ + +/** + * Convenience constructor with no filter. + */ +/*RuleBasedTransliterator::RuleBasedTransliterator( + const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UErrorCode& status) : + Transliterator(id, 0) { + UParseError parseError; + _construct(rules, direction,parseError, status); +}*/ + +/** + * Convenience constructor with no filter and FORWARD direction. + */ +/*RuleBasedTransliterator::RuleBasedTransliterator( + const UnicodeString& id, + const UnicodeString& rules, + UErrorCode& status) : + Transliterator(id, 0) { + UParseError parseError; + _construct(rules, UTRANS_FORWARD, parseError, status); +}*/ + +/** + * Convenience constructor with FORWARD direction. + */ +/*RuleBasedTransliterator::RuleBasedTransliterator( + const UnicodeString& id, + const UnicodeString& rules, + UnicodeFilter* adoptedFilter, + UErrorCode& status) : + Transliterator(id, adoptedFilter) { + UParseError parseError; + _construct(rules, UTRANS_FORWARD,parseError, status); +}*/ + +RuleBasedTransliterator::RuleBasedTransliterator(const UnicodeString& id, + const TransliterationRuleData* theData, + UnicodeFilter* adoptedFilter) : + Transliterator(id, adoptedFilter), + fData((TransliterationRuleData*)theData), // cast away const + isDataOwned(false) { + setMaximumContextLength(fData->ruleSet.getMaximumContextLength()); +} + +/** + * Internal constructor. + */ +RuleBasedTransliterator::RuleBasedTransliterator(const UnicodeString& id, + TransliterationRuleData* theData, + UBool isDataAdopted) : + Transliterator(id, 0), + fData(theData), + isDataOwned(isDataAdopted) { + setMaximumContextLength(fData->ruleSet.getMaximumContextLength()); +} + +/** + * Copy constructor. + */ +RuleBasedTransliterator::RuleBasedTransliterator( + const RuleBasedTransliterator& other) : + Transliterator(other), fData(other.fData), + isDataOwned(other.isDataOwned) { + + // The data object may or may not be owned. If it is not owned we + // share it; it is invariant. If it is owned, it's still + // invariant, but we need to copy it to prevent double-deletion. + // If this becomes a performance issue (if people do a lot of RBT + // copying -- unlikely) we can reference count the data object. + + // Only do a deep copy if this is owned data, that is, data that + // will be later deleted. System transliterators contain + // non-owned data. + if (isDataOwned) { + fData = new TransliterationRuleData(*other.fData); + } +} + +/** + * Destructor. + */ +RuleBasedTransliterator::~RuleBasedTransliterator() { + // Delete the data object only if we own it. + if (isDataOwned) { + delete fData; + } +} + +RuleBasedTransliterator* +RuleBasedTransliterator::clone() const { + return new RuleBasedTransliterator(*this); +} + +/** + * Implements {@link Transliterator#handleTransliterate}. + */ +void +RuleBasedTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index, + UBool isIncremental) const { + /* We keep contextStart and contextLimit fixed the entire time, + * relative to the text -- contextLimit may move numerically if + * text is inserted or removed. The start offset moves toward + * limit, with replacements happening under it. + * + * Example: rules 1. ab>x|y + * 2. yc>z + * + * |eabcd begin - no match, advance start + * e|abcd match rule 1 - change text & adjust start + * ex|ycd match rule 2 - change text & adjust start + * exz|d no match, advance start + * exzd| done + */ + + /* A rule like + * a>b|a + * creates an infinite loop. To prevent that, we put an arbitrary + * limit on the number of iterations that we take, one that is + * high enough that any reasonable rules are ok, but low enough to + * prevent a server from hanging. The limit is 16 times the + * number of characters n, unless n is so large that 16n exceeds a + * uint32_t. + */ + uint32_t loopCount = 0; + uint32_t loopLimit = index.limit - index.start; + if (loopLimit >= 0x10000000) { + loopLimit = 0xFFFFFFFF; + } else { + loopLimit <<= 4; + } + + // Transliterator locking. Rule-based Transliterators are not thread safe; concurrent + // operations must be prevented. + // A Complication: compound transliterators can result in recursive entries to this + // function, sometimes with different "This" objects, always with the same text. + // Double-locking must be prevented in these cases. + // + + UBool lockedMutexAtThisLevel = false; + + // Test whether this request is operating on the same text string as + // some other transliteration that is still in progress and holding the + // transliteration mutex. If so, do not lock the transliteration + // mutex again. + // + // gLockedText variable is protected by the global ICU mutex. + // Shared RBT data protected by transliteratorDataMutex. + // + // TODO(andy): Need a better scheme for handling this. + + static UMutex transliteratorDataMutex; + UBool needToLock; + { + Mutex m; + needToLock = (&text != gLockedText); + } + if (needToLock) { + umtx_lock(&transliteratorDataMutex); // Contention, longish waits possible here. + Mutex m; + gLockedText = &text; + lockedMutexAtThisLevel = true; + } + + // Check to make sure we don't dereference a null pointer. + if (fData != nullptr) { + while (index.start < index.limit && + loopCount <= loopLimit && + fData->ruleSet.transliterate(text, index, isIncremental)) { + ++loopCount; + } + } + if (lockedMutexAtThisLevel) { + { + Mutex m; + gLockedText = nullptr; + } + umtx_unlock(&transliteratorDataMutex); + } +} + +UnicodeString& RuleBasedTransliterator::toRules(UnicodeString& rulesSource, + UBool escapeUnprintable) const { + return fData->ruleSet.toRules(rulesSource, escapeUnprintable); +} + +/** + * Implement Transliterator framework + */ +void RuleBasedTransliterator::handleGetSourceSet(UnicodeSet& result) const { + fData->ruleSet.getSourceTargetSet(result, false); +} + +/** + * Override Transliterator framework + */ +UnicodeSet& RuleBasedTransliterator::getTargetSet(UnicodeSet& result) const { + return fData->ruleSet.getSourceTargetSet(result, true); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/rbt.h b/intl/icu/source/i18n/rbt.h new file mode 100644 index 0000000000..59fa286a45 --- /dev/null +++ b/intl/icu/source/i18n/rbt.h @@ -0,0 +1,223 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef RBT_H +#define RBT_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" +#include "unicode/utypes.h" +#include "unicode/parseerr.h" +#include "unicode/udata.h" + +#define U_ICUDATA_TRANSLIT U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "translit" + +U_NAMESPACE_BEGIN + +class TransliterationRuleData; + +/** + * RuleBasedTransliterator is a transliterator + * built from a set of rules as defined for + * Transliterator::createFromRules(). + * See the C++ class Transliterator documentation for the rule syntax. + * + * @author Alan Liu + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ +class RuleBasedTransliterator : public Transliterator { +private: + /** + * The data object is immutable, so we can freely share it with + * other instances of RBT, as long as we do NOT own this object. + * TODO: data is no longer immutable. See bugs #1866, 2155 + */ + TransliterationRuleData* fData; + + /** + * If true, we own the data object and must delete it. + */ + UBool isDataOwned; + +public: + + /** + * Constructs a new transliterator from the given rules. + * @param rules rules, separated by ';' + * @param direction either FORWARD or REVERSE. + * @exception IllegalArgumentException if rules are malformed. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + RuleBasedTransliterator(const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UnicodeFilter* adoptedFilter, + UParseError& parseError, + UErrorCode& status); + + /** + * Constructs a new transliterator from the given rules. + * @param rules rules, separated by ';' + * @param direction either FORWARD or REVERSE. + * @exception IllegalArgumentException if rules are malformed. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + /*RuleBasedTransliterator(const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UnicodeFilter* adoptedFilter, + UErrorCode& status);*/ + + /** + * Convenience constructor with no filter. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + /*RuleBasedTransliterator(const UnicodeString& id, + const UnicodeString& rules, + UTransDirection direction, + UErrorCode& status);*/ + + /** + * Convenience constructor with no filter and FORWARD direction. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + /*RuleBasedTransliterator(const UnicodeString& id, + const UnicodeString& rules, + UErrorCode& status);*/ + + /** + * Convenience constructor with FORWARD direction. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + /*RuleBasedTransliterator(const UnicodeString& id, + const UnicodeString& rules, + UnicodeFilter* adoptedFilter, + UErrorCode& status);*/ +private: + + friend class TransliteratorRegistry; // to access TransliterationRuleData convenience ctor + /** + * Convenience constructor. + * @param id the id for the transliterator. + * @param theData the rule data for the transliterator. + * @param adoptedFilter the filter for the transliterator + */ + RuleBasedTransliterator(const UnicodeString& id, + const TransliterationRuleData* theData, + UnicodeFilter* adoptedFilter = 0); + + + friend class Transliterator; // to access following ct + + /** + * Internal constructor. + * @param id the id for the transliterator. + * @param theData the rule data for the transliterator. + * @param isDataAdopted determine who will own the 'data' object. True, the caller should not delete 'data'. + */ + RuleBasedTransliterator(const UnicodeString& id, + TransliterationRuleData* data, + UBool isDataAdopted); + +public: + + /** + * Copy constructor. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + RuleBasedTransliterator(const RuleBasedTransliterator&); + + virtual ~RuleBasedTransliterator(); + + /** + * Implement Transliterator API. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual RuleBasedTransliterator* clone() const override; + +protected: + /** + * Implements {@link Transliterator#handleTransliterate}. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offsets, + UBool isIncremental) const override; + +public: + /** + * Return a representation of this transliterator as source rules. + * These rules will produce an equivalent transliterator if used + * to construct a new transliterator. + * @param result the string to receive the rules. Previous + * contents will be deleted. + * @param escapeUnprintable if true then convert unprintable + * character to their hex escape representations, \uxxxx or + * \Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + virtual UnicodeString& toRules(UnicodeString& result, + UBool escapeUnprintable) const override; + +protected: + /** + * Implement Transliterator framework + */ + virtual void handleGetSourceSet(UnicodeSet& result) const override; + +public: + /** + * Override Transliterator framework + */ + virtual UnicodeSet& getTargetSet(UnicodeSet& result) const override; + + /** + * Return the class ID for this class. This is useful only for + * comparing to a return value from getDynamicClassID(). For example: + * 

    +     * .      Base* polymorphic_pointer = createPolymorphicObject();
+     * .      if (polymorphic_pointer->getDynamicClassID() ==
+     * .          Derived::getStaticClassID()) ...
+     *
    + * @return The class ID for all objects of this class. + * @internal Use transliterator factory methods instead since this class will be removed in that release. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * Returns a unique class ID polymorphically. This method + * is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and + * clone() methods call this method. + * + * @return The class ID for this object. All objects of a given + * class have the same class ID. Objects of other classes have + * different class IDs. + */ + virtual UClassID getDynamicClassID() const override; + +private: + + void _construct(const UnicodeString& rules, + UTransDirection direction, + UParseError& parseError, + UErrorCode& status); +}; + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/rbt_data.cpp b/intl/icu/source/i18n/rbt_data.cpp new file mode 100644 index 0000000000..f4212848cb --- /dev/null +++ b/intl/icu/source/i18n/rbt_data.cpp @@ -0,0 +1,119 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" +#include "umutex.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "rbt_data.h" +#include "hash.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +TransliterationRuleData::TransliterationRuleData(UErrorCode& status) + : UMemory(), ruleSet(status), variableNames(status), + variables(0), variablesAreOwned(true) +{ + if (U_FAILURE(status)) { + return; + } + variableNames.setValueDeleter(uprv_deleteUObject); + variables = 0; + variablesLength = 0; +} + +TransliterationRuleData::TransliterationRuleData(const TransliterationRuleData& other) : + UMemory(other), ruleSet(other.ruleSet), + variablesAreOwned(true), + variablesBase(other.variablesBase), + variablesLength(other.variablesLength) +{ + UErrorCode status = U_ZERO_ERROR; + int32_t i = 0; + variableNames.setValueDeleter(uprv_deleteUObject); + int32_t pos = UHASH_FIRST; + const UHashElement *e; + while ((e = other.variableNames.nextElement(pos)) != 0) { + UnicodeString* value = + new UnicodeString(*(const UnicodeString*)e->value.pointer); + // Exit out if value could not be created. + if (value == nullptr) { + return; + } + variableNames.put(*(UnicodeString*)e->key.pointer, value, status); + } + + variables = 0; + if (other.variables != 0) { + variables = (UnicodeFunctor **)uprv_malloc(variablesLength * sizeof(UnicodeFunctor *)); + /* test for nullptr */ + if (variables == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (i=0; iclone(); + if (variables[i] == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + } + } + // Remove the array and exit if memory allocation error occurred. + if (U_FAILURE(status)) { + for (int32_t n = i-1; n >= 0; n--) { + delete variables[n]; + } + uprv_free(variables); + variables = nullptr; + return; + } + + // Do this last, _after_ setting up variables[]. + ruleSet.setData(this); // ruleSet must already be frozen +} + +TransliterationRuleData::~TransliterationRuleData() { + if (variablesAreOwned && variables != 0) { + for (int32_t i=0; i= 0 && i < variablesLength) ? variables[i] : 0; +} + +UnicodeMatcher* +TransliterationRuleData::lookupMatcher(UChar32 standIn) const { + UnicodeFunctor *f = lookup(standIn); + return (f != 0) ? f->toMatcher() : 0; +} + +UnicodeReplacer* +TransliterationRuleData::lookupReplacer(UChar32 standIn) const { + UnicodeFunctor *f = lookup(standIn); + return (f != 0) ? f->toReplacer() : 0; +} + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/rbt_data.h b/intl/icu/source/i18n/rbt_data.h new file mode 100644 index 0000000000..43cbb4795b --- /dev/null +++ b/intl/icu/source/i18n/rbt_data.h @@ -0,0 +1,154 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef RBT_DATA_H +#define RBT_DATA_H + +#include "unicode/utypes.h" +#include "unicode/uclean.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "rbt_set.h" +#include "hash.h" + +U_NAMESPACE_BEGIN + +class UnicodeFunctor; +class UnicodeMatcher; +class UnicodeReplacer; + +/** + * The rule data for a RuleBasedTransliterators. RBT objects hold + * a const pointer to a TRD object that they do not own. TRD objects + * are essentially the parsed rules in compact, usable form. The + * TRD objects themselves are held for the life of the process in + * a static cache owned by Transliterator. + * + * This class' API is a little asymmetric. There is a method to + * define a variable, but no way to define a set. This is because the + * sets are defined by the parser in a UVector, and the vector is + * copied into a fixed-size array here. Once this is done, no new + * sets may be defined. In practice, there is no need to do so, since + * generating the data and using it are discrete phases. When there + * is a need to access the set data during the parse phase, another + * data structure handles this. See the parsing code for more + * details. + */ +class TransliterationRuleData : public UMemory { + +public: + + // PUBLIC DATA MEMBERS + + /** + * Rule table. May be empty. + */ + TransliterationRuleSet ruleSet; + + /** + * Map variable name (String) to variable (UnicodeString). A variable name + * corresponds to zero or more characters, stored in a UnicodeString in + * this hash. One or more of these chars may also correspond to a + * UnicodeMatcher, in which case the character in the UnicodeString in this hash is + * a stand-in: it is an index for a secondary lookup in + * data.variables. The stand-in also represents the UnicodeMatcher in + * the stored rules. + */ + Hashtable variableNames; + + /** + * Map category variable (char16_t) to set (UnicodeFunctor). + * Variables that correspond to a set of characters are mapped + * from variable name to a stand-in character in data.variableNames. + * The stand-in then serves as a key in this hash to lookup the + * actual UnicodeFunctor object. In addition, the stand-in is + * stored in the rule text to represent the set of characters. + * variables[i] represents character (variablesBase + i). + */ + UnicodeFunctor** variables; + + /** + * Flag that indicates whether the variables are owned (if a single + * call to Transliterator::createFromRules() produces a CompoundTransliterator + * with more than one RuleBasedTransliterator as children, they all share + * the same variables list, so only the first one is considered to own + * the variables) + */ + UBool variablesAreOwned; + + /** + * The character that represents variables[0]. Characters + * variablesBase through variablesBase + + * variablesLength - 1 represent UnicodeFunctor objects. + */ + char16_t variablesBase; + + /** + * The length of variables. + */ + int32_t variablesLength; + +public: + + /** + * Constructor + * @param status Output param set to success/failure code on exit. + */ + TransliterationRuleData(UErrorCode& status); + + /** + * Copy Constructor + */ + TransliterationRuleData(const TransliterationRuleData&); + + /** + * destructor + */ + ~TransliterationRuleData(); + + /** + * Given a stand-in character, return the UnicodeFunctor that it + * represents, or nullptr if it doesn't represent anything. + * @param standIn the given stand-in character. + * @return the UnicodeFunctor that 'standIn' represents + */ + UnicodeFunctor* lookup(UChar32 standIn) const; + + /** + * Given a stand-in character, return the UnicodeMatcher that it + * represents, or nullptr if it doesn't represent anything or if it + * represents something that is not a matcher. + * @param standIn the given stand-in character. + * @return return the UnicodeMatcher that 'standIn' represents + */ + UnicodeMatcher* lookupMatcher(UChar32 standIn) const; + + /** + * Given a stand-in character, return the UnicodeReplacer that it + * represents, or nullptr if it doesn't represent anything or if it + * represents something that is not a replacer. + * @param standIn the given stand-in character. + * @return return the UnicodeReplacer that 'standIn' represents + */ + UnicodeReplacer* lookupReplacer(UChar32 standIn) const; + + +private: + TransliterationRuleData &operator=(const TransliterationRuleData &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/rbt_pars.cpp b/intl/icu/source/i18n/rbt_pars.cpp new file mode 100644 index 0000000000..10482d5edb --- /dev/null +++ b/intl/icu/source/i18n/rbt_pars.cpp @@ -0,0 +1,1773 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 1999-2016, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Date Name Description + * 11/17/99 aliu Creation. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "unicode/parseerr.h" +#include "unicode/parsepos.h" +#include "unicode/putil.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/uniset.h" +#include "unicode/utf16.h" +#include "cstring.h" +#include "funcrepl.h" +#include "hash.h" +#include "quant.h" +#include "rbt.h" +#include "rbt_data.h" +#include "rbt_pars.h" +#include "rbt_rule.h" +#include "strmatch.h" +#include "strrepl.h" +#include "unicode/symtable.h" +#include "tridpars.h" +#include "uvector.h" +#include "hash.h" +#include "patternprops.h" +#include "util.h" +#include "cmemory.h" +#include "uprops.h" +#include "putilimp.h" + +// Operators +#define VARIABLE_DEF_OP ((char16_t)0x003D) /*=*/ +#define FORWARD_RULE_OP ((char16_t)0x003E) /*>*/ +#define REVERSE_RULE_OP ((char16_t)0x003C) /*<*/ +#define FWDREV_RULE_OP ((char16_t)0x007E) /*~*/ // internal rep of <> op + +// Other special characters +#define QUOTE ((char16_t)0x0027) /*'*/ +#define ESCAPE ((char16_t)0x005C) /*\*/ +#define END_OF_RULE ((char16_t)0x003B) /*;*/ +#define RULE_COMMENT_CHAR ((char16_t)0x0023) /*#*/ + +#define SEGMENT_OPEN ((char16_t)0x0028) /*(*/ +#define SEGMENT_CLOSE ((char16_t)0x0029) /*)*/ +#define CONTEXT_ANTE ((char16_t)0x007B) /*{*/ +#define CONTEXT_POST ((char16_t)0x007D) /*}*/ +#define CURSOR_POS ((char16_t)0x007C) /*|*/ +#define CURSOR_OFFSET ((char16_t)0x0040) /*@*/ +#define ANCHOR_START ((char16_t)0x005E) /*^*/ +#define KLEENE_STAR ((char16_t)0x002A) /***/ +#define ONE_OR_MORE ((char16_t)0x002B) /*+*/ +#define ZERO_OR_ONE ((char16_t)0x003F) /*?*/ + +#define DOT ((char16_t)46) /*.*/ + +static const char16_t DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]"; + 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90, + 108, 58, 93, 92, 114, 92, 110, 36, 93, 0 +}; + +// A function is denoted &Source-Target/Variant(text) +#define FUNCTION ((char16_t)38) /*&*/ + +// Aliases for some of the syntax characters. These are provided so +// transliteration rules can be expressed in XML without clashing with +// XML syntax characters '<', '>', and '&'. +#define ALT_REVERSE_RULE_OP ((char16_t)0x2190) // Left Arrow +#define ALT_FORWARD_RULE_OP ((char16_t)0x2192) // Right Arrow +#define ALT_FWDREV_RULE_OP ((char16_t)0x2194) // Left Right Arrow +#define ALT_FUNCTION ((char16_t)0x2206) // Increment (~Greek Capital Delta) + +// Special characters disallowed at the top level +static const char16_t ILLEGAL_TOP[] = {41,0}; // ")" + +// Special characters disallowed within a segment +static const char16_t ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@" + +// Special characters disallowed within a function argument +static const char16_t ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@" + +// By definition, the ANCHOR_END special character is a +// trailing SymbolTable.SYMBOL_REF character. +// private static final char ANCHOR_END = '$'; + +static const char16_t gOPERATORS[] = { // "=><" + VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, + ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, + 0 +}; + +static const char16_t HALF_ENDERS[] = { // "=><;" + VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, + ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, + END_OF_RULE, + 0 +}; + +// These are also used in Transliterator::toRules() +static const int32_t ID_TOKEN_LEN = 2; +static const char16_t ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':' + +/* +commented out until we do real ::BEGIN/::END functionality +static const int32_t BEGIN_TOKEN_LEN = 5; +static const char16_t BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN' + +static const int32_t END_TOKEN_LEN = 3; +static const char16_t END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END' +*/ + +U_NAMESPACE_BEGIN + +//---------------------------------------------------------------------- +// BEGIN ParseData +//---------------------------------------------------------------------- + +/** + * This class implements the SymbolTable interface. It is used + * during parsing to give UnicodeSet access to variables that + * have been defined so far. Note that it uses variablesVector, + * _not_ data.setVariables. + */ +class ParseData : public UMemory, public SymbolTable { +public: + const TransliterationRuleData* data; // alias + + const UVector* variablesVector; // alias + + const Hashtable* variableNames; // alias + + ParseData(const TransliterationRuleData* data = 0, + const UVector* variablesVector = 0, + const Hashtable* variableNames = 0); + + virtual ~ParseData(); + + virtual const UnicodeString* lookup(const UnicodeString& s) const override; + + virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const override; + + virtual UnicodeString parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const override; + /** + * Return true if the given character is a matcher standin or a plain + * character (non standin). + */ + UBool isMatcher(UChar32 ch); + + /** + * Return true if the given character is a replacer standin or a plain + * character (non standin). + */ + UBool isReplacer(UChar32 ch); + +private: + ParseData(const ParseData &other); // forbid copying of this class + ParseData &operator=(const ParseData &other); // forbid copying of this class +}; + +ParseData::ParseData(const TransliterationRuleData* d, + const UVector* sets, + const Hashtable* vNames) : + data(d), variablesVector(sets), variableNames(vNames) {} + +ParseData::~ParseData() {} + +/** + * Implement SymbolTable API. + */ +const UnicodeString* ParseData::lookup(const UnicodeString& name) const { + return (const UnicodeString*) variableNames->get(name); +} + +/** + * Implement SymbolTable API. + */ +const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const { + // Note that we cannot use data.lookupSet() because the + // set array has not been constructed yet. + const UnicodeFunctor* set = nullptr; + int32_t i = ch - data->variablesBase; + if (i >= 0 && i < variablesVector->size()) { + int32_t j = ch - data->variablesBase; + set = (j < variablesVector->size()) ? + (UnicodeFunctor*) variablesVector->elementAt(j) : 0; + } + return set; +} + +/** + * Implement SymbolTable API. Parse out a symbol reference + * name. + */ +UnicodeString ParseData::parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const { + int32_t start = pos.getIndex(); + int32_t i = start; + UnicodeString result; + while (i < limit) { + char16_t c = text.charAt(i); + if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { + break; + } + ++i; + } + if (i == start) { // No valid name chars + return result; // Indicate failure with empty string + } + pos.setIndex(i); + text.extractBetween(start, i, result); + return result; +} + +UBool ParseData::isMatcher(UChar32 ch) { + // Note that we cannot use data.lookup() because the + // set array has not been constructed yet. + int32_t i = ch - data->variablesBase; + if (i >= 0 && i < variablesVector->size()) { + UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); + return f != nullptr && f->toMatcher() != nullptr; + } + return true; +} + +/** + * Return true if the given character is a replacer standin or a plain + * character (non standin). + */ +UBool ParseData::isReplacer(UChar32 ch) { + // Note that we cannot use data.lookup() because the + // set array has not been constructed yet. + int i = ch - data->variablesBase; + if (i >= 0 && i < variablesVector->size()) { + UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); + return f != nullptr && f->toReplacer() != nullptr; + } + return true; +} + +//---------------------------------------------------------------------- +// BEGIN RuleHalf +//---------------------------------------------------------------------- + +/** + * A class representing one side of a rule. This class knows how to + * parse half of a rule. It is tightly coupled to the method + * RuleBasedTransliterator.Parser.parseRule(). + */ +class RuleHalf : public UMemory { + +public: + + UnicodeString text; + + int32_t cursor; // position of cursor in text + int32_t ante; // position of ante context marker '{' in text + int32_t post; // position of post context marker '}' in text + + // Record the offset to the cursor either to the left or to the + // right of the key. This is indicated by characters on the output + // side that allow the cursor to be positioned arbitrarily within + // the matching text. For example, abc{def} > | @@@ xyz; changes + // def to xyz and moves the cursor to before abc. Offset characters + // must be at the start or end, and they cannot move the cursor past + // the ante- or postcontext text. Placeholders are only valid in + // output text. The length of the ante and post context is + // determined at runtime, because of supplementals and quantifiers. + int32_t cursorOffset; // only nonzero on output side + + // Position of first CURSOR_OFFSET on _right_. This will be -1 + // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc. + int32_t cursorOffsetPos; + + UBool anchorStart; + UBool anchorEnd; + + /** + * The segment number from 1..n of the next '(' we see + * during parsing; 1-based. + */ + int32_t nextSegmentNumber; + + TransliteratorParser& parser; + + //-------------------------------------------------- + // Methods + + RuleHalf(TransliteratorParser& parser); + ~RuleHalf(); + + int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); + + int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, + UnicodeString& buf, + const UnicodeString& illegal, + UBool isSegment, + UErrorCode& status); + + /** + * Remove context. + */ + void removeContext(); + + /** + * Return true if this half looks like valid output, that is, does not + * contain quantifiers or other special input-only elements. + */ + UBool isValidOutput(TransliteratorParser& parser); + + /** + * Return true if this half looks like valid input, that is, does not + * contain functions or other special output-only elements. + */ + UBool isValidInput(TransliteratorParser& parser); + + int syntaxError(UErrorCode code, + const UnicodeString& rule, + int32_t start, + UErrorCode& status) { + return parser.syntaxError(code, rule, start, status); + } + +private: + // Disallowed methods; no impl. + RuleHalf(const RuleHalf&); + RuleHalf& operator=(const RuleHalf&); +}; + +RuleHalf::RuleHalf(TransliteratorParser& p) : + parser(p) +{ + cursor = -1; + ante = -1; + post = -1; + cursorOffset = 0; + cursorOffsetPos = 0; + anchorStart = anchorEnd = false; + nextSegmentNumber = 1; +} + +RuleHalf::~RuleHalf() { +} + +/** + * Parse one side of a rule, stopping at either the limit, + * the END_OF_RULE character, or an operator. + * @return the index after the terminating character, or + * if limit was reached, limit + */ +int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { + int32_t start = pos; + text.truncate(0); + pos = parseSection(rule, pos, limit, text, UnicodeString(true, ILLEGAL_TOP, -1), false, status); + + if (cursorOffset > 0 && cursor != cursorOffsetPos) { + return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); + } + + return pos; +} + +/** + * Parse a section of one side of a rule, stopping at either + * the limit, the END_OF_RULE character, an operator, or a + * segment close character. This method parses both a + * top-level rule half and a segment within such a rule half. + * It calls itself recursively to parse segments and nested + * segments. + * @param buf buffer into which to accumulate the rule pattern + * characters, either literal characters from the rule or + * standins for UnicodeMatcher objects including segments. + * @param illegal the set of special characters that is illegal during + * this parse. + * @param isSegment if true, then we've already seen a '(' and + * pos on entry points right after it. Accumulate everything + * up to the closing ')', put it in a segment matcher object, + * generate a standin for it, and add the standin to buf. As + * a side effect, update the segments vector with a reference + * to the segment matcher. This works recursively for nested + * segments. If isSegment is false, just accumulate + * characters into buf. + * @return the index after the terminating character, or + * if limit was reached, limit + */ +int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, + UnicodeString& buf, + const UnicodeString& illegal, + UBool isSegment, UErrorCode& status) { + int32_t start = pos; + ParsePosition pp; + UnicodeString scratch; + UBool done = false; + int32_t quoteStart = -1; // Most recent 'single quoted string' + int32_t quoteLimit = -1; + int32_t varStart = -1; // Most recent $variableReference + int32_t varLimit = -1; + int32_t bufStart = buf.length(); + + while (pos < limit && !done) { + // Since all syntax characters are in the BMP, fetching + // 16-bit code units suffices here. + char16_t c = rule.charAt(pos++); + if (PatternProps::isWhiteSpace(c)) { + // Ignore whitespace. Note that this is not Unicode + // spaces, but Java spaces -- a subset, representing + // whitespace likely to be seen in code. + continue; + } + if (u_strchr(HALF_ENDERS, c) != nullptr) { + if (isSegment) { + // Unclosed segment + return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status); + } + break; + } + if (anchorEnd) { + // Text after a presumed end anchor is a syntax err + return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status); + } + if (UnicodeSet::resemblesPattern(rule, pos-1)) { + pp.setIndex(pos-1); // Backup to opening '[' + buf.append(parser.parseSet(rule, pp, status)); + if (U_FAILURE(status)) { + return syntaxError(U_MALFORMED_SET, rule, start, status); + } + pos = pp.getIndex(); + continue; + } + // Handle escapes + if (c == ESCAPE) { + if (pos == limit) { + return syntaxError(U_TRAILING_BACKSLASH, rule, start, status); + } + UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\' + if (escaped == (UChar32) -1) { + return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status); + } + if (!parser.checkVariableRange(escaped)) { + return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); + } + buf.append(escaped); + continue; + } + // Handle quoted matter + if (c == QUOTE) { + int32_t iq = rule.indexOf(QUOTE, pos); + if (iq == pos) { + buf.append(c); // Parse [''] outside quotes as ['] + ++pos; + } else { + /* This loop picks up a run of quoted text of the + * form 'aaaa' each time through. If this run + * hasn't really ended ('aaaa''bbbb') then it keeps + * looping, each time adding on a new run. When it + * reaches the final quote it breaks. + */ + quoteStart = buf.length(); + for (;;) { + if (iq < 0) { + return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status); + } + scratch.truncate(0); + rule.extractBetween(pos, iq, scratch); + buf.append(scratch); + pos = iq+1; + if (pos < limit && rule.charAt(pos) == QUOTE) { + // Parse [''] inside quotes as ['] + iq = rule.indexOf(QUOTE, pos+1); + // Continue looping + } else { + break; + } + } + quoteLimit = buf.length(); + + for (iq=quoteStart; iq= 0) { + syntaxError(U_ILLEGAL_CHARACTER, rule, start, status); + } + + switch (c) { + + //------------------------------------------------------ + // Elements allowed within and out of segments + //------------------------------------------------------ + case ANCHOR_START: + if (buf.length() == 0 && !anchorStart) { + anchorStart = true; + } else { + return syntaxError(U_MISPLACED_ANCHOR_START, + rule, start, status); + } + break; + case SEGMENT_OPEN: + { + // bufSegStart is the offset in buf to the first + // character of the segment we are parsing. + int32_t bufSegStart = buf.length(); + + // Record segment number now, since nextSegmentNumber + // will be incremented during the call to parseSection + // if there are nested segments. + int32_t segmentNumber = nextSegmentNumber++; // 1-based + + // Parse the segment + pos = parseSection(rule, pos, limit, buf, UnicodeString(true, ILLEGAL_SEG, -1), true, status); + + // After parsing a segment, the relevant characters are + // in buf, starting at offset bufSegStart. Extract them + // into a string matcher, and replace them with a + // standin for that matcher. + StringMatcher* m = + new StringMatcher(buf, bufSegStart, buf.length(), + segmentNumber, *parser.curData); + if (m == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + + // Record and associate object and segment number + parser.setSegmentObject(segmentNumber, m, status); + buf.truncate(bufSegStart); + buf.append(parser.getSegmentStandin(segmentNumber, status)); + } + break; + case FUNCTION: + case ALT_FUNCTION: + { + int32_t iref = pos; + TransliteratorIDParser::SingleID* single = + TransliteratorIDParser::parseFilterID(rule, iref); + // The next character MUST be a segment open + if (single == nullptr || + !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) { + return syntaxError(U_INVALID_FUNCTION, rule, start, status); + } + + Transliterator *t = single->createInstance(); + delete single; + if (t == nullptr) { + return syntaxError(U_INVALID_FUNCTION, rule, start, status); + } + + // bufSegStart is the offset in buf to the first + // character of the segment we are parsing. + int32_t bufSegStart = buf.length(); + + // Parse the segment + pos = parseSection(rule, iref, limit, buf, UnicodeString(true, ILLEGAL_FUNC, -1), true, status); + + // After parsing a segment, the relevant characters are + // in buf, starting at offset bufSegStart. + UnicodeString output; + buf.extractBetween(bufSegStart, buf.length(), output); + FunctionReplacer *r = + new FunctionReplacer(t, new StringReplacer(output, parser.curData)); + if (r == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + + // Replace the buffer contents with a stand-in + buf.truncate(bufSegStart); + buf.append(parser.generateStandInFor(r, status)); + } + break; + case SymbolTable::SYMBOL_REF: + // Handle variable references and segment references "$1" .. "$9" + { + // A variable reference must be followed immediately + // by a Unicode identifier start and zero or more + // Unicode identifier part characters, or by a digit + // 1..9 if it is a segment reference. + if (pos == limit) { + // A variable ref character at the end acts as + // an anchor to the context limit, as in perl. + anchorEnd = true; + break; + } + // Parse "$1" "$2" .. "$9" .. (no upper limit) + c = rule.charAt(pos); + int32_t r = u_digit(c, 10); + if (r >= 1 && r <= 9) { + r = ICU_Utility::parseNumber(rule, pos, 10); + if (r < 0) { + return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, + rule, start, status); + } + buf.append(parser.getSegmentStandin(r, status)); + } else { + pp.setIndex(pos); + UnicodeString name = parser.parseData-> + parseReference(rule, pp, limit); + if (name.length() == 0) { + // This means the '$' was not followed by a + // valid name. Try to interpret it as an + // end anchor then. If this also doesn't work + // (if we see a following character) then signal + // an error. + anchorEnd = true; + break; + } + pos = pp.getIndex(); + // If this is a variable definition statement, + // then the LHS variable will be undefined. In + // that case appendVariableDef() will append the + // special placeholder char variableLimit-1. + varStart = buf.length(); + parser.appendVariableDef(name, buf, status); + varLimit = buf.length(); + } + } + break; + case DOT: + buf.append(parser.getDotStandIn(status)); + break; + case KLEENE_STAR: + case ONE_OR_MORE: + case ZERO_OR_ONE: + // Quantifiers. We handle single characters, quoted strings, + // variable references, and segments. + // a+ matches aaa + // 'foo'+ matches foofoofoo + // $v+ matches xyxyxy if $v == xy + // (seg)+ matches segsegseg + { + if (isSegment && buf.length() == bufStart) { + // The */+ immediately follows '(' + return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status); + } + + int32_t qstart, qlimit; + // The */+ follows an isolated character or quote + // or variable reference + if (buf.length() == quoteLimit) { + // The */+ follows a 'quoted string' + qstart = quoteStart; + qlimit = quoteLimit; + } else if (buf.length() == varLimit) { + // The */+ follows a $variableReference + qstart = varStart; + qlimit = varLimit; + } else { + // The */+ follows a single character, possibly + // a segment standin + qstart = buf.length() - 1; + qlimit = qstart + 1; + } + + UnicodeFunctor *m = + new StringMatcher(buf, qstart, qlimit, 0, *parser.curData); + if (m == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + int32_t min = 0; + int32_t max = Quantifier::MAX; + switch (c) { + case ONE_OR_MORE: + min = 1; + break; + case ZERO_OR_ONE: + min = 0; + max = 1; + break; + // case KLEENE_STAR: + // do nothing -- min, max already set + } + m = new Quantifier(m, min, max); + if (m == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + buf.truncate(qstart); + buf.append(parser.generateStandInFor(m, status)); + } + break; + + //------------------------------------------------------ + // Elements allowed ONLY WITHIN segments + //------------------------------------------------------ + case SEGMENT_CLOSE: + // assert(isSegment); + // We're done parsing a segment. + done = true; + break; + + //------------------------------------------------------ + // Elements allowed ONLY OUTSIDE segments + //------------------------------------------------------ + case CONTEXT_ANTE: + if (ante >= 0) { + return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status); + } + ante = buf.length(); + break; + case CONTEXT_POST: + if (post >= 0) { + return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status); + } + post = buf.length(); + break; + case CURSOR_POS: + if (cursor >= 0) { + return syntaxError(U_MULTIPLE_CURSORS, rule, start, status); + } + cursor = buf.length(); + break; + case CURSOR_OFFSET: + if (cursorOffset < 0) { + if (buf.length() > 0) { + return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); + } + --cursorOffset; + } else if (cursorOffset > 0) { + if (buf.length() != cursorOffsetPos || cursor >= 0) { + return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); + } + ++cursorOffset; + } else { + if (cursor == 0 && buf.length() == 0) { + cursorOffset = -1; + } else if (cursor < 0) { + cursorOffsetPos = buf.length(); + cursorOffset = 1; + } else { + return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); + } + } + break; + + + //------------------------------------------------------ + // Non-special characters + //------------------------------------------------------ + default: + // Disallow unquoted characters other than [0-9A-Za-z] + // in the printable ASCII range. These characters are + // reserved for possible future use. + if (c >= 0x0021 && c <= 0x007E && + !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || + (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || + (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) { + return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); + } + buf.append(c); + break; + } + } + + return pos; +} + +/** + * Remove context. + */ +void RuleHalf::removeContext() { + //text = text.substring(ante < 0 ? 0 : ante, + // post < 0 ? text.length() : post); + if (post >= 0) { + text.remove(post); + } + if (ante >= 0) { + text.removeBetween(0, ante); + } + ante = post = -1; + anchorStart = anchorEnd = false; +} + +/** + * Return true if this half looks like valid output, that is, does not + * contain quantifiers or other special input-only elements. + */ +UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) { + for (int32_t i=0; iisReplacer(c)) { + return false; + } + } + return true; +} + +/** + * Return true if this half looks like valid input, that is, does not + * contain functions or other special output-only elements. + */ +UBool RuleHalf::isValidInput(TransliteratorParser& transParser) { + for (int32_t i=0; iisMatcher(c)) { + return false; + } + } + return true; +} + +//---------------------------------------------------------------------- +// PUBLIC API +//---------------------------------------------------------------------- + +/** + * Constructor. + */ +TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) : +dataVector(statusReturn), +idBlockVector(statusReturn), +variablesVector(statusReturn), +segmentObjects(statusReturn) +{ + idBlockVector.setDeleter(uprv_deleteUObject); + curData = nullptr; + compoundFilter = nullptr; + parseData = nullptr; + variableNames.setValueDeleter(uprv_deleteUObject); +} + +/** + * Destructor. + */ +TransliteratorParser::~TransliteratorParser() { + while (!dataVector.isEmpty()) + delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); + delete compoundFilter; + delete parseData; + while (!variablesVector.isEmpty()) + delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); +} + +void +TransliteratorParser::parse(const UnicodeString& rules, + UTransDirection transDirection, + UParseError& pe, + UErrorCode& ec) { + if (U_SUCCESS(ec)) { + parseRules(rules, transDirection, ec); + pe = parseError; + } +} + +/** + * Return the compound filter parsed by parse(). Caller owns result. + */ +UnicodeSet* TransliteratorParser::orphanCompoundFilter() { + UnicodeSet* f = compoundFilter; + compoundFilter = nullptr; + return f; +} + +//---------------------------------------------------------------------- +// Private implementation +//---------------------------------------------------------------------- + +/** + * Parse the given string as a sequence of rules, separated by newline + * characters ('\n'), and cause this object to implement those rules. Any + * previous rules are discarded. Typically this method is called exactly + * once, during construction. + * @exception IllegalArgumentException if there is a syntax error in the + * rules + */ +void TransliteratorParser::parseRules(const UnicodeString& rule, + UTransDirection theDirection, + UErrorCode& status) +{ + // Clear error struct + uprv_memset(&parseError, 0, sizeof(parseError)); + parseError.line = parseError.offset = -1; + + UBool parsingIDs = true; + int32_t ruleCount = 0; + + while (!dataVector.isEmpty()) { + delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); + } + if (U_FAILURE(status)) { + return; + } + + idBlockVector.removeAllElements(); + curData = nullptr; + direction = theDirection; + ruleCount = 0; + + delete compoundFilter; + compoundFilter = nullptr; + + while (!variablesVector.isEmpty()) { + delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); + } + variableNames.removeAll(); + parseData = new ParseData(0, &variablesVector, &variableNames); + if (parseData == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + + dotStandIn = (char16_t) -1; + + UnicodeString *tempstr = nullptr; // used for memory allocation error checking + UnicodeString str; // scratch + UnicodeString idBlockResult; + int32_t pos = 0; + int32_t limit = rule.length(); + + // The compound filter offset is an index into idBlockResult. + // If it is 0, then the compound filter occurred at the start, + // and it is the offset to the _start_ of the compound filter + // pattern. Otherwise it is the offset to the _limit_ of the + // compound filter pattern within idBlockResult. + compoundFilter = nullptr; + int32_t compoundFilterOffset = -1; + + while (pos < limit && U_SUCCESS(status)) { + char16_t c = rule.charAt(pos++); + if (PatternProps::isWhiteSpace(c)) { + // Ignore leading whitespace. + continue; + } + // Skip lines starting with the comment character + if (c == RULE_COMMENT_CHAR) { + pos = rule.indexOf((char16_t)0x000A /*\n*/, pos) + 1; + if (pos == 0) { + break; // No "\n" found; rest of rule is a comment + } + continue; // Either fall out or restart with next line + } + + // skip empty rules + if (c == END_OF_RULE) + continue; + + // keep track of how many rules we've seen + ++ruleCount; + + // We've found the start of a rule or ID. c is its first + // character, and pos points past c. + --pos; + // Look for an ID token. Must have at least ID_TOKEN_LEN + 1 + // chars left. + if ((pos + ID_TOKEN_LEN + 1) <= limit && + rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) { + pos += ID_TOKEN_LEN; + c = rule.charAt(pos); + while (PatternProps::isWhiteSpace(c) && pos < limit) { + ++pos; + c = rule.charAt(pos); + } + + int32_t p = pos; + + if (!parsingIDs) { + if (curData != nullptr) { + U_ASSERT(!dataVector.hasDeleter()); + if (direction == UTRANS_FORWARD) + dataVector.addElement(curData, status); + else + dataVector.insertElementAt(curData, 0, status); + if (U_FAILURE(status)) { + delete curData; + } + curData = nullptr; + } + parsingIDs = true; + } + + TransliteratorIDParser::SingleID* id = + TransliteratorIDParser::parseSingleID(rule, p, direction, status); + if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) { + // Successful ::ID parse. + + if (direction == UTRANS_FORWARD) { + idBlockResult.append(id->canonID).append(END_OF_RULE); + } else { + idBlockResult.insert(0, END_OF_RULE); + idBlockResult.insert(0, id->canonID); + } + + } else { + // Couldn't parse an ID. Try to parse a global filter + int32_t withParens = -1; + UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, nullptr); + if (f != nullptr) { + if (ICU_Utility::parseChar(rule, p, END_OF_RULE) + && (direction == UTRANS_FORWARD) == (withParens == 0)) + { + if (compoundFilter != nullptr) { + // Multiple compound filters + syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status); + delete f; + } else { + compoundFilter = f; + compoundFilterOffset = ruleCount; + } + } else { + delete f; + } + } else { + // Invalid ::id + // Can be parsed as neither an ID nor a global filter + syntaxError(U_INVALID_ID, rule, pos, status); + } + } + delete id; + pos = p; + } else { + if (parsingIDs) { + tempstr = new UnicodeString(idBlockResult); + // nullptr pointer check + if (tempstr == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + U_ASSERT(idBlockVector.hasDeleter()); + if (direction == UTRANS_FORWARD) + idBlockVector.adoptElement(tempstr, status); + else + idBlockVector.insertElementAt(tempstr, 0, status); + if (U_FAILURE(status)) { + return; + } + idBlockResult.remove(); + parsingIDs = false; + curData = new TransliterationRuleData(status); + // nullptr pointer check + if (curData == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + parseData->data = curData; + + // By default, rules use part of the private use area + // E000..F8FF for variables and other stand-ins. Currently + // the range F000..F8FF is typically sufficient. The 'use + // variable range' pragma allows rule sets to modify this. + setVariableRange(0xF000, 0xF8FF, status); + } + + if (resemblesPragma(rule, pos, limit)) { + int32_t ppp = parsePragma(rule, pos, limit, status); + if (ppp < 0) { + syntaxError(U_MALFORMED_PRAGMA, rule, pos, status); + } + pos = ppp; + // Parse a rule + } else { + pos = parseRule(rule, pos, limit, status); + } + } + } + + if (parsingIDs && idBlockResult.length() > 0) { + tempstr = new UnicodeString(idBlockResult); + // nullptr pointer check + if (tempstr == nullptr) { + // TODO: Testing, forcing this path, shows many memory leaks. ICU-21701 + // intltest translit/TransliteratorTest/TestInstantiation + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + if (direction == UTRANS_FORWARD) + idBlockVector.adoptElement(tempstr, status); + else + idBlockVector.insertElementAt(tempstr, 0, status); + if (U_FAILURE(status)) { + return; + } + } + else if (!parsingIDs && curData != nullptr) { + if (direction == UTRANS_FORWARD) { + dataVector.addElement(curData, status); + } else { + dataVector.insertElementAt(curData, 0, status); + } + if (U_FAILURE(status)) { + delete curData; + curData = nullptr; + } + } + + if (U_SUCCESS(status)) { + // Convert the set vector to an array + int32_t i, dataVectorSize = dataVector.size(); + for (i = 0; i < dataVectorSize; i++) { + TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); + data->variablesLength = variablesVector.size(); + if (data->variablesLength == 0) { + data->variables = 0; + } else { + data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*)); + // nullptr pointer check + if (data->variables == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + data->variablesAreOwned = (i == 0); + } + + for (int32_t j = 0; j < data->variablesLength; j++) { + data->variables[j] = + static_cast(variablesVector.elementAt(j)); + } + + data->variableNames.removeAll(); + int32_t p = UHASH_FIRST; + const UHashElement* he = variableNames.nextElement(p); + while (he != nullptr) { + UnicodeString* tempus = ((UnicodeString*)(he->value.pointer))->clone(); + if (tempus == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + data->variableNames.put(*((UnicodeString*)(he->key.pointer)), + tempus, status); + he = variableNames.nextElement(p); + } + } + variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed + + // Index the rules + if (compoundFilter != nullptr) { + if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) || + (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) { + status = U_MISPLACED_COMPOUND_FILTER; + } + } + + for (i = 0; i < dataVectorSize; i++) { + TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); + data->ruleSet.freeze(parseError, status); + } + if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) { + idBlockVector.removeElementAt(0); + } + } +} + +/** + * Set the variable range to [start, end] (inclusive). + */ +void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) { + if (start > end || start < 0 || end > 0xFFFF) { + status = U_MALFORMED_PRAGMA; + return; + } + + curData->variablesBase = (char16_t) start; + if (dataVector.size() == 0) { + variableNext = (char16_t) start; + variableLimit = (char16_t) (end + 1); + } +} + +/** + * Assert that the given character is NOT within the variable range. + * If it is, return false. This is necessary to ensure that the + * variable range does not overlap characters used in a rule. + */ +UBool TransliteratorParser::checkVariableRange(UChar32 ch) const { + return !(ch >= curData->variablesBase && ch < variableLimit); +} + +/** + * Set the maximum backup to 'backup', in response to a pragma + * statement. + */ +void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) { + //TODO Finish +} + +/** + * Begin normalizing all rules using the given mode, in response + * to a pragma statement. + */ +void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) { + //TODO Finish +} + +static const char16_t PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use " + +static const char16_t PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;" + +static const char16_t PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;" + +static const char16_t PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;" + +static const char16_t PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;" + +/** + * Return true if the given rule looks like a pragma. + * @param pos offset to the first non-whitespace character + * of the rule. + * @param limit pointer past the last character of the rule. + */ +UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) { + // Must start with /use\s/i + return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(true, PRAGMA_USE, 4), nullptr) >= 0; +} + +/** + * Parse a pragma. This method assumes resemblesPragma() has + * already returned true. + * @param pos offset to the first non-whitespace character + * of the rule. + * @param limit pointer past the last character of the rule. + * @return the position index after the final ';' of the pragma, + * or -1 on failure. + */ +int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { + int32_t array[2]; + + // resemblesPragma() has already returned true, so we + // know that pos points to /use\s/i; we can skip 4 characters + // immediately + pos += 4; + + // Here are the pragmas we recognize: + // use variable range 0xE000 0xEFFF; + // use maximum backup 16; + // use nfd rules; + // use nfc rules; + int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(true, PRAGMA_VARIABLE_RANGE, -1), array); + if (p >= 0) { + setVariableRange(array[0], array[1], status); + return p; + } + + p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(true, PRAGMA_MAXIMUM_BACKUP, -1), array); + if (p >= 0) { + pragmaMaximumBackup(array[0]); + return p; + } + + p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(true, PRAGMA_NFD_RULES, -1), nullptr); + if (p >= 0) { + pragmaNormalizeRules(UNORM_NFD); + return p; + } + + p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(true, PRAGMA_NFC_RULES, -1), nullptr); + if (p >= 0) { + pragmaNormalizeRules(UNORM_NFC); + return p; + } + + // Syntax error: unable to parse pragma + return -1; +} + +/** + * MAIN PARSER. Parse the next rule in the given rule string, starting + * at pos. Return the index after the last character parsed. Do not + * parse characters at or after limit. + * + * Important: The character at pos must be a non-whitespace character + * that is not the comment character. + * + * This method handles quoting, escaping, and whitespace removal. It + * parses the end-of-rule character. It recognizes context and cursor + * indicators. Once it does a lexical breakdown of the rule at pos, it + * creates a rule object and adds it to our rule list. + */ +int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { + // Locate the left side, operator, and right side + int32_t start = pos; + char16_t op = 0; + int32_t i; + + // Set up segments data + segmentStandins.truncate(0); + segmentObjects.removeAllElements(); + + // Use pointers to automatics to make swapping possible. + RuleHalf _left(*this), _right(*this); + RuleHalf* left = &_left; + RuleHalf* right = &_right; + + undefinedVariableName.remove(); + pos = left->parse(rule, pos, limit, status); + if (U_FAILURE(status)) { + return start; + } + + if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == nullptr) { + return syntaxError(U_MISSING_OPERATOR, rule, start, status); + } + ++pos; + + // Found an operator char. Check for forward-reverse operator. + if (op == REVERSE_RULE_OP && + (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) { + ++pos; + op = FWDREV_RULE_OP; + } + + // Translate alternate op characters. + switch (op) { + case ALT_FORWARD_RULE_OP: + op = FORWARD_RULE_OP; + break; + case ALT_REVERSE_RULE_OP: + op = REVERSE_RULE_OP; + break; + case ALT_FWDREV_RULE_OP: + op = FWDREV_RULE_OP; + break; + } + + pos = right->parse(rule, pos, limit, status); + if (U_FAILURE(status)) { + return start; + } + + if (pos < limit) { + if (rule.charAt(--pos) == END_OF_RULE) { + ++pos; + } else { + // RuleHalf parser must have terminated at an operator + return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); + } + } + + if (op == VARIABLE_DEF_OP) { + // LHS is the name. RHS is a single character, either a literal + // or a set (already parsed). If RHS is longer than one + // character, it is either a multi-character string, or multiple + // sets, or a mixture of chars and sets -- syntax error. + + // We expect to see a single undefined variable (the one being + // defined). + if (undefinedVariableName.length() == 0) { + // "Missing '$' or duplicate definition" + return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status); + } + if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) { + // "Malformed LHS" + return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); + } + if (left->anchorStart || left->anchorEnd || + right->anchorStart || right->anchorEnd) { + return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); + } + // We allow anything on the right, including an empty string. + UnicodeString* value = new UnicodeString(right->text); + // nullptr pointer check + if (value == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + variableNames.put(undefinedVariableName, value, status); + ++variableLimit; + return pos; + } + + // If this is not a variable definition rule, we shouldn't have + // any undefined variable names. + if (undefinedVariableName.length() != 0) { + return syntaxError(// "Undefined variable $" + undefinedVariableName, + U_UNDEFINED_VARIABLE, + rule, start, status); + } + + // Verify segments + if (segmentStandins.length() > segmentObjects.size()) { + syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status); + } + for (i=0; iremoveContext(); + left->cursor = -1; + left->cursorOffset = 0; + } + + // Normalize context + if (left->ante < 0) { + left->ante = 0; + } + if (left->post < 0) { + left->post = left->text.length(); + } + + // Context is only allowed on the input side. Cursors are only + // allowed on the output side. Segment delimiters can only appear + // on the left, and references on the right. Cursor offset + // cannot appear without an explicit cursor. Cursor offset + // cannot place the cursor outside the limits of the context. + // Anchors are only allowed on the input side. + if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 || + (right->cursorOffset != 0 && right->cursor < 0) || + // - The following two checks were used to ensure that the + // - the cursor offset stayed within the ante- or postcontext. + // - However, with the addition of quantifiers, we have to + // - allow arbitrary cursor offsets and do runtime checking. + //(right->cursorOffset > (left->text.length() - left->post)) || + //(-right->cursorOffset > left->ante) || + right->anchorStart || right->anchorEnd || + !left->isValidInput(*this) || !right->isValidOutput(*this) || + left->ante > left->post) { + + return syntaxError(U_MALFORMED_RULE, rule, start, status); + } + + // Flatten segment objects vector to an array + UnicodeFunctor** segmentsArray = nullptr; + if (segmentObjects.size() > 0) { + segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *)); + // Null pointer check + if (segmentsArray == nullptr) { + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + segmentObjects.toArray((void**) segmentsArray); + } + TransliterationRule* temptr = new TransliterationRule( + left->text, left->ante, left->post, + right->text, right->cursor, right->cursorOffset, + segmentsArray, + segmentObjects.size(), + left->anchorStart, left->anchorEnd, + curData, + status); + //Null pointer check + if (temptr == nullptr) { + uprv_free(segmentsArray); + return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); + } + + curData->ruleSet.addRule(temptr, status); + + return pos; +} + +/** + * Called by main parser upon syntax error. Search the rule string + * for the probable end of the rule. Of course, if the error is that + * the end of rule marker is missing, then the rule end will not be found. + * In any case the rule start will be correctly reported. + * @param msg error description + * @param rule pattern string + * @param start position of first character of current rule + */ +int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode, + const UnicodeString& rule, + int32_t pos, + UErrorCode& status) +{ + parseError.offset = pos; + parseError.line = 0 ; /* we are not using line numbers */ + + // for pre-context + const int32_t LEN = U_PARSE_CONTEXT_LEN - 1; + int32_t start = uprv_max(pos - LEN, 0); + int32_t stop = pos; + + rule.extract(start,stop-start,parseError.preContext); + //null terminate the buffer + parseError.preContext[stop-start] = 0; + + //for post-context + start = pos; + stop = uprv_min(pos + LEN, rule.length()); + + rule.extract(start,stop-start,parseError.postContext); + //null terminate the buffer + parseError.postContext[stop-start]= 0; + + status = (UErrorCode)parseErrorCode; + return pos; + +} + +/** + * Parse a UnicodeSet out, store it, and return the stand-in character + * used to represent it. + */ +char16_t TransliteratorParser::parseSet(const UnicodeString& rule, + ParsePosition& pos, + UErrorCode& status) { + UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status); + // Null pointer check + if (set == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return (char16_t)0x0000; // Return empty character with error. + } + set->compact(); + return generateStandInFor(set, status); +} + +/** + * Generate and return a stand-in for a new UnicodeFunctor. Store + * the matcher (adopt it). + */ +char16_t TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) { + // assert(obj != null); + + // Look up previous stand-in, if any. This is a short list + // (typical n is 0, 1, or 2); linear search is optimal. + for (int32_t i=0; ivariablesBase + i); + } + } + + if (variableNext >= variableLimit) { + delete adopted; + status = U_VARIABLE_RANGE_EXHAUSTED; + return 0; + } + variablesVector.addElement(adopted, status); + if (U_FAILURE(status)) { + delete adopted; + return 0; + } + return variableNext++; +} + +/** + * Return the standin for segment seg (1-based). + */ +char16_t TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) { + // Special character used to indicate an empty spot + char16_t empty = curData->variablesBase - 1; + while (segmentStandins.length() < seg) { + segmentStandins.append(empty); + } + char16_t c = segmentStandins.charAt(seg-1); + if (c == empty) { + if (variableNext >= variableLimit) { + status = U_VARIABLE_RANGE_EXHAUSTED; + return 0; + } + c = variableNext++; + // Set a placeholder in the primary variables vector that will be + // filled in later by setSegmentObject(). We know that we will get + // called first because setSegmentObject() will call us. + variablesVector.addElement((void*) nullptr, status); + segmentStandins.setCharAt(seg-1, c); + } + return c; +} + +/** + * Set the object for segment seg (1-based). + */ +void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) { + // Since we call parseSection() recursively, nested + // segments will result in segment i+1 getting parsed + // and stored before segment i; be careful with the + // vector handling here. + if (segmentObjects.size() < seg) { + segmentObjects.setSize(seg, status); + } + if (U_FAILURE(status)) { + return; + } + int32_t index = getSegmentStandin(seg, status) - curData->variablesBase; + if (segmentObjects.elementAt(seg-1) != nullptr || + variablesVector.elementAt(index) != nullptr) { + // should never happen + if (U_SUCCESS(status)) {status = U_INTERNAL_TRANSLITERATOR_ERROR;} + return; + } + // Note: neither segmentObjects or variablesVector has an object deleter function. + segmentObjects.setElementAt(adopted, seg-1); + variablesVector.setElementAt(adopted, index); +} + +/** + * Return the stand-in for the dot set. It is allocated the first + * time and reused thereafter. + */ +char16_t TransliteratorParser::getDotStandIn(UErrorCode& status) { + if (dotStandIn == (char16_t) -1) { + UnicodeSet* tempus = new UnicodeSet(UnicodeString(true, DOT_SET, -1), status); + // Null pointer check. + if (tempus == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return (char16_t)0x0000; + } + dotStandIn = generateStandInFor(tempus, status); + } + return dotStandIn; +} + +/** + * Append the value of the given variable name to the given + * UnicodeString. + */ +void TransliteratorParser::appendVariableDef(const UnicodeString& name, + UnicodeString& buf, + UErrorCode& status) { + const UnicodeString* s = (const UnicodeString*) variableNames.get(name); + if (s == nullptr) { + // We allow one undefined variable so that variable definition + // statements work. For the first undefined variable we return + // the special placeholder variableLimit-1, and save the variable + // name. + if (undefinedVariableName.length() == 0) { + undefinedVariableName = name; + if (variableNext >= variableLimit) { + // throw new RuntimeException("Private use variables exhausted"); + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + buf.append((char16_t) --variableLimit); + } else { + //throw new IllegalArgumentException("Undefined variable $" + // + name); + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + } else { + buf.append(*s); + } +} + +/** + * Glue method to get around access restrictions in C++. + */ +/*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) { + return Transliterator::createBasicInstance(id, canonID); +}*/ + +U_NAMESPACE_END + +U_CAPI int32_t +utrans_stripRules(const char16_t *source, int32_t sourceLen, char16_t *target, UErrorCode *status) { + U_NAMESPACE_USE + + //const char16_t *sourceStart = source; + const char16_t *targetStart = target; + const char16_t *sourceLimit = source+sourceLen; + char16_t *targetLimit = target+sourceLen; + UChar32 c = 0; + UBool quoted = false; + int32_t index; + + uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR); + + /* read the rules into the buffer */ + while (source < sourceLimit) + { + index=0; + U16_NEXT_UNSAFE(source, index, c); + source+=index; + if(c == QUOTE) { + quoted = (UBool)!quoted; + } + else if (!quoted) { + if (c == RULE_COMMENT_CHAR) { + /* skip comments and all preceding spaces */ + while (targetStart < target && *(target - 1) == 0x0020) { + target--; + } + do { + if (source == sourceLimit) { + c = U_SENTINEL; + break; + } + c = *(source++); + } + while (c != CR && c != LF); + if (c < 0) { + break; + } + } + else if (c == ESCAPE && source < sourceLimit) { + UChar32 c2 = *source; + if (c2 == CR || c2 == LF) { + /* A backslash at the end of a line. */ + /* Since we're stripping lines, ignore the backslash. */ + source++; + continue; + } + if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */ + int32_t escapeOffset = 0; + UnicodeString escapedStr(source, 5); + c2 = escapedStr.unescapeAt(escapeOffset); + + if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0) + { + *status = U_PARSE_ERROR; + return 0; + } + if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) { + /* It was escaped for a reason. Write what it was suppose to be. */ + source+=5; + c = c2; + } + } + else if (c2 == QUOTE) { + /* \' seen. Make sure we don't do anything when we see it again. */ + quoted = (UBool)!quoted; + } + } + } + if (c == CR || c == LF) + { + /* ignore spaces carriage returns, and all leading spaces on the next line. + * and line feed unless in the form \uXXXX + */ + quoted = false; + while (source < sourceLimit) { + c = *(source); + if (c != CR && c != LF && c != 0x0020) { + break; + } + source++; + } + continue; + } + + /* Append char16_t * after dissembling if c > 0xffff*/ + index=0; + U16_APPEND_UNSAFE(target, index, c); + target+=index; + } + if (target < targetLimit) { + *target = 0; + } + return (int32_t)(target-targetStart); +} + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/rbt_pars.h b/intl/icu/source/i18n/rbt_pars.h new file mode 100644 index 0000000000..11cf50e756 --- /dev/null +++ b/intl/icu/source/i18n/rbt_pars.h @@ -0,0 +1,357 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2011, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef RBT_PARS_H +#define RBT_PARS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION +#ifdef __cplusplus + +#include "unicode/uobject.h" +#include "unicode/parseerr.h" +#include "unicode/unorm.h" +#include "rbt.h" +#include "hash.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +class TransliterationRuleData; +class UnicodeFunctor; +class ParseData; +class RuleHalf; +class ParsePosition; +class StringMatcher; + +class TransliteratorParser : public UMemory { + + public: + + /** + * A Vector of TransliterationRuleData objects, one for each discrete group + * of rules in the rule set + */ + UVector dataVector; + + /** + * PUBLIC data member. + * A Vector of UnicodeStrings containing all of the ID blocks in the rule set + */ + UVector idBlockVector; + + /** + * PUBLIC data member containing the parsed compound filter, if any. + */ + UnicodeSet* compoundFilter; + + private: + + /** + * The current data object for which we are parsing rules + */ + TransliterationRuleData* curData; + + UTransDirection direction; + + /** + * Parse error information. + */ + UParseError parseError; + + /** + * Temporary symbol table used during parsing. + */ + ParseData* parseData; + + /** + * Temporary vector of matcher variables. When parsing is complete, this + * is copied into the array data.variables. As with data.variables, + * element 0 corresponds to character data.variablesBase. + */ + UVector variablesVector; + + /** + * Temporary table of variable names. When parsing is complete, this is + * copied into data.variableNames. + */ + Hashtable variableNames; + + /** + * String of standins for segments. Used during the parsing of a single + * rule. segmentStandins.charAt(0) is the standin for "$1" and corresponds + * to StringMatcher object segmentObjects.elementAt(0), etc. + */ + UnicodeString segmentStandins; + + /** + * Vector of StringMatcher objects for segments. Used during the + * parsing of a single rule. + * segmentStandins.charAt(0) is the standin for "$1" and corresponds + * to StringMatcher object segmentObjects.elementAt(0), etc. + */ + UVector segmentObjects; + + /** + * The next available stand-in for variables. This starts at some point in + * the private use area (discovered dynamically) and increments up toward + * variableLimit. At any point during parsing, available + * variables are variableNext..variableLimit-1. + */ + char16_t variableNext; + + /** + * The last available stand-in for variables. This is discovered + * dynamically. At any point during parsing, available variables are + * variableNext..variableLimit-1. + */ + char16_t variableLimit; + + /** + * When we encounter an undefined variable, we do not immediately signal + * an error, in case we are defining this variable, e.g., "$a = [a-z];". + * Instead, we save the name of the undefined variable, and substitute + * in the placeholder char variableLimit - 1, and decrement + * variableLimit. + */ + UnicodeString undefinedVariableName; + + /** + * The stand-in character for the 'dot' set, represented by '.' in + * patterns. This is allocated the first time it is needed, and + * reused thereafter. + */ + char16_t dotStandIn; + +public: + + /** + * Constructor. + */ + TransliteratorParser(UErrorCode &statusReturn); + + /** + * Destructor. + */ + ~TransliteratorParser(); + + /** + * Parse the given string as a sequence of rules, separated by newline + * characters ('\n'), and cause this object to implement those rules. Any + * previous rules are discarded. Typically this method is called exactly + * once after construction. + * + * Parse the given rules, in the given direction. After this call + * returns, query the public data members for results. The caller + * owns the 'data' and 'compoundFilter' data members after this + * call returns. + * @param rules rules, separated by ';' + * @param direction either FORWARD or REVERSE. + * @param pe Struct to receive information on position + * of error if an error is encountered + * @param ec Output param set to success/failure code. + */ + void parse(const UnicodeString& rules, + UTransDirection direction, + UParseError& pe, + UErrorCode& ec); + + /** + * Return the compound filter parsed by parse(). Caller owns result. + * @return the compound filter parsed by parse(). + */ + UnicodeSet* orphanCompoundFilter(); + +private: + + /** + * Return a representation of this transliterator as source rules. + * @param rules Output param to receive the rules. + * @param direction either FORWARD or REVERSE. + */ + void parseRules(const UnicodeString& rules, + UTransDirection direction, + UErrorCode& status); + + /** + * MAIN PARSER. Parse the next rule in the given rule string, starting + * at pos. Return the index after the last character parsed. Do not + * parse characters at or after limit. + * + * Important: The character at pos must be a non-whitespace character + * that is not the comment character. + * + * This method handles quoting, escaping, and whitespace removal. It + * parses the end-of-rule character. It recognizes context and cursor + * indicators. Once it does a lexical breakdown of the rule at pos, it + * creates a rule object and adds it to our rule list. + * @param rules Output param to receive the rules. + * @param pos the starting position. + * @param limit pointer past the last character of the rule. + * @return the index after the last character parsed. + */ + int32_t parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); + + /** + * Set the variable range to [start, end] (inclusive). + * @param start the start value of the range. + * @param end the end value of the range. + */ + void setVariableRange(int32_t start, int32_t end, UErrorCode& status); + + /** + * Assert that the given character is NOT within the variable range. + * If it is, return false. This is necessary to ensure that the + * variable range does not overlap characters used in a rule. + * @param ch the given character. + * @return True, if the given character is NOT within the variable range. + */ + UBool checkVariableRange(UChar32 ch) const; + + /** + * Set the maximum backup to 'backup', in response to a pragma + * statement. + * @param backup the new value to be set. + */ + void pragmaMaximumBackup(int32_t backup); + + /** + * Begin normalizing all rules using the given mode, in response + * to a pragma statement. + * @param mode the given mode. + */ + void pragmaNormalizeRules(UNormalizationMode mode); + + /** + * Return true if the given rule looks like a pragma. + * @param pos offset to the first non-whitespace character + * of the rule. + * @param limit pointer past the last character of the rule. + * @return true if the given rule looks like a pragma. + */ + static UBool resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit); + + /** + * Parse a pragma. This method assumes resemblesPragma() has + * already returned true. + * @param pos offset to the first non-whitespace character + * of the rule. + * @param limit pointer past the last character of the rule. + * @return the position index after the final ';' of the pragma, + * or -1 on failure. + */ + int32_t parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); + + /** + * Called by main parser upon syntax error. Search the rule string + * for the probable end of the rule. Of course, if the error is that + * the end of rule marker is missing, then the rule end will not be found. + * In any case the rule start will be correctly reported. + * @param parseErrorCode error code. + * @param msg error description. + * @param start position of first character of current rule. + * @return start position of first character of current rule. + */ + int32_t syntaxError(UErrorCode parseErrorCode, const UnicodeString&, int32_t start, + UErrorCode& status); + + /** + * Parse a UnicodeSet out, store it, and return the stand-in character + * used to represent it. + * + * @param rule the rule for UnicodeSet. + * @param pos the position in pattern at which to start parsing. + * @return the stand-in character used to represent it. + */ + char16_t parseSet(const UnicodeString& rule, + ParsePosition& pos, + UErrorCode& status); + + /** + * Generate and return a stand-in for a new UnicodeFunctor. Store + * the matcher (adopt it). + * @param adopted the UnicodeFunctor to be adopted. + * @return a stand-in for a new UnicodeFunctor. + */ + char16_t generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status); + + /** + * Return the standin for segment seg (1-based). + * @param seg the given segment. + * @return the standIn character for the given segment. + */ + char16_t getSegmentStandin(int32_t seg, UErrorCode& status); + + /** + * Set the object for segment seg (1-based). + * @param seg the given segment. + * @param adopted the StringMatcher to be adopted. + */ + void setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status); + + /** + * Return the stand-in for the dot set. It is allocated the first + * time and reused thereafter. + * @return the stand-in for the dot set. + */ + char16_t getDotStandIn(UErrorCode& status); + + /** + * Append the value of the given variable name to the given + * UnicodeString. + * @param name the variable name to be appended. + * @param buf the given UnicodeString to append to. + */ + void appendVariableDef(const UnicodeString& name, + UnicodeString& buf, + UErrorCode& status); + + /** + * Glue method to get around access restrictions in C++. + */ + /*static Transliterator* createBasicInstance(const UnicodeString& id, + const UnicodeString* canonID);*/ + + friend class RuleHalf; + + // Disallowed methods; no impl. + /** + * Copy constructor + */ + TransliteratorParser(const TransliteratorParser&); + + /** + * Assignment operator + */ + TransliteratorParser& operator=(const TransliteratorParser&); +}; + +U_NAMESPACE_END + +#endif /* #ifdef __cplusplus */ + +/** + * Strip/convert the following from the transliterator rules: + * comments + * newlines + * white space at the beginning and end of a line + * unescape \u notation + * + * The target must be equal in size as the source. + * @internal + */ +U_CAPI int32_t +utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status); + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/rbt_rule.cpp b/intl/icu/source/i18n/rbt_rule.cpp new file mode 100644 index 0000000000..da8e4eaa1f --- /dev/null +++ b/intl/icu/source/i18n/rbt_rule.cpp @@ -0,0 +1,559 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 1999-2011, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Date Name Description + * 11/17/99 aliu Creation. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/rep.h" +#include "unicode/unifilt.h" +#include "unicode/uniset.h" +#include "unicode/utf16.h" +#include "rbt_rule.h" +#include "rbt_data.h" +#include "cmemory.h" +#include "strmatch.h" +#include "strrepl.h" +#include "util.h" +#include "putilimp.h" + +static const char16_t FORWARD_OP[] = {32,62,32,0}; // " > " + +U_NAMESPACE_BEGIN + +/** + * Construct a new rule with the given input, output text, and other + * attributes. A cursor position may be specified for the output text. + * @param input input string, including key and optional ante and + * post context + * @param anteContextPos offset into input to end of ante context, or -1 if + * none. Must be <= input.length() if not -1. + * @param postContextPos offset into input to start of post context, or -1 + * if none. Must be <= input.length() if not -1, and must be >= + * anteContextPos. + * @param output output string + * @param cursorPosition offset into output at which cursor is located, or -1 if + * none. If less than zero, then the cursor is placed after the + * output; that is, -1 is equivalent to + * output.length(). If greater than + * output.length() then an exception is thrown. + * @param segs array of UnicodeFunctors corresponding to input pattern + * segments, or null if there are none. The array itself is adopted, + * but the pointers within it are not. + * @param segsCount number of elements in segs[] + * @param anchorStart true if the the rule is anchored on the left to + * the context start + * @param anchorEnd true if the rule is anchored on the right to the + * context limit + */ +TransliterationRule::TransliterationRule(const UnicodeString& input, + int32_t anteContextPos, int32_t postContextPos, + const UnicodeString& outputStr, + int32_t cursorPosition, int32_t cursorOffset, + UnicodeFunctor** segs, + int32_t segsCount, + UBool anchorStart, UBool anchorEnd, + const TransliterationRuleData* theData, + UErrorCode& status) : + UMemory(), + segments(0), + data(theData) { + + if (U_FAILURE(status)) { + return; + } + // Do range checks only when warranted to save time + if (anteContextPos < 0) { + anteContextLength = 0; + } else { + if (anteContextPos > input.length()) { + // throw new IllegalArgumentException("Invalid ante context"); + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + anteContextLength = anteContextPos; + } + if (postContextPos < 0) { + keyLength = input.length() - anteContextLength; + } else { + if (postContextPos < anteContextLength || + postContextPos > input.length()) { + // throw new IllegalArgumentException("Invalid post context"); + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + keyLength = postContextPos - anteContextLength; + } + if (cursorPosition < 0) { + cursorPosition = outputStr.length(); + } else if (cursorPosition > outputStr.length()) { + // throw new IllegalArgumentException("Invalid cursor position"); + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + // We don't validate the segments array. The caller must + // guarantee that the segments are well-formed (that is, that + // all $n references in the output refer to indices of this + // array, and that no array elements are null). + this->segments = segs; + this->segmentsCount = segsCount; + + pattern = input; + flags = 0; + if (anchorStart) { + flags |= ANCHOR_START; + } + if (anchorEnd) { + flags |= ANCHOR_END; + } + + anteContext = nullptr; + if (anteContextLength > 0) { + anteContext = new StringMatcher(pattern, 0, anteContextLength, + false, *data); + /* test for nullptr */ + if (anteContext == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + key = nullptr; + if (keyLength > 0) { + key = new StringMatcher(pattern, anteContextLength, anteContextLength + keyLength, + false, *data); + /* test for nullptr */ + if (key == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + int32_t postContextLength = pattern.length() - keyLength - anteContextLength; + postContext = nullptr; + if (postContextLength > 0) { + postContext = new StringMatcher(pattern, anteContextLength + keyLength, pattern.length(), + false, *data); + /* test for nullptr */ + if (postContext == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + this->output = new StringReplacer(outputStr, cursorPosition + cursorOffset, data); + /* test for nullptr */ + if (this->output == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } +} + +/** + * Copy constructor. + */ +TransliterationRule::TransliterationRule(TransliterationRule& other) : + UMemory(other), + anteContext(nullptr), + key(nullptr), + postContext(nullptr), + pattern(other.pattern), + anteContextLength(other.anteContextLength), + keyLength(other.keyLength), + flags(other.flags), + data(other.data) { + + segments = nullptr; + segmentsCount = 0; + if (other.segmentsCount > 0) { + segments = (UnicodeFunctor **)uprv_malloc(other.segmentsCount * sizeof(UnicodeFunctor *)); + uprv_memcpy(segments, other.segments, (size_t)other.segmentsCount*sizeof(segments[0])); + } + + if (other.anteContext != nullptr) { + anteContext = other.anteContext->clone(); + } + if (other.key != nullptr) { + key = other.key->clone(); + } + if (other.postContext != nullptr) { + postContext = other.postContext->clone(); + } + output = other.output->clone(); +} + +TransliterationRule::~TransliterationRule() { + uprv_free(segments); + delete anteContext; + delete key; + delete postContext; + delete output; +} + +/** + * Return the preceding context length. This method is needed to + * support the Transliterator method + * getMaximumContextLength(). Internally, this is + * implemented as the anteContextLength, optionally plus one if + * there is a start anchor. The one character anchor gap is + * needed to make repeated incremental transliteration with + * anchors work. + */ +int32_t TransliterationRule::getContextLength() const { + return anteContextLength + ((flags & ANCHOR_START) ? 1 : 0); +} + +/** + * Internal method. Returns 8-bit index value for this rule. + * This is the low byte of the first character of the key, + * unless the first character of the key is a set. If it's a + * set, or otherwise can match multiple keys, the index value is -1. + */ +int16_t TransliterationRule::getIndexValue() const { + if (anteContextLength == pattern.length()) { + // A pattern with just ante context {such as foo)>bar} can + // match any key. + return -1; + } + UChar32 c = pattern.char32At(anteContextLength); + return (int16_t)(data->lookupMatcher(c) == nullptr ? (c & 0xFF) : -1); +} + +/** + * Internal method. Returns true if this rule matches the given + * index value. The index value is an 8-bit integer, 0..255, + * representing the low byte of the first character of the key. + * It matches this rule if it matches the first character of the + * key, or if the first character of the key is a set, and the set + * contains any character with a low byte equal to the index + * value. If the rule contains only ante context, as in foo)>bar, + * then it will match any key. + */ +UBool TransliterationRule::matchesIndexValue(uint8_t v) const { + // Delegate to the key, or if there is none, to the postContext. + // If there is neither then we match any key; return true. + UnicodeMatcher *m = (key != nullptr) ? key : postContext; + return (m != nullptr) ? m->matchesIndexValue(v) : true; +} + +/** + * Return true if this rule masks another rule. If r1 masks r2 then + * r1 matches any input string that r2 matches. If r1 masks r2 and r2 masks + * r1 then r1 == r2. Examples: "a>x" masks "ab>y". "a>x" masks "a[b]>y". + * "[c]a>x" masks "[dc]a>y". + */ +UBool TransliterationRule::masks(const TransliterationRule& r2) const { + /* Rule r1 masks rule r2 if the string formed of the + * antecontext, key, and postcontext overlaps in the following + * way: + * + * r1: aakkkpppp + * r2: aaakkkkkpppp + * ^ + * + * The strings must be aligned at the first character of the + * key. The length of r1 to the left of the alignment point + * must be <= the length of r2 to the left; ditto for the + * right. The characters of r1 must equal (or be a superset + * of) the corresponding characters of r2. The superset + * operation should be performed to check for UnicodeSet + * masking. + * + * Anchors: Two patterns that differ only in anchors only + * mask one another if they are exactly equal, and r2 has + * all the anchors r1 has (optionally, plus some). Here Y + * means the row masks the column, N means it doesn't. + * + * ab ^ab ab$ ^ab$ + * ab Y Y Y Y + * ^ab N Y N Y + * ab$ N N Y Y + * ^ab$ N N N Y + * + * Post context: {a}b masks ab, but not vice versa, since {a}b + * matches everything ab matches, and {a}b matches {|a|}b but ab + * does not. Pre context is different (a{b} does not align with + * ab). + */ + + /* LIMITATION of the current mask algorithm: Some rule + * maskings are currently not detected. For example, + * "{Lu}]a>x" masks "A]a>y". This can be added later. TODO + */ + + int32_t len = pattern.length(); + int32_t left = anteContextLength; + int32_t left2 = r2.anteContextLength; + int32_t right = len - left; + int32_t right2 = r2.pattern.length() - left2; + int32_t cachedCompare = r2.pattern.compare(left2 - left, len, pattern); + + // TODO Clean this up -- some logic might be combinable with the + // next statement. + + // Test for anchor masking + if (left == left2 && right == right2 && + keyLength <= r2.keyLength && + 0 == cachedCompare) { + // The following boolean logic implements the table above + return (flags == r2.flags) || + (!(flags & ANCHOR_START) && !(flags & ANCHOR_END)) || + ((r2.flags & ANCHOR_START) && (r2.flags & ANCHOR_END)); + } + + return left <= left2 && + (right < right2 || + (right == right2 && keyLength <= r2.keyLength)) && + (0 == cachedCompare); +} + +static inline int32_t posBefore(const Replaceable& str, int32_t pos) { + return (pos > 0) ? + pos - U16_LENGTH(str.char32At(pos-1)) : + pos - 1; +} + +static inline int32_t posAfter(const Replaceable& str, int32_t pos) { + return (pos >= 0 && pos < str.length()) ? + pos + U16_LENGTH(str.char32At(pos)) : + pos + 1; +} + +/** + * Attempt a match and replacement at the given position. Return + * the degree of match between this rule and the given text. The + * degree of match may be mismatch, a partial match, or a full + * match. A mismatch means at least one character of the text + * does not match the context or key. A partial match means some + * context and key characters match, but the text is not long + * enough to match all of them. A full match means all context + * and key characters match. + * + * If a full match is obtained, perform a replacement, update pos, + * and return U_MATCH. Otherwise both text and pos are unchanged. + * + * @param text the text + * @param pos the position indices + * @param incremental if true, test for partial matches that may + * be completed by additional text inserted at pos.limit. + * @return one of U_MISMATCH, + * U_PARTIAL_MATCH, or U_MATCH. If + * incremental is false then U_PARTIAL_MATCH will not be returned. + */ +UMatchDegree TransliterationRule::matchAndReplace(Replaceable& text, + UTransPosition& pos, + UBool incremental) const { + // Matching and replacing are done in one method because the + // replacement operation needs information obtained during the + // match. Another way to do this is to have the match method + // create a match result struct with relevant offsets, and to pass + // this into the replace method. + + // ============================ MATCH =========================== + + // Reset segment match data + if (segments != nullptr) { + for (int32_t i=0; iresetMatch(); + } + } + +// int32_t lenDelta, keyLimit; + int32_t keyLimit; + + // ------------------------ Ante Context ------------------------ + + // A mismatch in the ante context, or with the start anchor, + // is an outright U_MISMATCH regardless of whether we are + // incremental or not. + int32_t oText; // offset into 'text' +// int32_t newStart = 0; + int32_t minOText; + + // Note (1): We process text in 16-bit code units, rather than + // 32-bit code points. This works because stand-ins are + // always in the BMP and because we are doing a literal match + // operation, which can be done 16-bits at a time. + + int32_t anteLimit = posBefore(text, pos.contextStart); + + UMatchDegree match; + + // Start reverse match at char before pos.start + oText = posBefore(text, pos.start); + + if (anteContext != nullptr) { + match = anteContext->matches(text, oText, anteLimit, false); + if (match != U_MATCH) { + return U_MISMATCH; + } + } + + minOText = posAfter(text, oText); + + // ------------------------ Start Anchor ------------------------ + + if (((flags & ANCHOR_START) != 0) && oText != anteLimit) { + return U_MISMATCH; + } + + // -------------------- Key and Post Context -------------------- + + oText = pos.start; + + if (key != nullptr) { + match = key->matches(text, oText, pos.limit, incremental); + if (match != U_MATCH) { + return match; + } + } + + keyLimit = oText; + + if (postContext != nullptr) { + if (incremental && keyLimit == pos.limit) { + // The key matches just before pos.limit, and there is + // a postContext. Since we are in incremental mode, + // we must assume more characters may be inserted at + // pos.limit -- this is a partial match. + return U_PARTIAL_MATCH; + } + + match = postContext->matches(text, oText, pos.contextLimit, incremental); + if (match != U_MATCH) { + return match; + } + } + + // ------------------------- Stop Anchor ------------------------ + + if (((flags & ANCHOR_END)) != 0) { + if (oText != pos.contextLimit) { + return U_MISMATCH; + } + if (incremental) { + return U_PARTIAL_MATCH; + } + } + + // =========================== REPLACE ========================== + + // We have a full match. The key is between pos.start and + // keyLimit. + + int32_t newStart; + int32_t newLength = output->toReplacer()->replace(text, pos.start, keyLimit, newStart); + int32_t lenDelta = newLength - (keyLimit - pos.start); + + oText += lenDelta; + pos.limit += lenDelta; + pos.contextLimit += lenDelta; + // Restrict new value of start to [minOText, min(oText, pos.limit)]. + pos.start = uprv_max(minOText, uprv_min(uprv_min(oText, pos.limit), newStart)); + return U_MATCH; +} + +/** + * Create a source string that represents this rule. Append it to the + * given string. + */ +UnicodeString& TransliterationRule::toRule(UnicodeString& rule, + UBool escapeUnprintable) const { + + // Accumulate special characters (and non-specials following them) + // into quoteBuf. Append quoteBuf, within single quotes, when + // a non-quoted element must be inserted. + UnicodeString str, quoteBuf; + + // Do not emit the braces '{' '}' around the pattern if there + // is neither anteContext nor postContext. + UBool emitBraces = + (anteContext != nullptr) || (postContext != nullptr); + + // Emit start anchor + if ((flags & ANCHOR_START) != 0) { + rule.append((char16_t)94/*^*/); + } + + // Emit the input pattern + ICU_Utility::appendToRule(rule, anteContext, escapeUnprintable, quoteBuf); + + if (emitBraces) { + ICU_Utility::appendToRule(rule, (char16_t) 0x007B /*{*/, true, escapeUnprintable, quoteBuf); + } + + ICU_Utility::appendToRule(rule, key, escapeUnprintable, quoteBuf); + + if (emitBraces) { + ICU_Utility::appendToRule(rule, (char16_t) 0x007D /*}*/, true, escapeUnprintable, quoteBuf); + } + + ICU_Utility::appendToRule(rule, postContext, escapeUnprintable, quoteBuf); + + // Emit end anchor + if ((flags & ANCHOR_END) != 0) { + rule.append((char16_t)36/*$*/); + } + + ICU_Utility::appendToRule(rule, UnicodeString(true, FORWARD_OP, 3), true, escapeUnprintable, quoteBuf); + + // Emit the output pattern + + ICU_Utility::appendToRule(rule, output->toReplacer()->toReplacerPattern(str, escapeUnprintable), + true, escapeUnprintable, quoteBuf); + + ICU_Utility::appendToRule(rule, (char16_t) 0x003B /*;*/, true, escapeUnprintable, quoteBuf); + + return rule; +} + +void TransliterationRule::setData(const TransliterationRuleData* d) { + data = d; + if (anteContext != nullptr) anteContext->setData(d); + if (postContext != nullptr) postContext->setData(d); + if (key != nullptr) key->setData(d); + // assert(output != nullptr); + output->setData(d); + // Don't have to do segments since they are in the context or key +} + +/** + * Union the set of all characters that may be modified by this rule + * into the given set. + */ +void TransliterationRule::addSourceSetTo(UnicodeSet& toUnionTo) const { + int32_t limit = anteContextLength + keyLength; + for (int32_t i=anteContextLength; ilookupMatcher(ch); + if (matcher == nullptr) { + toUnionTo.add(ch); + } else { + matcher->addMatchSetTo(toUnionTo); + } + } +} + +/** + * Union the set of all characters that may be emitted by this rule + * into the given set. + */ +void TransliterationRule::addTargetSetTo(UnicodeSet& toUnionTo) const { + output->toReplacer()->addReplacementSetTo(toUnionTo); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +//eof diff --git a/intl/icu/source/i18n/rbt_rule.h b/intl/icu/source/i18n/rbt_rule.h new file mode 100644 index 0000000000..c6f5151d4c --- /dev/null +++ b/intl/icu/source/i18n/rbt_rule.h @@ -0,0 +1,310 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +* Copyright (C) {1999-2001}, International Business Machines Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef RBT_RULE_H +#define RBT_RULE_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/utrans.h" +#include "unicode/unimatch.h" + +U_NAMESPACE_BEGIN + +class Replaceable; +class TransliterationRuleData; +class StringMatcher; +class UnicodeFunctor; + +/** + * A transliteration rule used by + * RuleBasedTransliterator. + * TransliterationRule is an immutable object. + * + *

    A rule consists of an input pattern and an output string. When + * the input pattern is matched, the output string is emitted. The + * input pattern consists of zero or more characters which are matched + * exactly (the key) and optional context. Context must match if it + * is specified. Context may be specified before the key, after the + * key, or both. The key, preceding context, and following context + * may contain variables. Variables represent a set of Unicode + * characters, such as the letters a through z. + * Variables are detected by looking up each character in a supplied + * variable list to see if it has been so defined. + * + *

    A rule may contain segments in its input string and segment + * references in its output string. A segment is a substring of the + * input pattern, indicated by an offset and limit. The segment may + * be in the preceding or following context. It may not span a + * context boundary. A segment reference is a special character in + * the output string that causes a segment of the input string (not + * the input pattern) to be copied to the output string. The range of + * special characters that represent segment references is defined by + * RuleBasedTransliterator.Data. + * + * @author Alan Liu + */ +class TransliterationRule : public UMemory { + +private: + + // TODO Eliminate the pattern and keyLength data members. They + // are used only by masks() and getIndexValue() which are called + // only during build time, not during run-time. Perhaps these + // methods and pattern/keyLength can be isolated into a separate + // object. + + /** + * The match that must occur before the key, or null if there is no + * preceding context. + */ + StringMatcher *anteContext; + + /** + * The matcher object for the key. If null, then the key is empty. + */ + StringMatcher *key; + + /** + * The match that must occur after the key, or null if there is no + * following context. + */ + StringMatcher *postContext; + + /** + * The object that performs the replacement if the key, + * anteContext, and postContext are matched. Never null. + */ + UnicodeFunctor* output; + + /** + * The string that must be matched, consisting of the anteContext, key, + * and postContext, concatenated together, in that order. Some components + * may be empty (zero length). + * @see anteContextLength + * @see keyLength + */ + UnicodeString pattern; + + /** + * An array of matcher objects corresponding to the input pattern + * segments. If there are no segments this is null. N.B. This is + * a UnicodeMatcher for generality, but in practice it is always a + * StringMatcher. In the future we may generalize this, but for + * now we sometimes cast down to StringMatcher. + * + * The array is owned, but the pointers within it are not. + */ + UnicodeFunctor** segments; + + /** + * The number of elements in segments[] or zero if segments is nullptr. + */ + int32_t segmentsCount; + + /** + * The length of the string that must match before the key. If + * zero, then there is no matching requirement before the key. + * Substring [0,anteContextLength) of pattern is the anteContext. + */ + int32_t anteContextLength; + + /** + * The length of the key. Substring [anteContextLength, + * anteContextLength + keyLength) is the key. + + */ + int32_t keyLength; + + /** + * Miscellaneous attributes. + */ + int8_t flags; + + /** + * Flag attributes. + */ + enum { + ANCHOR_START = 1, + ANCHOR_END = 2 + }; + + /** + * An alias pointer to the data for this rule. The data provides + * lookup services for matchers and segments. + */ + const TransliterationRuleData* data; + +public: + + /** + * Construct a new rule with the given input, output text, and other + * attributes. A cursor position may be specified for the output text. + * @param input input string, including key and optional ante and + * post context. + * @param anteContextPos offset into input to end of ante context, or -1 if + * none. Must be <= input.length() if not -1. + * @param postContextPos offset into input to start of post context, or -1 + * if none. Must be <= input.length() if not -1, and must be >= + * anteContextPos. + * @param outputStr output string. + * @param cursorPosition offset into output at which cursor is located, or -1 if + * none. If less than zero, then the cursor is placed after the + * output; that is, -1 is equivalent to + * output.length(). If greater than + * output.length() then an exception is thrown. + * @param cursorOffset an offset to be added to cursorPos to position the + * cursor either in the ante context, if < 0, or in the post context, if > + * 0. For example, the rule "abc{def} > | @@@ xyz;" changes "def" to + * "xyz" and moves the cursor to before "a". It would have a cursorOffset + * of -3. + * @param segs array of UnicodeMatcher corresponding to input pattern + * segments, or null if there are none. The array itself is adopted, + * but the pointers within it are not. + * @param segsCount number of elements in segs[]. + * @param anchorStart true if the the rule is anchored on the left to + * the context start. + * @param anchorEnd true if the rule is anchored on the right to the + * context limit. + * @param data the rule data. + * @param status Output parameter filled in with success or failure status. + */ + TransliterationRule(const UnicodeString& input, + int32_t anteContextPos, int32_t postContextPos, + const UnicodeString& outputStr, + int32_t cursorPosition, int32_t cursorOffset, + UnicodeFunctor** segs, + int32_t segsCount, + UBool anchorStart, UBool anchorEnd, + const TransliterationRuleData* data, + UErrorCode& status); + + /** + * Copy constructor. + * @param other the object to be copied. + */ + TransliterationRule(TransliterationRule& other); + + /** + * Destructor. + */ + virtual ~TransliterationRule(); + + /** + * Change the data object that this rule belongs to. Used + * internally by the TransliterationRuleData copy constructor. + * @param data the new data value to be set. + */ + void setData(const TransliterationRuleData* data); + + /** + * Return the preceding context length. This method is needed to + * support the Transliterator method + * getMaximumContextLength(). Internally, this is + * implemented as the anteContextLength, optionally plus one if + * there is a start anchor. The one character anchor gap is + * needed to make repeated incremental transliteration with + * anchors work. + * @return the preceding context length. + */ + virtual int32_t getContextLength() const; + + /** + * Internal method. Returns 8-bit index value for this rule. + * This is the low byte of the first character of the key, + * unless the first character of the key is a set. If it's a + * set, or otherwise can match multiple keys, the index value is -1. + * @return 8-bit index value for this rule. + */ + int16_t getIndexValue() const; + + /** + * Internal method. Returns true if this rule matches the given + * index value. The index value is an 8-bit integer, 0..255, + * representing the low byte of the first character of the key. + * It matches this rule if it matches the first character of the + * key, or if the first character of the key is a set, and the set + * contains any character with a low byte equal to the index + * value. If the rule contains only ante context, as in foo)>bar, + * then it will match any key. + * @param v the given index value. + * @return true if this rule matches the given index value. + */ + UBool matchesIndexValue(uint8_t v) const; + + /** + * Return true if this rule masks another rule. If r1 masks r2 then + * r1 matches any input string that r2 matches. If r1 masks r2 and r2 masks + * r1 then r1 == r2. Examples: "a>x" masks "ab>y". "a>x" masks "a[b]>y". + * "[c]a>x" masks "[dc]a>y". + * @param r2 the given rule to be compared with. + * @return true if this rule masks 'r2' + */ + virtual UBool masks(const TransliterationRule& r2) const; + + /** + * Attempt a match and replacement at the given position. Return + * the degree of match between this rule and the given text. The + * degree of match may be mismatch, a partial match, or a full + * match. A mismatch means at least one character of the text + * does not match the context or key. A partial match means some + * context and key characters match, but the text is not long + * enough to match all of them. A full match means all context + * and key characters match. + * + * If a full match is obtained, perform a replacement, update pos, + * and return U_MATCH. Otherwise both text and pos are unchanged. + * + * @param text the text + * @param pos the position indices + * @param incremental if true, test for partial matches that may + * be completed by additional text inserted at pos.limit. + * @return one of U_MISMATCH, + * U_PARTIAL_MATCH, or U_MATCH. If + * incremental is false then U_PARTIAL_MATCH will not be returned. + */ + UMatchDegree matchAndReplace(Replaceable& text, + UTransPosition& pos, + UBool incremental) const; + + /** + * Create a rule string that represents this rule object. Append + * it to the given string. + */ + virtual UnicodeString& toRule(UnicodeString& pat, + UBool escapeUnprintable) const; + + /** + * Union the set of all characters that may be modified by this rule + * into the given set. + */ + void addSourceSetTo(UnicodeSet& toUnionTo) const; + + /** + * Union the set of all characters that may be emitted by this rule + * into the given set. + */ + void addTargetSetTo(UnicodeSet& toUnionTo) const; + + private: + + friend class StringMatcher; + + TransliterationRule &operator=(const TransliterationRule &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/rbt_set.cpp b/intl/icu/source/i18n/rbt_set.cpp new file mode 100644 index 0000000000..c0a2ccd868 --- /dev/null +++ b/intl/icu/source/i18n/rbt_set.cpp @@ -0,0 +1,466 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* + ********************************************************************** + * Copyright (C) 1999-2011, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + * Date Name Description + * 11/17/99 aliu Creation. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/utf16.h" +#include "rbt_set.h" +#include "rbt_rule.h" +#include "cmemory.h" +#include "putilimp.h" + +U_CDECL_BEGIN +static void U_CALLCONV _deleteRule(void *rule) { + delete (icu::TransliterationRule *)rule; +} +U_CDECL_END + +//---------------------------------------------------------------------- +// BEGIN Debugging support +//---------------------------------------------------------------------- + +// #define DEBUG_RBT + +#ifdef DEBUG_RBT +#include +#include "charstr.h" + +/** + * @param appendTo result is appended to this param. + * @param input the string being transliterated + * @param pos the index struct + */ +static UnicodeString& _formatInput(UnicodeString &appendTo, + const UnicodeString& input, + const UTransPosition& pos) { + // Output a string of the form aaa{bbb|ccc|ddd}eee, where + // the {} indicate the context start and limit, and the || + // indicate the start and limit. + if (0 <= pos.contextStart && + pos.contextStart <= pos.start && + pos.start <= pos.limit && + pos.limit <= pos.contextLimit && + pos.contextLimit <= input.length()) { + + UnicodeString a, b, c, d, e; + input.extractBetween(0, pos.contextStart, a); + input.extractBetween(pos.contextStart, pos.start, b); + input.extractBetween(pos.start, pos.limit, c); + input.extractBetween(pos.limit, pos.contextLimit, d); + input.extractBetween(pos.contextLimit, input.length(), e); + appendTo.append(a).append((char16_t)123/*{*/).append(b). + append((char16_t)124/*|*/).append(c).append((char16_t)124/*|*/).append(d). + append((char16_t)125/*}*/).append(e); + } else { + appendTo.append("INVALID UTransPosition"); + //appendTo.append((UnicodeString)"INVALID UTransPosition {cs=" + + // pos.contextStart + ", s=" + pos.start + ", l=" + + // pos.limit + ", cl=" + pos.contextLimit + "} on " + + // input); + } + return appendTo; +} + +// Append a hex string to the target +UnicodeString& _appendHex(uint32_t number, + int32_t digits, + UnicodeString& target) { + static const char16_t digitString[] = { + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0 + }; + while (digits--) { + target += digitString[(number >> (digits*4)) & 0xF]; + } + return target; +} + +// Replace nonprintable characters with unicode escapes +UnicodeString& _escape(const UnicodeString &source, + UnicodeString &target) { + for (int32_t i = 0; i < source.length(); ) { + UChar32 ch = source.char32At(i); + i += U16_LENGTH(ch); + if (ch < 0x09 || (ch > 0x0A && ch < 0x20)|| ch > 0x7E) { + if (ch <= 0xFFFF) { + target += "\\u"; + _appendHex(ch, 4, target); + } else { + target += "\\U"; + _appendHex(ch, 8, target); + } + } else { + target += ch; + } + } + return target; +} + +inline void _debugOut(const char* msg, TransliterationRule* rule, + const Replaceable& theText, UTransPosition& pos) { + UnicodeString buf(msg, ""); + if (rule) { + UnicodeString r; + rule->toRule(r, true); + buf.append((char16_t)32).append(r); + } + buf.append(UnicodeString(" => ", "")); + UnicodeString* text = (UnicodeString*)&theText; + _formatInput(buf, *text, pos); + UnicodeString esc; + _escape(buf, esc); + CharString cbuf(esc); + printf("%s\n", (const char*) cbuf); +} + +#else +#define _debugOut(msg, rule, theText, pos) +#endif + +//---------------------------------------------------------------------- +// END Debugging support +//---------------------------------------------------------------------- + +// Fill the precontext and postcontext with the patterns of the rules +// that are masking one another. +static void maskingError(const icu::TransliterationRule& rule1, + const icu::TransliterationRule& rule2, + UParseError& parseError) { + icu::UnicodeString r; + int32_t len; + + parseError.line = parseError.offset = -1; + + // for pre-context + rule1.toRule(r, false); + len = uprv_min(r.length(), U_PARSE_CONTEXT_LEN-1); + r.extract(0, len, parseError.preContext); + parseError.preContext[len] = 0; + + //for post-context + r.truncate(0); + rule2.toRule(r, false); + len = uprv_min(r.length(), U_PARSE_CONTEXT_LEN-1); + r.extract(0, len, parseError.postContext); + parseError.postContext[len] = 0; +} + +U_NAMESPACE_BEGIN + +/** + * Construct a new empty rule set. + */ +TransliterationRuleSet::TransliterationRuleSet(UErrorCode& status) : + UMemory(), ruleVector(nullptr), rules(nullptr), index {}, maxContextLength(0) { + LocalPointer lpRuleVector(new UVector(_deleteRule, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + ruleVector = lpRuleVector.orphan(); +} + +/** + * Copy constructor. + */ +TransliterationRuleSet::TransliterationRuleSet(const TransliterationRuleSet& other) : + UMemory(other), + ruleVector(nullptr), + rules(nullptr), + maxContextLength(other.maxContextLength) { + + int32_t i, len; + uprv_memcpy(index, other.index, sizeof(index)); + UErrorCode status = U_ZERO_ERROR; + LocalPointer lpRuleVector(new UVector(_deleteRule, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + ruleVector = lpRuleVector.orphan(); + if (other.ruleVector != nullptr && U_SUCCESS(status)) { + len = other.ruleVector->size(); + for (i=0; i tempTranslitRule( + new TransliterationRule(*(TransliterationRule*)other.ruleVector->elementAt(i)), status); + ruleVector->adoptElement(tempTranslitRule.orphan(), status); + } + } + if (other.rules != 0 && U_SUCCESS(status)) { + UParseError p; + freeze(p, status); + } +} + +/** + * Destructor. + */ +TransliterationRuleSet::~TransliterationRuleSet() { + delete ruleVector; // This deletes the contained rules + uprv_free(rules); +} + +void TransliterationRuleSet::setData(const TransliterationRuleData* d) { + /** + * We assume that the ruleset has already been frozen. + */ + int32_t len = index[256]; // see freeze() + for (int32_t i=0; isetData(d); + } +} + +/** + * Return the maximum context length. + * @return the length of the longest preceding context. + */ +int32_t TransliterationRuleSet::getMaximumContextLength() const { + return maxContextLength; +} + +/** + * Add a rule to this set. Rules are added in order, and order is + * significant. The last call to this method must be followed by + * a call to freeze() before the rule set is used. + * + *

    If freeze() has already been called, calling addRule() + * unfreezes the rules, and freeze() must be called again. + * + * @param adoptedRule the rule to add + */ +void TransliterationRuleSet::addRule(TransliterationRule* adoptedRule, + UErrorCode& status) { + LocalPointer lpAdoptedRule(adoptedRule); + ruleVector->adoptElement(lpAdoptedRule.orphan(), status); + if (U_FAILURE(status)) { + return; + } + + int32_t len; + if ((len = adoptedRule->getContextLength()) > maxContextLength) { + maxContextLength = len; + } + + uprv_free(rules); + rules = 0; +} + +/** + * Check this for masked rules and index it to optimize performance. + * The sequence of operations is: (1) add rules to a set using + * addRule(); (2) freeze the set using + * freeze(); (3) use the rule set. If + * addRule() is called after calling this method, it + * invalidates this object, and this method must be called again. + * That is, freeze() may be called multiple times, + * although for optimal performance it shouldn't be. + */ +void TransliterationRuleSet::freeze(UParseError& parseError,UErrorCode& status) { + /* Construct the rule array and index table. We reorder the + * rules by sorting them into 256 bins. Each bin contains all + * rules matching the index value for that bin. A rule + * matches an index value if string whose first key character + * has a low byte equal to the index value can match the rule. + * + * Each bin contains zero or more rules, in the same order + * they were found originally. However, the total rules in + * the bins may exceed the number in the original vector, + * since rules that have a variable as their first key + * character will generally fall into more than one bin. + * + * That is, each bin contains all rules that either have that + * first index value as their first key character, or have + * a set containing the index value as their first character. + */ + int32_t n = ruleVector->size(); + int32_t j; + int16_t x; + UVector v(2*n, status); // heuristic; adjust as needed + + if (U_FAILURE(status)) { + return; + } + + /* Precompute the index values. This saves a LOT of time. + * Be careful not to call malloc(0). + */ + int16_t* indexValue = (int16_t*) uprv_malloc( sizeof(int16_t) * (n > 0 ? n : 1) ); + /* test for nullptr */ + if (indexValue == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (j=0; jelementAt(j); + indexValue[j] = r->getIndexValue(); + } + for (x=0; x<256; ++x) { + index[x] = v.size(); + for (j=0; j= 0) { + if (indexValue[j] == x) { + v.addElement(ruleVector->elementAt(j), status); + } + } else { + // If the indexValue is < 0, then the first key character is + // a set, and we must use the more time-consuming + // matchesIndexValue check. In practice this happens + // rarely, so we seldom treat this code path. + TransliterationRule* r = (TransliterationRule*) ruleVector->elementAt(j); + if (r->matchesIndexValue((uint8_t)x)) { + v.addElement(r, status); + } + } + } + } + uprv_free(indexValue); + index[256] = v.size(); + if (U_FAILURE(status)) { + return; + } + + /* Freeze things into an array. + */ + uprv_free(rules); // Contains alias pointers + + /* You can't do malloc(0)! */ + if (v.size() == 0) { + rules = nullptr; + return; + } + rules = (TransliterationRule **)uprv_malloc(v.size() * sizeof(TransliterationRule *)); + /* test for nullptr */ + if (rules == 0) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + for (j=0; jmasks(*r2)) { +//| if (errors == null) { +//| errors = new StringBuffer(); +//| } else { +//| errors.append("\n"); +//| } +//| errors.append("Rule " + r1 + " masks " + r2); + status = U_RULE_MASK_ERROR; + maskingError(*r1, *r2, parseError); + return; + } + } + } + } + + //if (errors != null) { + // throw new IllegalArgumentException(errors.toString()); + //} +} + +/** + * Transliterate the given text with the given UTransPosition + * indices. Return true if the transliteration should continue + * or false if it should halt (because of a U_PARTIAL_MATCH match). + * Note that false is only ever returned if isIncremental is true. + * @param text the text to be transliterated + * @param pos the position indices, which will be updated + * @param incremental if true, assume new text may be inserted + * at index.limit, and return false if there is a partial match. + * @return true unless a U_PARTIAL_MATCH has been obtained, + * indicating that transliteration should stop until more text + * arrives. + */ +UBool TransliterationRuleSet::transliterate(Replaceable& text, + UTransPosition& pos, + UBool incremental) { + int16_t indexByte = (int16_t) (text.char32At(pos.start) & 0xFF); + for (int32_t i=index[indexByte]; imatchAndReplace(text, pos, incremental); + switch (m) { + case U_MATCH: + _debugOut("match", rules[i], text, pos); + return true; + case U_PARTIAL_MATCH: + _debugOut("partial match", rules[i], text, pos); + return false; + default: /* Ram: added default to make GCC happy */ + break; + } + } + // No match or partial match from any rule + pos.start += U16_LENGTH(text.char32At(pos.start)); + _debugOut("no match", nullptr, text, pos); + return true; +} + +/** + * Create rule strings that represents this rule set. + */ +UnicodeString& TransliterationRuleSet::toRules(UnicodeString& ruleSource, + UBool escapeUnprintable) const { + int32_t i; + int32_t count = ruleVector->size(); + ruleSource.truncate(0); + for (i=0; ielementAt(i); + r->toRule(ruleSource, escapeUnprintable); + } + return ruleSource; +} + +/** + * Return the set of all characters that may be modified + * (getTarget=false) or emitted (getTarget=true) by this set. + */ +UnicodeSet& TransliterationRuleSet::getSourceTargetSet(UnicodeSet& result, + UBool getTarget) const +{ + result.clear(); + int32_t count = ruleVector->size(); + for (int32_t i=0; ielementAt(i); + if (getTarget) { + r->addTargetSetTo(result); + } else { + r->addSourceSetTo(result); + } + } + return result; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/rbt_set.h b/intl/icu/source/i18n/rbt_set.h new file mode 100644 index 0000000000..fb436ff67e --- /dev/null +++ b/intl/icu/source/i18n/rbt_set.h @@ -0,0 +1,167 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2007, International Business Machines Corporation +* and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef RBT_SET_H +#define RBT_SET_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "unicode/utrans.h" +#include "uvector.h" + +U_NAMESPACE_BEGIN + +class Replaceable; +class TransliterationRule; +class TransliterationRuleData; +class UnicodeFilter; +class UnicodeString; +class UnicodeSet; + +/** + * A set of rules for a RuleBasedTransliterator. + * @author Alan Liu + */ +class TransliterationRuleSet : public UMemory { + /** + * Vector of rules, in the order added. This is used while the + * rule set is getting built. After that, freeze() reorders and + * indexes the rules into rules[]. Any given rule is stored once + * in ruleVector, and one or more times in rules[]. ruleVector + * owns and deletes the rules. + */ + UVector* ruleVector; + + /** + * Sorted and indexed table of rules. This is created by freeze() + * from the rules in ruleVector. It contains alias pointers to + * the rules in ruleVector. It is zero before freeze() is called + * and non-zero thereafter. + */ + TransliterationRule** rules; + + /** + * Index table. For text having a first character c, compute x = c&0xFF. + * Now use rules[index[x]..index[x+1]-1]. This index table is created by + * freeze(). Before freeze() is called it contains garbage. + */ + int32_t index[257]; + + /** + * Length of the longest preceding context + */ + int32_t maxContextLength; + +public: + + /** + * Construct a new empty rule set. + * @param status Output parameter filled in with success or failure status. + */ + TransliterationRuleSet(UErrorCode& status); + + /** + * Copy constructor. + */ + TransliterationRuleSet(const TransliterationRuleSet&); + + /** + * Destructor. + */ + virtual ~TransliterationRuleSet(); + + /** + * Change the data object that this rule belongs to. Used + * internally by the TransliterationRuleData copy constructor. + * @param data the new data value to be set. + */ + void setData(const TransliterationRuleData* data); + + /** + * Return the maximum context length. + * @return the length of the longest preceding context. + */ + virtual int32_t getMaximumContextLength() const; + + /** + * Add a rule to this set. Rules are added in order, and order is + * significant. The last call to this method must be followed by + * a call to freeze() before the rule set is used. + * This method must not be called after freeze() has been + * called. + * + * @param adoptedRule the rule to add + */ + virtual void addRule(TransliterationRule* adoptedRule, + UErrorCode& status); + + /** + * Check this for masked rules and index it to optimize performance. + * The sequence of operations is: (1) add rules to a set using + * addRule(); (2) freeze the set using + * freeze(); (3) use the rule set. If + * addRule() is called after calling this method, it + * invalidates this object, and this method must be called again. + * That is, freeze() may be called multiple times, + * although for optimal performance it shouldn't be. + * @param parseError A pointer to UParseError to receive information about errors + * occurred. + * @param status Output parameter filled in with success or failure status. + */ + virtual void freeze(UParseError& parseError, UErrorCode& status); + + /** + * Transliterate the given text with the given UTransPosition + * indices. Return true if the transliteration should continue + * or false if it should halt (because of a U_PARTIAL_MATCH match). + * Note that false is only ever returned if isIncremental is true. + * @param text the text to be transliterated + * @param index the position indices, which will be updated + * @param isIncremental if true, assume new text may be inserted + * at index.limit, and return false if thrre is a partial match. + * @return true unless a U_PARTIAL_MATCH has been obtained, + * indicating that transliteration should stop until more text + * arrives. + */ + UBool transliterate(Replaceable& text, + UTransPosition& index, + UBool isIncremental); + + /** + * Create rule strings that represents this rule set. + * @param result string to receive the rule strings. Current + * contents will be deleted. + * @param escapeUnprintable True, will escape the unprintable characters + * @return A reference to 'result'. + */ + virtual UnicodeString& toRules(UnicodeString& result, + UBool escapeUnprintable) const; + + /** + * Return the set of all characters that may be modified + * (getTarget=false) or emitted (getTarget=true) by this set. + */ + UnicodeSet& getSourceTargetSet(UnicodeSet& result, + UBool getTarget) const; + +private: + + TransliterationRuleSet &operator=(const TransliterationRuleSet &other); // forbid copying of this class +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/rbtz.cpp b/intl/icu/source/i18n/rbtz.cpp new file mode 100644 index 0000000000..e4d71436c5 --- /dev/null +++ b/intl/icu/source/i18n/rbtz.cpp @@ -0,0 +1,936 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2013, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "utypeinfo.h" // for 'typeid' to work + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/rbtz.h" +#include "unicode/gregocal.h" +#include "uvector.h" +#include "gregoimp.h" +#include "cmemory.h" +#include "umutex.h" + +U_NAMESPACE_BEGIN + +/** + * A struct representing a time zone transition + */ +struct Transition : public UMemory { + UDate time; + TimeZoneRule* from; + TimeZoneRule* to; +}; + +U_CDECL_BEGIN +static void U_CALLCONV +deleteTransition(void* obj) { + delete static_cast(obj); +} +U_CDECL_END + +static UBool compareRules(UVector* rules1, UVector* rules2) { + if (rules1 == nullptr && rules2 == nullptr) { + return true; + } else if (rules1 == nullptr || rules2 == nullptr) { + return false; + } + int32_t size = rules1->size(); + if (size != rules2->size()) { + return false; + } + for (int32_t i = 0; i < size; i++) { + TimeZoneRule *r1 = (TimeZoneRule*)rules1->elementAt(i); + TimeZoneRule *r2 = (TimeZoneRule*)rules2->elementAt(i); + if (*r1 != *r2) { + return false; + } + } + return true; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedTimeZone) + +RuleBasedTimeZone::RuleBasedTimeZone(const UnicodeString& id, InitialTimeZoneRule* initialRule) +: BasicTimeZone(id), fInitialRule(initialRule), fHistoricRules(nullptr), fFinalRules(nullptr), + fHistoricTransitions(nullptr), fUpToDate(false) { +} + +RuleBasedTimeZone::RuleBasedTimeZone(const RuleBasedTimeZone& source) +: BasicTimeZone(source), fInitialRule(source.fInitialRule->clone()), + fHistoricTransitions(nullptr), fUpToDate(false) { + fHistoricRules = copyRules(source.fHistoricRules); + fFinalRules = copyRules(source.fFinalRules); + if (source.fUpToDate) { + UErrorCode status = U_ZERO_ERROR; + complete(status); + } +} + +RuleBasedTimeZone::~RuleBasedTimeZone() { + deleteTransitions(); + deleteRules(); +} + +RuleBasedTimeZone& +RuleBasedTimeZone::operator=(const RuleBasedTimeZone& right) { + if (*this != right) { + BasicTimeZone::operator=(right); + deleteRules(); + fInitialRule = right.fInitialRule->clone(); + fHistoricRules = copyRules(right.fHistoricRules); + fFinalRules = copyRules(right.fFinalRules); + deleteTransitions(); + fUpToDate = false; + } + return *this; +} + +bool +RuleBasedTimeZone::operator==(const TimeZone& that) const { + if (this == &that) { + return true; + } + if (typeid(*this) != typeid(that) || !BasicTimeZone::operator==(that)) { + return false; + } + RuleBasedTimeZone *rbtz = (RuleBasedTimeZone*)&that; + if (*fInitialRule != *(rbtz->fInitialRule)) { + return false; + } + if (compareRules(fHistoricRules, rbtz->fHistoricRules) + && compareRules(fFinalRules, rbtz->fFinalRules)) { + return true; + } + return false; +} + +bool +RuleBasedTimeZone::operator!=(const TimeZone& that) const { + return !operator==(that); +} + +void +RuleBasedTimeZone::addTransitionRule(TimeZoneRule* rule, UErrorCode& status) { + LocalPointerlpRule(rule); + if (U_FAILURE(status)) { + return; + } + AnnualTimeZoneRule* atzrule = dynamic_cast(rule); + if (atzrule != nullptr && atzrule->getEndYear() == AnnualTimeZoneRule::MAX_YEAR) { + // A final rule + if (fFinalRules == nullptr) { + LocalPointer lpFinalRules(new UVector(uprv_deleteUObject, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + fFinalRules = lpFinalRules.orphan(); + } else if (fFinalRules->size() >= 2) { + // Cannot handle more than two final rules + status = U_INVALID_STATE_ERROR; + return; + } + fFinalRules->adoptElement(lpRule.orphan(), status); + } else { + // Non-final rule + if (fHistoricRules == nullptr) { + LocalPointer lpHistoricRules(new UVector(uprv_deleteUObject, nullptr, status), status); + if (U_FAILURE(status)) { + return; + } + fHistoricRules = lpHistoricRules.orphan(); + } + fHistoricRules->adoptElement(lpRule.orphan(), status); + } + // Mark dirty, so transitions are recalculated at next complete() call + fUpToDate = false; +} + + +void +RuleBasedTimeZone::completeConst(UErrorCode& status) const { + static UMutex gLock; + if (U_FAILURE(status)) { + return; + } + umtx_lock(&gLock); + if (!fUpToDate) { + RuleBasedTimeZone *ncThis = const_cast(this); + ncThis->complete(status); + } + umtx_unlock(&gLock); +} + +void +RuleBasedTimeZone::complete(UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + if (fUpToDate) { + return; + } + // Make sure either no final rules or a pair of AnnualTimeZoneRules + // are available. + if (fFinalRules != nullptr && fFinalRules->size() != 2) { + status = U_INVALID_STATE_ERROR; + return; + } + + // Create a TimezoneTransition and add to the list + if (fHistoricRules != nullptr || fFinalRules != nullptr) { + TimeZoneRule *curRule = fInitialRule; + UDate lastTransitionTime = MIN_MILLIS; + + // Build the transition array which represents historical time zone + // transitions. + if (fHistoricRules != nullptr && fHistoricRules->size() > 0) { + int32_t i; + int32_t historicCount = fHistoricRules->size(); + LocalMemory done((bool *)uprv_malloc(sizeof(bool) * historicCount)); + if (done == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + goto cleanup; + } + for (i = 0; i < historicCount; i++) { + done[i] = false; + } + while (true) { + int32_t curStdOffset = curRule->getRawOffset(); + int32_t curDstSavings = curRule->getDSTSavings(); + UDate nextTransitionTime = MAX_MILLIS; + TimeZoneRule *nextRule = nullptr; + TimeZoneRule *r = nullptr; + UBool avail; + UDate tt; + UnicodeString curName, name; + curRule->getName(curName); + + for (i = 0; i < historicCount; i++) { + if (done[i]) { + continue; + } + r = (TimeZoneRule*)fHistoricRules->elementAt(i); + avail = r->getNextStart(lastTransitionTime, curStdOffset, curDstSavings, false, tt); + if (!avail) { + // No more transitions from this rule - skip this rule next time + done[i] = true; + } else { + r->getName(name); + if (*r == *curRule || + (name == curName && r->getRawOffset() == curRule->getRawOffset() + && r->getDSTSavings() == curRule->getDSTSavings())) { + continue; + } + if (tt < nextTransitionTime) { + nextTransitionTime = tt; + nextRule = r; + } + } + } + + if (nextRule == nullptr) { + // Check if all historic rules are done + UBool bDoneAll = true; + for (int32_t j = 0; j < historicCount; j++) { + if (!done[j]) { + bDoneAll = false; + break; + } + } + if (bDoneAll) { + break; + } + } + + if (fFinalRules != nullptr) { + // Check if one of final rules has earlier transition date + for (i = 0; i < 2 /* fFinalRules->size() */; i++) { + TimeZoneRule *fr = (TimeZoneRule*)fFinalRules->elementAt(i); + if (*fr == *curRule) { + continue; + } + r = (TimeZoneRule*)fFinalRules->elementAt(i); + avail = r->getNextStart(lastTransitionTime, curStdOffset, curDstSavings, false, tt); + if (avail) { + if (tt < nextTransitionTime) { + nextTransitionTime = tt; + nextRule = r; + } + } + } + } + + if (nextRule == nullptr) { + // Nothing more + break; + } + + if (fHistoricTransitions == nullptr) { + LocalPointer lpHistoricTransitions( + new UVector(deleteTransition, nullptr, status), status); + if (U_FAILURE(status)) { + goto cleanup; + } + fHistoricTransitions = lpHistoricTransitions.orphan(); + } + LocalPointer trst(new Transition, status); + if (U_FAILURE(status)) { + goto cleanup; + } + trst->time = nextTransitionTime; + trst->from = curRule; + trst->to = nextRule; + fHistoricTransitions->adoptElement(trst.orphan(), status); + if (U_FAILURE(status)) { + goto cleanup; + } + lastTransitionTime = nextTransitionTime; + curRule = nextRule; + } + } + if (fFinalRules != nullptr) { + if (fHistoricTransitions == nullptr) { + LocalPointer lpHistoricTransitions( + new UVector(deleteTransition, nullptr, status), status); + if (U_FAILURE(status)) { + goto cleanup; + } + fHistoricTransitions = lpHistoricTransitions.orphan(); + } + // Append the first transition for each + TimeZoneRule *rule0 = (TimeZoneRule*)fFinalRules->elementAt(0); + TimeZoneRule *rule1 = (TimeZoneRule*)fFinalRules->elementAt(1); + UDate tt0, tt1; + UBool avail0 = rule0->getNextStart(lastTransitionTime, curRule->getRawOffset(), curRule->getDSTSavings(), false, tt0); + UBool avail1 = rule1->getNextStart(lastTransitionTime, curRule->getRawOffset(), curRule->getDSTSavings(), false, tt1); + if (!avail0 || !avail1) { + // Should not happen, because both rules are permanent + status = U_INVALID_STATE_ERROR; + goto cleanup; + } + LocalPointer final0(new Transition, status); + LocalPointer final1(new Transition, status); + if (U_FAILURE(status)) { + goto cleanup; + } + if (tt0 < tt1) { + final0->time = tt0; + final0->from = curRule; + final0->to = rule0; + rule1->getNextStart(tt0, rule0->getRawOffset(), rule0->getDSTSavings(), false, final1->time); + final1->from = rule0; + final1->to = rule1; + } else { + final0->time = tt1; + final0->from = curRule; + final0->to = rule1; + rule0->getNextStart(tt1, rule1->getRawOffset(), rule1->getDSTSavings(), false, final1->time); + final1->from = rule1; + final1->to = rule0; + } + fHistoricTransitions->adoptElement(final0.orphan(), status); + fHistoricTransitions->adoptElement(final1.orphan(), status); + if (U_FAILURE(status)) { + goto cleanup; + } + } + } + fUpToDate = true; + return; + +cleanup: + deleteTransitions(); + fUpToDate = false; +} + +RuleBasedTimeZone* +RuleBasedTimeZone::clone() const { + return new RuleBasedTimeZone(*this); +} + +int32_t +RuleBasedTimeZone::getOffset(uint8_t era, int32_t year, int32_t month, int32_t day, + uint8_t dayOfWeek, int32_t millis, UErrorCode& status) const { + if (U_FAILURE(status)) { + return 0; + } + if (month < UCAL_JANUARY || month > UCAL_DECEMBER) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } else { + return getOffset(era, year, month, day, dayOfWeek, millis, + Grego::monthLength(year, month), status); + } +} + +int32_t +RuleBasedTimeZone::getOffset(uint8_t era, int32_t year, int32_t month, int32_t day, + uint8_t /*dayOfWeek*/, int32_t millis, + int32_t /*monthLength*/, UErrorCode& status) const { + // dayOfWeek and monthLength are unused + if (U_FAILURE(status)) { + return 0; + } + if (era == GregorianCalendar::BC) { + // Convert to extended year + year = 1 - year; + } + int32_t rawOffset, dstOffset; + UDate time = (UDate)Grego::fieldsToDay(year, month, day) * U_MILLIS_PER_DAY + millis; + getOffsetInternal(time, true, kDaylight, kStandard, rawOffset, dstOffset, status); + if (U_FAILURE(status)) { + return 0; + } + return (rawOffset + dstOffset); +} + +void +RuleBasedTimeZone::getOffset(UDate date, UBool local, int32_t& rawOffset, + int32_t& dstOffset, UErrorCode& status) const { + getOffsetInternal(date, local, kFormer, kLatter, rawOffset, dstOffset, status); +} + +void RuleBasedTimeZone::getOffsetFromLocal(UDate date, UTimeZoneLocalOption nonExistingTimeOpt, + UTimeZoneLocalOption duplicatedTimeOpt, + int32_t& rawOffset, int32_t& dstOffset, UErrorCode& status) const { + getOffsetInternal(date, true, nonExistingTimeOpt, duplicatedTimeOpt, rawOffset, dstOffset, status); +} + + +/* + * The internal getOffset implementation + */ +void +RuleBasedTimeZone::getOffsetInternal(UDate date, UBool local, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt, + int32_t& rawOffset, int32_t& dstOffset, + UErrorCode& status) const { + rawOffset = 0; + dstOffset = 0; + + if (U_FAILURE(status)) { + return; + } + if (!fUpToDate) { + // Transitions are not yet resolved. We cannot do it here + // because this method is const. Thus, do nothing and return + // error status. + status = U_INVALID_STATE_ERROR; + return; + } + const TimeZoneRule *rule = nullptr; + if (fHistoricTransitions == nullptr) { + rule = fInitialRule; + } else { + UDate tstart = getTransitionTime((Transition*)fHistoricTransitions->elementAt(0), + local, NonExistingTimeOpt, DuplicatedTimeOpt); + if (date < tstart) { + rule = fInitialRule; + } else { + int32_t idx = fHistoricTransitions->size() - 1; + UDate tend = getTransitionTime((Transition*)fHistoricTransitions->elementAt(idx), + local, NonExistingTimeOpt, DuplicatedTimeOpt); + if (date > tend) { + if (fFinalRules != nullptr) { + rule = findRuleInFinal(date, local, NonExistingTimeOpt, DuplicatedTimeOpt); + } + if (rule == nullptr) { + // no final rules or the given time is before the first transition + // specified by the final rules -> use the last rule + rule = ((Transition*)fHistoricTransitions->elementAt(idx))->to; + } + } else { + // Find a historical transition + while (idx >= 0) { + if (date >= getTransitionTime((Transition*)fHistoricTransitions->elementAt(idx), + local, NonExistingTimeOpt, DuplicatedTimeOpt)) { + break; + } + idx--; + } + rule = ((Transition*)fHistoricTransitions->elementAt(idx))->to; + } + } + } + if (rule != nullptr) { + rawOffset = rule->getRawOffset(); + dstOffset = rule->getDSTSavings(); + } +} + +void +RuleBasedTimeZone::setRawOffset(int32_t /*offsetMillis*/) { + // We don't support this operation at this moment. + // Nothing to do! +} + +int32_t +RuleBasedTimeZone::getRawOffset() const { + // Note: This implementation returns standard GMT offset + // as of current time. + UErrorCode status = U_ZERO_ERROR; + int32_t raw, dst; + getOffset(uprv_getUTCtime(), false, raw, dst, status); + return raw; +} + +UBool +RuleBasedTimeZone::useDaylightTime() const { + // Note: This implementation returns true when + // daylight saving time is used as of now or + // after the next transition. + UErrorCode status = U_ZERO_ERROR; + UDate now = uprv_getUTCtime(); + int32_t raw, dst; + getOffset(now, false, raw, dst, status); + if (dst != 0) { + return true; + } + // If DST is not used now, check if DST is used after the next transition + UDate time; + TimeZoneRule *from, *to; + UBool avail = findNext(now, false, time, from, to); + if (avail && to->getDSTSavings() != 0) { + return true; + } + return false; +} + +UBool +RuleBasedTimeZone::inDaylightTime(UDate date, UErrorCode& status) const { + if (U_FAILURE(status)) { + return false; + } + int32_t raw, dst; + getOffset(date, false, raw, dst, status); + if (dst != 0) { + return true; + } + return false; +} + +UBool +RuleBasedTimeZone::hasSameRules(const TimeZone& other) const { + if (this == &other) { + return true; + } + if (typeid(*this) != typeid(other)) { + return false; + } + const RuleBasedTimeZone& that = static_cast(other); + if (*fInitialRule != *(that.fInitialRule)) { + return false; + } + if (compareRules(fHistoricRules, that.fHistoricRules) + && compareRules(fFinalRules, that.fFinalRules)) { + return true; + } + return false; +} + +UBool +RuleBasedTimeZone::getNextTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const { + UErrorCode status = U_ZERO_ERROR; + completeConst(status); + if (U_FAILURE(status)) { + return false; + } + UDate transitionTime; + TimeZoneRule *fromRule, *toRule; + UBool found = findNext(base, inclusive, transitionTime, fromRule, toRule); + if (found) { + result.setTime(transitionTime); + result.setFrom(*fromRule); + result.setTo(*toRule); + return true; + } + return false; +} + +UBool +RuleBasedTimeZone::getPreviousTransition(UDate base, UBool inclusive, TimeZoneTransition& result) const { + UErrorCode status = U_ZERO_ERROR; + completeConst(status); + if (U_FAILURE(status)) { + return false; + } + UDate transitionTime; + TimeZoneRule *fromRule, *toRule; + UBool found = findPrev(base, inclusive, transitionTime, fromRule, toRule); + if (found) { + result.setTime(transitionTime); + result.setFrom(*fromRule); + result.setTo(*toRule); + return true; + } + return false; +} + +int32_t +RuleBasedTimeZone::countTransitionRules(UErrorCode& /*status*/) const { + int32_t count = 0; + if (fHistoricRules != nullptr) { + count += fHistoricRules->size(); + } + if (fFinalRules != nullptr) { + count += fFinalRules->size(); + } + return count; +} + +void +RuleBasedTimeZone::getTimeZoneRules(const InitialTimeZoneRule*& initial, + const TimeZoneRule* trsrules[], + int32_t& trscount, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + // Initial rule + initial = fInitialRule; + + // Transition rules + int32_t cnt = 0; + int32_t idx; + if (fHistoricRules != nullptr && cnt < trscount) { + int32_t historicCount = fHistoricRules->size(); + idx = 0; + while (cnt < trscount && idx < historicCount) { + trsrules[cnt++] = (const TimeZoneRule*)fHistoricRules->elementAt(idx++); + } + } + if (fFinalRules != nullptr && cnt < trscount) { + int32_t finalCount = fFinalRules->size(); + idx = 0; + while (cnt < trscount && idx < finalCount) { + trsrules[cnt++] = (const TimeZoneRule*)fFinalRules->elementAt(idx++); + } + } + // Set the result length + trscount = cnt; +} + +void +RuleBasedTimeZone::deleteRules() { + delete fInitialRule; + fInitialRule = nullptr; + if (fHistoricRules != nullptr) { + delete fHistoricRules; + fHistoricRules = nullptr; + } + if (fFinalRules != nullptr) { + delete fFinalRules; + fFinalRules = nullptr; + } +} + +void +RuleBasedTimeZone::deleteTransitions() { + if (fHistoricTransitions != nullptr) { + delete fHistoricTransitions; + } + fHistoricTransitions = nullptr; +} + +UVector* +RuleBasedTimeZone::copyRules(UVector* source) { + if (source == nullptr) { + return nullptr; + } + UErrorCode ec = U_ZERO_ERROR; + int32_t size = source->size(); + LocalPointer rules(new UVector(uprv_deleteUObject, nullptr, size, ec), ec); + if (U_FAILURE(ec)) { + return nullptr; + } + int32_t i; + for (i = 0; i < size; i++) { + LocalPointer rule(((TimeZoneRule*)source->elementAt(i))->clone(), ec); + rules->adoptElement(rule.orphan(), ec); + if (U_FAILURE(ec)) { + return nullptr; + } + } + return rules.orphan(); +} + +TimeZoneRule* +RuleBasedTimeZone::findRuleInFinal(UDate date, UBool local, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt) const { + if (fFinalRules == nullptr) { + return nullptr; + } + + AnnualTimeZoneRule* fr0 = (AnnualTimeZoneRule*)fFinalRules->elementAt(0); + AnnualTimeZoneRule* fr1 = (AnnualTimeZoneRule*)fFinalRules->elementAt(1); + if (fr0 == nullptr || fr1 == nullptr) { + return nullptr; + } + + UDate start0, start1; + UDate base; + int32_t localDelta; + + base = date; + if (local) { + localDelta = getLocalDelta(fr1->getRawOffset(), fr1->getDSTSavings(), + fr0->getRawOffset(), fr0->getDSTSavings(), + NonExistingTimeOpt, DuplicatedTimeOpt); + base -= localDelta; + } + UBool avail0 = fr0->getPreviousStart(base, fr1->getRawOffset(), fr1->getDSTSavings(), true, start0); + + base = date; + if (local) { + localDelta = getLocalDelta(fr0->getRawOffset(), fr0->getDSTSavings(), + fr1->getRawOffset(), fr1->getDSTSavings(), + NonExistingTimeOpt, DuplicatedTimeOpt); + base -= localDelta; + } + UBool avail1 = fr1->getPreviousStart(base, fr0->getRawOffset(), fr0->getDSTSavings(), true, start1); + + if (!avail0 || !avail1) { + if (avail0) { + return fr0; + } else if (avail1) { + return fr1; + } + // Both rules take effect after the given time + return nullptr; + } + + return (start0 > start1) ? fr0 : fr1; +} + +UBool +RuleBasedTimeZone::findNext(UDate base, UBool inclusive, UDate& transitionTime, + TimeZoneRule*& fromRule, TimeZoneRule*& toRule) const { + if (fHistoricTransitions == nullptr) { + return false; + } + UBool isFinal = false; + UBool found = false; + Transition result; + Transition *tzt = (Transition*)fHistoricTransitions->elementAt(0); + UDate tt = tzt->time; + if (tt > base || (inclusive && tt == base)) { + result = *tzt; + found = true; + } else { + int32_t idx = fHistoricTransitions->size() - 1; + tzt = (Transition*)fHistoricTransitions->elementAt(idx); + tt = tzt->time; + if (inclusive && tt == base) { + result = *tzt; + found = true; + } else if (tt <= base) { + if (fFinalRules != nullptr) { + // Find a transion time with finalRules + TimeZoneRule *r0 = (TimeZoneRule*)fFinalRules->elementAt(0); + TimeZoneRule *r1 = (TimeZoneRule*)fFinalRules->elementAt(1); + UDate start0, start1; + UBool avail0 = r0->getNextStart(base, r1->getRawOffset(), r1->getDSTSavings(), inclusive, start0); + UBool avail1 = r1->getNextStart(base, r0->getRawOffset(), r0->getDSTSavings(), inclusive, start1); + // avail0/avail1 should be always true + if (!avail0 && !avail1) { + return false; + } + if (!avail1 || start0 < start1) { + result.time = start0; + result.from = r1; + result.to = r0; + } else { + result.time = start1; + result.from = r0; + result.to = r1; + } + isFinal = true; + found = true; + } + } else { + // Find a transition within the historic transitions + idx--; + Transition *prev = tzt; + while (idx > 0) { + tzt = (Transition*)fHistoricTransitions->elementAt(idx); + tt = tzt->time; + if (tt < base || (!inclusive && tt == base)) { + break; + } + idx--; + prev = tzt; + } + result.time = prev->time; + result.from = prev->from; + result.to = prev->to; + found = true; + } + } + if (found) { + // For now, this implementation ignore transitions with only zone name changes. + if (result.from->getRawOffset() == result.to->getRawOffset() + && result.from->getDSTSavings() == result.to->getDSTSavings()) { + if (isFinal) { + return false; + } else { + // No offset changes. Try next one if not final + return findNext(result.time, false /* always exclusive */, + transitionTime, fromRule, toRule); + } + } + transitionTime = result.time; + fromRule = result.from; + toRule = result.to; + return true; + } + return false; +} + +UBool +RuleBasedTimeZone::findPrev(UDate base, UBool inclusive, UDate& transitionTime, + TimeZoneRule*& fromRule, TimeZoneRule*& toRule) const { + if (fHistoricTransitions == nullptr) { + return false; + } + UBool found = false; + Transition result; + Transition *tzt = (Transition*)fHistoricTransitions->elementAt(0); + UDate tt = tzt->time; + if (inclusive && tt == base) { + result = *tzt; + found = true; + } else if (tt < base) { + int32_t idx = fHistoricTransitions->size() - 1; + tzt = (Transition*)fHistoricTransitions->elementAt(idx); + tt = tzt->time; + if (inclusive && tt == base) { + result = *tzt; + found = true; + } else if (tt < base) { + if (fFinalRules != nullptr) { + // Find a transion time with finalRules + TimeZoneRule *r0 = (TimeZoneRule*)fFinalRules->elementAt(0); + TimeZoneRule *r1 = (TimeZoneRule*)fFinalRules->elementAt(1); + UDate start0, start1; + UBool avail0 = r0->getPreviousStart(base, r1->getRawOffset(), r1->getDSTSavings(), inclusive, start0); + UBool avail1 = r1->getPreviousStart(base, r0->getRawOffset(), r0->getDSTSavings(), inclusive, start1); + // avail0/avail1 should be always true + if (!avail0 && !avail1) { + return false; + } + if (!avail1 || start0 > start1) { + result.time = start0; + result.from = r1; + result.to = r0; + } else { + result.time = start1; + result.from = r0; + result.to = r1; + } + } else { + result = *tzt; + } + found = true; + } else { + // Find a transition within the historic transitions + idx--; + while (idx >= 0) { + tzt = (Transition*)fHistoricTransitions->elementAt(idx); + tt = tzt->time; + if (tt < base || (inclusive && tt == base)) { + break; + } + idx--; + } + result = *tzt; + found = true; + } + } + if (found) { + // For now, this implementation ignore transitions with only zone name changes. + if (result.from->getRawOffset() == result.to->getRawOffset() + && result.from->getDSTSavings() == result.to->getDSTSavings()) { + // No offset changes. Try next one if not final + return findPrev(result.time, false /* always exclusive */, + transitionTime, fromRule, toRule); + } + transitionTime = result.time; + fromRule = result.from; + toRule = result.to; + return true; + } + return false; +} + +UDate +RuleBasedTimeZone::getTransitionTime(Transition* transition, UBool local, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt) const { + UDate time = transition->time; + if (local) { + time += getLocalDelta(transition->from->getRawOffset(), transition->from->getDSTSavings(), + transition->to->getRawOffset(), transition->to->getDSTSavings(), + NonExistingTimeOpt, DuplicatedTimeOpt); + } + return time; +} + +int32_t +RuleBasedTimeZone::getLocalDelta(int32_t rawBefore, int32_t dstBefore, int32_t rawAfter, int32_t dstAfter, + int32_t NonExistingTimeOpt, int32_t DuplicatedTimeOpt) const { + int32_t delta = 0; + + int32_t offsetBefore = rawBefore + dstBefore; + int32_t offsetAfter = rawAfter + dstAfter; + + UBool dstToStd = (dstBefore != 0) && (dstAfter == 0); + UBool stdToDst = (dstBefore == 0) && (dstAfter != 0); + + if (offsetAfter - offsetBefore >= 0) { + // Positive transition, which makes a non-existing local time range + if (((NonExistingTimeOpt & kStdDstMask) == kStandard && dstToStd) + || ((NonExistingTimeOpt & kStdDstMask) == kDaylight && stdToDst)) { + delta = offsetBefore; + } else if (((NonExistingTimeOpt & kStdDstMask) == kStandard && stdToDst) + || ((NonExistingTimeOpt & kStdDstMask) == kDaylight && dstToStd)) { + delta = offsetAfter; + } else if ((NonExistingTimeOpt & kFormerLatterMask) == kLatter) { + delta = offsetBefore; + } else { + // Interprets the time with rule before the transition, + // default for non-existing time range + delta = offsetAfter; + } + } else { + // Negative transition, which makes a duplicated local time range + if (((DuplicatedTimeOpt & kStdDstMask) == kStandard && dstToStd) + || ((DuplicatedTimeOpt & kStdDstMask) == kDaylight && stdToDst)) { + delta = offsetAfter; + } else if (((DuplicatedTimeOpt & kStdDstMask) == kStandard && stdToDst) + || ((DuplicatedTimeOpt & kStdDstMask) == kDaylight && dstToStd)) { + delta = offsetBefore; + } else if ((DuplicatedTimeOpt & kFormerLatterMask) == kFormer) { + delta = offsetBefore; + } else { + // Interprets the time with rule after the transition, + // default for duplicated local time range + delta = offsetAfter; + } + } + return delta; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof + diff --git a/intl/icu/source/i18n/regexcmp.cpp b/intl/icu/source/i18n/regexcmp.cpp new file mode 100644 index 0000000000..0a0d095ca4 --- /dev/null +++ b/intl/icu/source/i18n/regexcmp.cpp @@ -0,0 +1,4675 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// file: regexcmp.cpp +// +// Copyright (C) 2002-2016 International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains the ICU regular expression compiler, which is responsible +// for processing a regular expression pattern into the compiled form that +// is used by the match finding engine. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + +#include "unicode/ustring.h" +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/uchriter.h" +#include "unicode/parsepos.h" +#include "unicode/parseerr.h" +#include "unicode/regex.h" +#include "unicode/utf.h" +#include "unicode/utf16.h" +#include "patternprops.h" +#include "putilimp.h" +#include "cmemory.h" +#include "cstr.h" +#include "cstring.h" +#include "uvectr32.h" +#include "uvectr64.h" +#include "uassert.h" +#include "uinvchar.h" + +#include "regeximp.h" +#include "regexcst.h" // Contains state table for the regex pattern parser. + // generated by a Perl script. +#include "regexcmp.h" +#include "regexst.h" +#include "regextxt.h" + + + +U_NAMESPACE_BEGIN + + +//------------------------------------------------------------------------------ +// +// Constructor. +// +//------------------------------------------------------------------------------ +RegexCompile::RegexCompile(RegexPattern *rxp, UErrorCode &status) : + fParenStack(status), fSetStack(uprv_deleteUObject, nullptr, status), fSetOpStack(status) +{ + // Lazy init of all shared global sets (needed for init()'s empty text) + RegexStaticSets::initGlobals(&status); + + fStatus = &status; + + fRXPat = rxp; + fScanIndex = 0; + fLastChar = -1; + fPeekChar = -1; + fLineNum = 1; + fCharNum = 0; + fQuoteMode = false; + fInBackslashQuote = false; + fModeFlags = fRXPat->fFlags | 0x80000000; + fEOLComments = true; + + fMatchOpenParen = -1; + fMatchCloseParen = -1; + fCaptureName = nullptr; + fLastSetLiteral = U_SENTINEL; + + if (U_SUCCESS(status) && U_FAILURE(rxp->fDeferredStatus)) { + status = rxp->fDeferredStatus; + } +} + +static const char16_t chAmp = 0x26; // '&' +static const char16_t chDash = 0x2d; // '-' + + +//------------------------------------------------------------------------------ +// +// Destructor +// +//------------------------------------------------------------------------------ +RegexCompile::~RegexCompile() { + delete fCaptureName; // Normally will be nullptr, but can exist if pattern + // compilation stops with a syntax error. +} + +static inline void addCategory(UnicodeSet *set, int32_t value, UErrorCode& ec) { + set->addAll(UnicodeSet().applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, value, ec)); +} + +//------------------------------------------------------------------------------ +// +// Compile regex pattern. The state machine for rexexp pattern parsing is here. +// The state tables are hand-written in the file regexcst.txt, +// and converted to the form used here by a perl +// script regexcst.pl +// +//------------------------------------------------------------------------------ +void RegexCompile::compile( + const UnicodeString &pat, // Source pat to be compiled. + UParseError &pp, // Error position info + UErrorCode &e) // Error Code +{ + fRXPat->fPatternString = new UnicodeString(pat); + UText patternText = UTEXT_INITIALIZER; + utext_openConstUnicodeString(&patternText, fRXPat->fPatternString, &e); + + if (U_SUCCESS(e)) { + compile(&patternText, pp, e); + utext_close(&patternText); + } +} + +// +// compile, UText mode +// All the work is actually done here. +// +void RegexCompile::compile( + UText *pat, // Source pat to be compiled. + UParseError &pp, // Error position info + UErrorCode &e) // Error Code +{ + fStatus = &e; + fParseErr = &pp; + fStackPtr = 0; + fStack[fStackPtr] = 0; + + if (U_FAILURE(*fStatus)) { + return; + } + + // There should be no pattern stuff in the RegexPattern object. They can not be reused. + U_ASSERT(fRXPat->fPattern == nullptr || utext_nativeLength(fRXPat->fPattern) == 0); + + // Prepare the RegexPattern object to receive the compiled pattern. + fRXPat->fPattern = utext_clone(fRXPat->fPattern, pat, false, true, fStatus); + if (U_FAILURE(*fStatus)) { + return; + } + + // Initialize the pattern scanning state machine + fPatternLength = utext_nativeLength(pat); + uint16_t state = 1; + const RegexTableEl *tableEl; + + // UREGEX_LITERAL force entire pattern to be treated as a literal string. + if (fModeFlags & UREGEX_LITERAL) { + fQuoteMode = true; + } + + nextChar(fC); // Fetch the first char from the pattern string. + + // + // Main loop for the regex pattern parsing state machine. + // Runs once per state transition. + // Each time through optionally performs, depending on the state table, + // - an advance to the the next pattern char + // - an action to be performed. + // - pushing or popping a state to/from the local state return stack. + // file regexcst.txt is the source for the state table. The logic behind + // recongizing the pattern syntax is there, not here. + // + for (;;) { + // Bail out if anything has gone wrong. + // Regex pattern parsing stops on the first error encountered. + if (U_FAILURE(*fStatus)) { + break; + } + + U_ASSERT(state != 0); + + // Find the state table element that matches the input char from the pattern, or the + // class of the input character. Start with the first table row for this + // state, then linearly scan forward until we find a row that matches the + // character. The last row for each state always matches all characters, so + // the search will stop there, if not before. + // + tableEl = &gRuleParseStateTable[state]; + REGEX_SCAN_DEBUG_PRINTF(("char, line, col = (\'%c\', %d, %d) state=%s ", + fC.fChar, fLineNum, fCharNum, RegexStateNames[state])); + + for (;;) { // loop through table rows belonging to this state, looking for one + // that matches the current input char. + REGEX_SCAN_DEBUG_PRINTF((".")); + if (tableEl->fCharClass < 127 && fC.fQuoted == false && tableEl->fCharClass == fC.fChar) { + // Table row specified an individual character, not a set, and + // the input character is not quoted, and + // the input character matched it. + break; + } + if (tableEl->fCharClass == 255) { + // Table row specified default, match anything character class. + break; + } + if (tableEl->fCharClass == 254 && fC.fQuoted) { + // Table row specified "quoted" and the char was quoted. + break; + } + if (tableEl->fCharClass == 253 && fC.fChar == (UChar32)-1) { + // Table row specified eof and we hit eof on the input. + break; + } + + if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && + fC.fQuoted == false && // char is not escaped && + fC.fChar != (UChar32)-1) { // char is not EOF + U_ASSERT(tableEl->fCharClass <= 137); + if (RegexStaticSets::gStaticSets->fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) { + // Table row specified a character class, or set of characters, + // and the current char matches it. + break; + } + } + + // No match on this row, advance to the next row for this state, + tableEl++; + } + REGEX_SCAN_DEBUG_PRINTF(("\n")); + + // + // We've found the row of the state table that matches the current input + // character from the rules string. + // Perform any action specified by this row in the state table. + if (doParseActions(tableEl->fAction) == false) { + // Break out of the state machine loop if the + // the action signalled some kind of error, or + // the action was to exit, occurs on normal end-of-rules-input. + break; + } + + if (tableEl->fPushState != 0) { + fStackPtr++; + if (fStackPtr >= kStackSize) { + error(U_REGEX_INTERNAL_ERROR); + REGEX_SCAN_DEBUG_PRINTF(("RegexCompile::parse() - state stack overflow.\n")); + fStackPtr--; + } + fStack[fStackPtr] = tableEl->fPushState; + } + + // + // NextChar. This is where characters are actually fetched from the pattern. + // Happens under control of the 'n' tag in the state table. + // + if (tableEl->fNextChar) { + nextChar(fC); + } + + // Get the next state from the table entry, or from the + // state stack if the next state was specified as "pop". + if (tableEl->fNextState != 255) { + state = tableEl->fNextState; + } else { + state = fStack[fStackPtr]; + fStackPtr--; + if (fStackPtr < 0) { + // state stack underflow + // This will occur if the user pattern has mis-matched parentheses, + // with extra close parens. + // + fStackPtr++; + error(U_REGEX_MISMATCHED_PAREN); + } + } + + } + + if (U_FAILURE(*fStatus)) { + // Bail out if the pattern had errors. + return; + } + + // + // The pattern has now been read and processed, and the compiled code generated. + // + + // + // The pattern's fFrameSize so far has accumulated the requirements for + // storage for capture parentheses, counters, etc. that are encountered + // in the pattern. Add space for the two variables that are always + // present in the saved state: the input string position (int64_t) and + // the position in the compiled pattern. + // + allocateStackData(RESTACKFRAME_HDRCOUNT); + + // + // Optimization pass 1: NOPs, back-references, and case-folding + // + stripNOPs(); + + // + // Get bounds for the minimum and maximum length of a string that this + // pattern can match. Used to avoid looking for matches in strings that + // are too short. + // + fRXPat->fMinMatchLen = minMatchLength(3, fRXPat->fCompiledPat->size()-1); + + // + // Optimization pass 2: match start type + // + matchStartType(); + + // + // Set up fast latin-1 range sets + // + int32_t numSets = fRXPat->fSets->size(); + fRXPat->fSets8 = new Regex8BitSet[numSets]; + // Null pointer check. + if (fRXPat->fSets8 == nullptr) { + e = *fStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + int32_t i; + for (i=0; ifSets->elementAt(i); + fRXPat->fSets8[i].init(s); + } + +} + + + + + +//------------------------------------------------------------------------------ +// +// doParseAction Do some action during regex pattern parsing. +// Called by the parse state machine. +// +// Generation of the match engine PCode happens here, or +// in functions called from the parse actions defined here. +// +// +//------------------------------------------------------------------------------ +UBool RegexCompile::doParseActions(int32_t action) +{ + UBool returnVal = true; + + switch ((Regex_PatternParseAction)action) { + + case doPatStart: + // Start of pattern compiles to: + //0 SAVE 2 Fall back to position of FAIL + //1 jmp 3 + //2 FAIL Stop if we ever reach here. + //3 NOP Dummy, so start of pattern looks the same as + // the start of an ( grouping. + //4 NOP Resreved, will be replaced by a save if there are + // OR | operators at the top level + appendOp(URX_STATE_SAVE, 2); + appendOp(URX_JMP, 3); + appendOp(URX_FAIL, 0); + + // Standard open nonCapture paren action emits the two NOPs and + // sets up the paren stack frame. + doParseActions(doOpenNonCaptureParen); + break; + + case doPatFinish: + // We've scanned to the end of the pattern + // The end of pattern compiles to: + // URX_END + // which will stop the runtime match engine. + // Encountering end of pattern also behaves like a close paren, + // and forces fixups of the State Save at the beginning of the compiled pattern + // and of any OR operations at the top level. + // + handleCloseParen(); + if (fParenStack.size() > 0) { + // Missing close paren in pattern. + error(U_REGEX_MISMATCHED_PAREN); + } + + // add the END operation to the compiled pattern. + appendOp(URX_END, 0); + + // Terminate the pattern compilation state machine. + returnVal = false; + break; + + + + case doOrOperator: + // Scanning a '|', as in (A|B) + { + // Generate code for any pending literals preceding the '|' + fixLiterals(false); + + // Insert a SAVE operation at the start of the pattern section preceding + // this OR at this level. This SAVE will branch the match forward + // to the right hand side of the OR in the event that the left hand + // side fails to match and backtracks. Locate the position for the + // save from the location on the top of the parentheses stack. + int32_t savePosition = fParenStack.popi(); + int32_t op = (int32_t)fRXPat->fCompiledPat->elementAti(savePosition); + U_ASSERT(URX_TYPE(op) == URX_NOP); // original contents of reserved location + op = buildOp(URX_STATE_SAVE, fRXPat->fCompiledPat->size()+1); + fRXPat->fCompiledPat->setElementAt(op, savePosition); + + // Append an JMP operation into the compiled pattern. The operand for + // the JMP will eventually be the location following the ')' for the + // group. This will be patched in later, when the ')' is encountered. + appendOp(URX_JMP, 0); + + // Push the position of the newly added JMP op onto the parentheses stack. + // This registers if for fixup when this block's close paren is encountered. + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); + + // Append a NOP to the compiled pattern. This is the slot reserved + // for a SAVE in the event that there is yet another '|' following + // this one. + appendOp(URX_NOP, 0); + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); + } + break; + + + case doBeginNamedCapture: + // Scanning (?append(fC.fChar); + break; + + case doBadNamedCapture: + error(U_REGEX_INVALID_CAPTURE_GROUP_NAME); + break; + + case doOpenCaptureParen: + // Open Capturing Paren, possibly named. + // Compile to a + // - NOP, which later may be replaced by a save-state if the + // parenthesized group gets a * quantifier, followed by + // - START_CAPTURE n where n is stack frame offset to the capture group variables. + // - NOP, which may later be replaced by a save-state if there + // is an '|' alternation within the parens. + // + // Each capture group gets three slots in the save stack frame: + // 0: Capture Group start position (in input string being matched.) + // 1: Capture Group end position. + // 2: Start of Match-in-progress. + // The first two locations are for a completed capture group, and are + // referred to by back references and the like. + // The third location stores the capture start position when an START_CAPTURE is + // encountered. This will be promoted to a completed capture when (and if) the corresponding + // END_CAPTURE is encountered. + { + fixLiterals(); + appendOp(URX_NOP, 0); + int32_t varsLoc = allocateStackData(3); // Reserve three slots in match stack frame. + appendOp(URX_START_CAPTURE, varsLoc); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the two NOPs. Depending on what follows in the pattern, the + // NOPs may be changed to SAVE_STATE or JMP ops, with a target + // address of the end of the parenthesized group. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(capturing, *fStatus); // Frame type. + fParenStack.push(fRXPat->fCompiledPat->size()-3, *fStatus); // The first NOP location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP loc + + // Save the mapping from group number to stack frame variable position. + fRXPat->fGroupMap->addElement(varsLoc, *fStatus); + + // If this is a named capture group, add the name->group number mapping. + if (fCaptureName != nullptr) { + if (!fRXPat->initNamedCaptureMap()) { + if (U_SUCCESS(*fStatus)) { + error(fRXPat->fDeferredStatus); + } + break; + } + int32_t groupNumber = fRXPat->fGroupMap->size(); + int32_t previousMapping = uhash_puti(fRXPat->fNamedCaptureMap, fCaptureName, groupNumber, fStatus); + fCaptureName = nullptr; // hash table takes ownership of the name (key) string. + if (previousMapping > 0 && U_SUCCESS(*fStatus)) { + error(U_REGEX_INVALID_CAPTURE_GROUP_NAME); + } + } + } + break; + + case doOpenNonCaptureParen: + // Open non-caputuring (grouping only) Paren. + // Compile to a + // - NOP, which later may be replaced by a save-state if the + // parenthesized group gets a * quantifier, followed by + // - NOP, which may later be replaced by a save-state if there + // is an '|' alternation within the parens. + { + fixLiterals(); + appendOp(URX_NOP, 0); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the two NOPs. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(plain, *fStatus); // Begin a new frame. + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP loc + } + break; + + + case doOpenAtomicParen: + // Open Atomic Paren. (?> + // Compile to a + // - NOP, which later may be replaced if the parenthesized group + // has a quantifier, followed by + // - STO_SP save state stack position, so it can be restored at the ")" + // - NOP, which may later be replaced by a save-state if there + // is an '|' alternation within the parens. + { + fixLiterals(); + appendOp(URX_NOP, 0); + int32_t varLoc = allocateData(1); // Reserve a data location for saving the state stack ptr. + appendOp(URX_STO_SP, varLoc); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the two NOPs. Depending on what follows in the pattern, the + // NOPs may be changed to SAVE_STATE or JMP ops, with a target + // address of the end of the parenthesized group. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(atomic, *fStatus); // Frame type. + fParenStack.push(fRXPat->fCompiledPat->size()-3, *fStatus); // The first NOP + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP + } + break; + + + case doOpenLookAhead: + // Positive Look-ahead (?= stuff ) + // + // Note: Addition of transparent input regions, with the need to + // restore the original regions when failing out of a lookahead + // block, complicated this sequence. Some combined opcodes + // might make sense - or might not, lookahead aren't that common. + // + // Caution: min match length optimization knows about this + // sequence; don't change without making updates there too. + // + // Compiles to + // 1 LA_START dataLoc Saves SP, Input Pos, Active input region. + // 2. STATE_SAVE 4 on failure of lookahead, goto 4 + // 3 JMP 6 continue ... + // + // 4. LA_END Look Ahead failed. Restore regions. + // 5. BACKTRACK and back track again. + // + // 6. NOP reserved for use by quantifiers on the block. + // Look-ahead can't have quantifiers, but paren stack + // compile time conventions require the slot anyhow. + // 7. NOP may be replaced if there is are '|' ops in the block. + // 8. code for parenthesized stuff. + // 9. LA_END + // + // Four data slots are reserved, for saving state on entry to the look-around + // 0: stack pointer on entry. + // 1: input position on entry. + // 2: fActiveStart, the active bounds start on entry. + // 3: fActiveLimit, the active bounds limit on entry. + { + fixLiterals(); + int32_t dataLoc = allocateData(4); + appendOp(URX_LA_START, dataLoc); + appendOp(URX_STATE_SAVE, fRXPat->fCompiledPat->size()+ 2); + appendOp(URX_JMP, fRXPat->fCompiledPat->size()+ 3); + appendOp(URX_LA_END, dataLoc); + appendOp(URX_BACKTRACK, 0); + appendOp(URX_NOP, 0); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the NOPs. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(lookAhead, *fStatus); // Frame type. + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP location + } + break; + + case doOpenLookAheadNeg: + // Negated Lookahead. (?! stuff ) + // Compiles to + // 1. LA_START dataloc + // 2. SAVE_STATE 7 // Fail within look-ahead block restores to this state, + // // which continues with the match. + // 3. NOP // Std. Open Paren sequence, for possible '|' + // 4. code for parenthesized stuff. + // 5. LA_END // Cut back stack, remove saved state from step 2. + // 6. BACKTRACK // code in block succeeded, so neg. lookahead fails. + // 7. END_LA // Restore match region, in case look-ahead was using + // an alternate (transparent) region. + // Four data slots are reserved, for saving state on entry to the look-around + // 0: stack pointer on entry. + // 1: input position on entry. + // 2: fActiveStart, the active bounds start on entry. + // 3: fActiveLimit, the active bounds limit on entry. + { + fixLiterals(); + int32_t dataLoc = allocateData(4); + appendOp(URX_LA_START, dataLoc); + appendOp(URX_STATE_SAVE, 0); // dest address will be patched later. + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the StateSave and NOP. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(negLookAhead, *fStatus); // Frame type + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The STATE_SAVE location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP location + + // Instructions #5 - #7 will be added when the ')' is encountered. + } + break; + + case doOpenLookBehind: + { + // Compile a (?<= look-behind open paren. + // + // Compiles to + // 0 URX_LB_START dataLoc + // 1 URX_LB_CONT dataLoc + // 2 MinMatchLen + // 3 MaxMatchLen + // 4 URX_NOP Standard '(' boilerplate. + // 5 URX_NOP Reserved slot for use with '|' ops within (block). + // 6 + // 7 URX_LB_END dataLoc # Check match len, restore input len + // 8 URX_LA_END dataLoc # Restore stack, input pos + // + // Allocate a block of matcher data, to contain (when running a match) + // 0: Stack ptr on entry + // 1: Input Index on entry + // 2: fActiveStart, the active bounds start on entry. + // 3: fActiveLimit, the active bounds limit on entry. + // 4: Start index of match current match attempt. + // The first four items must match the layout of data for LA_START / LA_END + + // Generate match code for any pending literals. + fixLiterals(); + + // Allocate data space + int32_t dataLoc = allocateData(5); + + // Emit URX_LB_START + appendOp(URX_LB_START, dataLoc); + + // Emit URX_LB_CONT + appendOp(URX_LB_CONT, dataLoc); + appendOp(URX_RESERVED_OP, 0); // MinMatchLength. To be filled later. + appendOp(URX_RESERVED_OP, 0); // MaxMatchLength. To be filled later. + + // Emit the NOPs + appendOp(URX_NOP, 0); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the URX_LB_CONT and the NOP. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(lookBehind, *fStatus); // Frame type + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The 2nd NOP location + + // The final two instructions will be added when the ')' is encountered. + } + + break; + + case doOpenLookBehindNeg: + { + // Compile a (? + // 8 URX_LBN_END dataLoc # Check match len, cause a FAIL + // 9 ... + // + // Allocate a block of matcher data, to contain (when running a match) + // 0: Stack ptr on entry + // 1: Input Index on entry + // 2: fActiveStart, the active bounds start on entry. + // 3: fActiveLimit, the active bounds limit on entry. + // 4: Start index of match current match attempt. + // The first four items must match the layout of data for LA_START / LA_END + + // Generate match code for any pending literals. + fixLiterals(); + + // Allocate data space + int32_t dataLoc = allocateData(5); + + // Emit URX_LB_START + appendOp(URX_LB_START, dataLoc); + + // Emit URX_LBN_CONT + appendOp(URX_LBN_CONT, dataLoc); + appendOp(URX_RESERVED_OP, 0); // MinMatchLength. To be filled later. + appendOp(URX_RESERVED_OP, 0); // MaxMatchLength. To be filled later. + appendOp(URX_RESERVED_OP, 0); // Continue Loc. To be filled later. + + // Emit the NOPs + appendOp(URX_NOP, 0); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the URX_LB_CONT and the NOP. + fParenStack.push(fModeFlags, *fStatus); // Match mode state + fParenStack.push(lookBehindN, *fStatus); // Frame type + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The 2nd NOP location + + // The final two instructions will be added when the ')' is encountered. + } + break; + + case doConditionalExpr: + // Conditionals such as (?(1)a:b) + case doPerlInline: + // Perl inline-conditionals. (?{perl code}a|b) We're not perl, no way to do them. + error(U_REGEX_UNIMPLEMENTED); + break; + + + case doCloseParen: + handleCloseParen(); + if (fParenStack.size() <= 0) { + // Extra close paren, or missing open paren. + error(U_REGEX_MISMATCHED_PAREN); + } + break; + + case doNOP: + break; + + + case doBadOpenParenType: + case doRuleError: + error(U_REGEX_RULE_SYNTAX); + break; + + + case doMismatchedParenErr: + error(U_REGEX_MISMATCHED_PAREN); + break; + + case doPlus: + // Normal '+' compiles to + // 1. stuff to be repeated (already built) + // 2. jmp-sav 1 + // 3. ... + // + // Or, if the item to be repeated can match a zero length string, + // 1. STO_INP_LOC data-loc + // 2. body of stuff to be repeated + // 3. JMP_SAV_X 2 + // 4. ... + + // + // Or, if the item to be repeated is simple + // 1. Item to be repeated. + // 2. LOOP_SR_I set number (assuming repeated item is a set ref) + // 3. LOOP_C stack location + { + int32_t topLoc = blockTopLoc(false); // location of item #1 + int32_t frameLoc; + + // Check for simple constructs, which may get special optimized code. + if (topLoc == fRXPat->fCompiledPat->size() - 1) { + int32_t repeatedOp = (int32_t)fRXPat->fCompiledPat->elementAti(topLoc); + + if (URX_TYPE(repeatedOp) == URX_SETREF) { + // Emit optimized code for [char set]+ + appendOp(URX_LOOP_SR_I, URX_VAL(repeatedOp)); + frameLoc = allocateStackData(1); + appendOp(URX_LOOP_C, frameLoc); + break; + } + + if (URX_TYPE(repeatedOp) == URX_DOTANY || + URX_TYPE(repeatedOp) == URX_DOTANY_ALL || + URX_TYPE(repeatedOp) == URX_DOTANY_UNIX) { + // Emit Optimized code for .+ operations. + int32_t loopOpI = buildOp(URX_LOOP_DOT_I, 0); + if (URX_TYPE(repeatedOp) == URX_DOTANY_ALL) { + // URX_LOOP_DOT_I operand is a flag indicating ". matches any" mode. + loopOpI |= 1; + } + if (fModeFlags & UREGEX_UNIX_LINES) { + loopOpI |= 2; + } + appendOp(loopOpI); + frameLoc = allocateStackData(1); + appendOp(URX_LOOP_C, frameLoc); + break; + } + + } + + // General case. + + // Check for minimum match length of zero, which requires + // extra loop-breaking code. + if (minMatchLength(topLoc, fRXPat->fCompiledPat->size()-1) == 0) { + // Zero length match is possible. + // Emit the code sequence that can handle it. + insertOp(topLoc); + frameLoc = allocateStackData(1); + + int32_t op = buildOp(URX_STO_INP_LOC, frameLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc); + + appendOp(URX_JMP_SAV_X, topLoc+1); + } else { + // Simpler code when the repeated body must match something non-empty + appendOp(URX_JMP_SAV, topLoc); + } + } + break; + + case doNGPlus: + // Non-greedy '+?' compiles to + // 1. stuff to be repeated (already built) + // 2. state-save 1 + // 3. ... + { + int32_t topLoc = blockTopLoc(false); + appendOp(URX_STATE_SAVE, topLoc); + } + break; + + + case doOpt: + // Normal (greedy) ? quantifier. + // Compiles to + // 1. state save 3 + // 2. body of optional block + // 3. ... + // Insert the state save into the compiled pattern, and we're done. + { + int32_t saveStateLoc = blockTopLoc(true); + int32_t saveStateOp = buildOp(URX_STATE_SAVE, fRXPat->fCompiledPat->size()); + fRXPat->fCompiledPat->setElementAt(saveStateOp, saveStateLoc); + } + break; + + case doNGOpt: + // Non-greedy ?? quantifier + // compiles to + // 1. jmp 4 + // 2. body of optional block + // 3 jmp 5 + // 4. state save 2 + // 5 ... + // This code is less than ideal, with two jmps instead of one, because we can only + // insert one instruction at the top of the block being iterated. + { + int32_t jmp1_loc = blockTopLoc(true); + int32_t jmp2_loc = fRXPat->fCompiledPat->size(); + + int32_t jmp1_op = buildOp(URX_JMP, jmp2_loc+1); + fRXPat->fCompiledPat->setElementAt(jmp1_op, jmp1_loc); + + appendOp(URX_JMP, jmp2_loc+2); + + appendOp(URX_STATE_SAVE, jmp1_loc+1); + } + break; + + + case doStar: + // Normal (greedy) * quantifier. + // Compiles to + // 1. STATE_SAVE 4 + // 2. body of stuff being iterated over + // 3. JMP_SAV 2 + // 4. ... + // + // Or, if the body is a simple [Set], + // 1. LOOP_SR_I set number + // 2. LOOP_C stack location + // ... + // + // Or if this is a .* + // 1. LOOP_DOT_I (. matches all mode flag) + // 2. LOOP_C stack location + // + // Or, if the body can match a zero-length string, to inhibit infinite loops, + // 1. STATE_SAVE 5 + // 2. STO_INP_LOC data-loc + // 3. body of stuff + // 4. JMP_SAV_X 2 + // 5. ... + { + // location of item #1, the STATE_SAVE + int32_t topLoc = blockTopLoc(false); + int32_t dataLoc = -1; + + // Check for simple *, where the construct being repeated + // compiled to single opcode, and might be optimizable. + if (topLoc == fRXPat->fCompiledPat->size() - 1) { + int32_t repeatedOp = (int32_t)fRXPat->fCompiledPat->elementAti(topLoc); + + if (URX_TYPE(repeatedOp) == URX_SETREF) { + // Emit optimized code for a [char set]* + int32_t loopOpI = buildOp(URX_LOOP_SR_I, URX_VAL(repeatedOp)); + fRXPat->fCompiledPat->setElementAt(loopOpI, topLoc); + dataLoc = allocateStackData(1); + appendOp(URX_LOOP_C, dataLoc); + break; + } + + if (URX_TYPE(repeatedOp) == URX_DOTANY || + URX_TYPE(repeatedOp) == URX_DOTANY_ALL || + URX_TYPE(repeatedOp) == URX_DOTANY_UNIX) { + // Emit Optimized code for .* operations. + int32_t loopOpI = buildOp(URX_LOOP_DOT_I, 0); + if (URX_TYPE(repeatedOp) == URX_DOTANY_ALL) { + // URX_LOOP_DOT_I operand is a flag indicating . matches any mode. + loopOpI |= 1; + } + if ((fModeFlags & UREGEX_UNIX_LINES) != 0) { + loopOpI |= 2; + } + fRXPat->fCompiledPat->setElementAt(loopOpI, topLoc); + dataLoc = allocateStackData(1); + appendOp(URX_LOOP_C, dataLoc); + break; + } + } + + // Emit general case code for this * + // The optimizations did not apply. + + int32_t saveStateLoc = blockTopLoc(true); + int32_t jmpOp = buildOp(URX_JMP_SAV, saveStateLoc+1); + + // Check for minimum match length of zero, which requires + // extra loop-breaking code. + if (minMatchLength(saveStateLoc, fRXPat->fCompiledPat->size()-1) == 0) { + insertOp(saveStateLoc); + dataLoc = allocateStackData(1); + + int32_t op = buildOp(URX_STO_INP_LOC, dataLoc); + fRXPat->fCompiledPat->setElementAt(op, saveStateLoc+1); + jmpOp = buildOp(URX_JMP_SAV_X, saveStateLoc+2); + } + + // Locate the position in the compiled pattern where the match will continue + // after completing the *. (4 or 5 in the comment above) + int32_t continueLoc = fRXPat->fCompiledPat->size()+1; + + // Put together the save state op and store it into the compiled code. + int32_t saveStateOp = buildOp(URX_STATE_SAVE, continueLoc); + fRXPat->fCompiledPat->setElementAt(saveStateOp, saveStateLoc); + + // Append the URX_JMP_SAV or URX_JMPX operation to the compiled pattern. + appendOp(jmpOp); + } + break; + + case doNGStar: + // Non-greedy *? quantifier + // compiles to + // 1. JMP 3 + // 2. body of stuff being iterated over + // 3. STATE_SAVE 2 + // 4 ... + { + int32_t jmpLoc = blockTopLoc(true); // loc 1. + int32_t saveLoc = fRXPat->fCompiledPat->size(); // loc 3. + int32_t jmpOp = buildOp(URX_JMP, saveLoc); + fRXPat->fCompiledPat->setElementAt(jmpOp, jmpLoc); + appendOp(URX_STATE_SAVE, jmpLoc+1); + } + break; + + + case doIntervalInit: + // The '{' opening an interval quantifier was just scanned. + // Init the counter variables that will accumulate the values as the digits + // are scanned. + fIntervalLow = 0; + fIntervalUpper = -1; + break; + + case doIntevalLowerDigit: + // Scanned a digit from the lower value of an {lower,upper} interval + { + int32_t digitValue = u_charDigitValue(fC.fChar); + U_ASSERT(digitValue >= 0); + int64_t val = (int64_t)fIntervalLow*10 + digitValue; + if (val > INT32_MAX) { + error(U_REGEX_NUMBER_TOO_BIG); + } else { + fIntervalLow = (int32_t)val; + } + } + break; + + case doIntervalUpperDigit: + // Scanned a digit from the upper value of an {lower,upper} interval + { + if (fIntervalUpper < 0) { + fIntervalUpper = 0; + } + int32_t digitValue = u_charDigitValue(fC.fChar); + U_ASSERT(digitValue >= 0); + int64_t val = (int64_t)fIntervalUpper*10 + digitValue; + if (val > INT32_MAX) { + error(U_REGEX_NUMBER_TOO_BIG); + } else { + fIntervalUpper = (int32_t)val; + } + } + break; + + case doIntervalSame: + // Scanned a single value interval like {27}. Upper = Lower. + fIntervalUpper = fIntervalLow; + break; + + case doInterval: + // Finished scanning a normal {lower,upper} interval. Generate the code for it. + if (compileInlineInterval() == false) { + compileInterval(URX_CTR_INIT, URX_CTR_LOOP); + } + break; + + case doPossessiveInterval: + // Finished scanning a Possessive {lower,upper}+ interval. Generate the code for it. + { + // Remember the loc for the top of the block being looped over. + // (Can not reserve a slot in the compiled pattern at this time, because + // compileInterval needs to reserve also, and blockTopLoc can only reserve + // once per block.) + int32_t topLoc = blockTopLoc(false); + + // Produce normal looping code. + compileInterval(URX_CTR_INIT, URX_CTR_LOOP); + + // Surround the just-emitted normal looping code with a STO_SP ... LD_SP + // just as if the loop was inclosed in atomic parentheses. + + // First the STO_SP before the start of the loop + insertOp(topLoc); + + int32_t varLoc = allocateData(1); // Reserve a data location for saving the + int32_t op = buildOp(URX_STO_SP, varLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc); + + int32_t loopOp = (int32_t)fRXPat->fCompiledPat->popi(); + U_ASSERT(URX_TYPE(loopOp) == URX_CTR_LOOP && URX_VAL(loopOp) == topLoc); + loopOp++; // point LoopOp after the just-inserted STO_SP + fRXPat->fCompiledPat->push(loopOp, *fStatus); + + // Then the LD_SP after the end of the loop + appendOp(URX_LD_SP, varLoc); + } + + break; + + case doNGInterval: + // Finished scanning a non-greedy {lower,upper}? interval. Generate the code for it. + compileInterval(URX_CTR_INIT_NG, URX_CTR_LOOP_NG); + break; + + case doIntervalError: + error(U_REGEX_BAD_INTERVAL); + break; + + case doLiteralChar: + // We've just scanned a "normal" character from the pattern, + literalChar(fC.fChar); + break; + + + case doEscapedLiteralChar: + // We've just scanned an backslashed escaped character with no + // special meaning. It represents itself. + if ((fModeFlags & UREGEX_ERROR_ON_UNKNOWN_ESCAPES) != 0 && + ((fC.fChar >= 0x41 && fC.fChar<= 0x5A) || // in [A-Z] + (fC.fChar >= 0x61 && fC.fChar <= 0x7a))) { // in [a-z] + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + } + literalChar(fC.fChar); + break; + + + case doDotAny: + // scanned a ".", match any single character. + { + fixLiterals(false); + if (fModeFlags & UREGEX_DOTALL) { + appendOp(URX_DOTANY_ALL, 0); + } else if (fModeFlags & UREGEX_UNIX_LINES) { + appendOp(URX_DOTANY_UNIX, 0); + } else { + appendOp(URX_DOTANY, 0); + } + } + break; + + case doCaret: + { + fixLiterals(false); + if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) { + appendOp(URX_CARET, 0); + } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) { + appendOp(URX_CARET_M, 0); + } else if ((fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) { + appendOp(URX_CARET, 0); // Only testing true start of input. + } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) { + appendOp(URX_CARET_M_UNIX, 0); + } + } + break; + + case doDollar: + { + fixLiterals(false); + if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) { + appendOp(URX_DOLLAR, 0); + } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) { + appendOp(URX_DOLLAR_M, 0); + } else if ((fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) { + appendOp(URX_DOLLAR_D, 0); + } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) { + appendOp(URX_DOLLAR_MD, 0); + } + } + break; + + case doBackslashA: + fixLiterals(false); + appendOp(URX_CARET, 0); + break; + + case doBackslashB: + { + #if UCONFIG_NO_BREAK_ITERATION==1 + if (fModeFlags & UREGEX_UWORD) { + error(U_UNSUPPORTED_ERROR); + } + #endif + fixLiterals(false); + int32_t op = (fModeFlags & UREGEX_UWORD)? URX_BACKSLASH_BU : URX_BACKSLASH_B; + appendOp(op, 1); + } + break; + + case doBackslashb: + { + #if UCONFIG_NO_BREAK_ITERATION==1 + if (fModeFlags & UREGEX_UWORD) { + error(U_UNSUPPORTED_ERROR); + } + #endif + fixLiterals(false); + int32_t op = (fModeFlags & UREGEX_UWORD)? URX_BACKSLASH_BU : URX_BACKSLASH_B; + appendOp(op, 0); + } + break; + + case doBackslashD: + fixLiterals(false); + appendOp(URX_BACKSLASH_D, 1); + break; + + case doBackslashd: + fixLiterals(false); + appendOp(URX_BACKSLASH_D, 0); + break; + + case doBackslashG: + fixLiterals(false); + appendOp(URX_BACKSLASH_G, 0); + break; + + case doBackslashH: + fixLiterals(false); + appendOp(URX_BACKSLASH_H, 1); + break; + + case doBackslashh: + fixLiterals(false); + appendOp(URX_BACKSLASH_H, 0); + break; + + case doBackslashR: + fixLiterals(false); + appendOp(URX_BACKSLASH_R, 0); + break; + + case doBackslashS: + fixLiterals(false); + appendOp(URX_STAT_SETREF_N, URX_ISSPACE_SET); + break; + + case doBackslashs: + fixLiterals(false); + appendOp(URX_STATIC_SETREF, URX_ISSPACE_SET); + break; + + case doBackslashV: + fixLiterals(false); + appendOp(URX_BACKSLASH_V, 1); + break; + + case doBackslashv: + fixLiterals(false); + appendOp(URX_BACKSLASH_V, 0); + break; + + case doBackslashW: + fixLiterals(false); + appendOp(URX_STAT_SETREF_N, URX_ISWORD_SET); + break; + + case doBackslashw: + fixLiterals(false); + appendOp(URX_STATIC_SETREF, URX_ISWORD_SET); + break; + + case doBackslashX: + #if UCONFIG_NO_BREAK_ITERATION==1 + // Grapheme Cluster Boundary requires ICU break iteration. + error(U_UNSUPPORTED_ERROR); + #endif + fixLiterals(false); + appendOp(URX_BACKSLASH_X, 0); + break; + + case doBackslashZ: + fixLiterals(false); + appendOp(URX_DOLLAR, 0); + break; + + case doBackslashz: + fixLiterals(false); + appendOp(URX_BACKSLASH_Z, 0); + break; + + case doEscapeError: + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + break; + + case doExit: + fixLiterals(false); + returnVal = false; + break; + + case doProperty: + { + fixLiterals(false); + UnicodeSet *theSet = scanProp(); + compileSet(theSet); + } + break; + + case doNamedChar: + { + UChar32 c = scanNamedChar(); + literalChar(c); + } + break; + + + case doBackRef: + // BackReference. Somewhat unusual in that the front-end can not completely parse + // the regular expression, because the number of digits to be consumed + // depends on the number of capture groups that have been defined. So + // we have to do it here instead. + { + int32_t numCaptureGroups = fRXPat->fGroupMap->size(); + int32_t groupNum = 0; + UChar32 c = fC.fChar; + + for (;;) { + // Loop once per digit, for max allowed number of digits in a back reference. + int32_t digit = u_charDigitValue(c); + groupNum = groupNum * 10 + digit; + if (groupNum >= numCaptureGroups) { + break; + } + c = peekCharLL(); + if (RegexStaticSets::gStaticSets->fRuleDigitsAlias->contains(c) == false) { + break; + } + nextCharLL(); + } + + // Scan of the back reference in the source regexp is complete. Now generate + // the compiled code for it. + // Because capture groups can be forward-referenced by back-references, + // we fill the operand with the capture group number. At the end + // of compilation, it will be changed to the variable's location. + U_ASSERT(groupNum > 0); // Shouldn't happen. '\0' begins an octal escape sequence, + // and shouldn't enter this code path at all. + fixLiterals(false); + if (fModeFlags & UREGEX_CASE_INSENSITIVE) { + appendOp(URX_BACKREF_I, groupNum); + } else { + appendOp(URX_BACKREF, groupNum); + } + } + break; + + case doBeginNamedBackRef: + U_ASSERT(fCaptureName == nullptr); + fCaptureName = new UnicodeString; + if (fCaptureName == nullptr) { + error(U_MEMORY_ALLOCATION_ERROR); + } + break; + + case doContinueNamedBackRef: + fCaptureName->append(fC.fChar); + break; + + case doCompleteNamedBackRef: + { + int32_t groupNumber = + fRXPat->fNamedCaptureMap ? uhash_geti(fRXPat->fNamedCaptureMap, fCaptureName) : 0; + if (groupNumber == 0) { + // Group name has not been defined. + // Could be a forward reference. If we choose to support them at some + // future time, extra mechanism will be required at this point. + error(U_REGEX_INVALID_CAPTURE_GROUP_NAME); + } else { + // Given the number, handle identically to a \n numbered back reference. + // See comments above, under doBackRef + fixLiterals(false); + if (fModeFlags & UREGEX_CASE_INSENSITIVE) { + appendOp(URX_BACKREF_I, groupNumber); + } else { + appendOp(URX_BACKREF, groupNumber); + } + } + delete fCaptureName; + fCaptureName = nullptr; + break; + } + + case doPossessivePlus: + // Possessive ++ quantifier. + // Compiles to + // 1. STO_SP + // 2. body of stuff being iterated over + // 3. STATE_SAVE 5 + // 4. JMP 2 + // 5. LD_SP + // 6. ... + // + // Note: TODO: This is pretty inefficient. A mass of saved state is built up + // then unconditionally discarded. Perhaps introduce a new opcode. Ticket 6056 + // + { + // Emit the STO_SP + int32_t topLoc = blockTopLoc(true); + int32_t stoLoc = allocateData(1); // Reserve the data location for storing save stack ptr. + int32_t op = buildOp(URX_STO_SP, stoLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc); + + // Emit the STATE_SAVE + appendOp(URX_STATE_SAVE, fRXPat->fCompiledPat->size()+2); + + // Emit the JMP + appendOp(URX_JMP, topLoc+1); + + // Emit the LD_SP + appendOp(URX_LD_SP, stoLoc); + } + break; + + case doPossessiveStar: + // Possessive *+ quantifier. + // Compiles to + // 1. STO_SP loc + // 2. STATE_SAVE 5 + // 3. body of stuff being iterated over + // 4. JMP 2 + // 5. LD_SP loc + // 6 ... + // TODO: do something to cut back the state stack each time through the loop. + { + // Reserve two slots at the top of the block. + int32_t topLoc = blockTopLoc(true); + insertOp(topLoc); + + // emit STO_SP loc + int32_t stoLoc = allocateData(1); // Reserve the data location for storing save stack ptr. + int32_t op = buildOp(URX_STO_SP, stoLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc); + + // Emit the SAVE_STATE 5 + int32_t L7 = fRXPat->fCompiledPat->size()+1; + op = buildOp(URX_STATE_SAVE, L7); + fRXPat->fCompiledPat->setElementAt(op, topLoc+1); + + // Append the JMP operation. + appendOp(URX_JMP, topLoc+1); + + // Emit the LD_SP loc + appendOp(URX_LD_SP, stoLoc); + } + break; + + case doPossessiveOpt: + // Possessive ?+ quantifier. + // Compiles to + // 1. STO_SP loc + // 2. SAVE_STATE 5 + // 3. body of optional block + // 4. LD_SP loc + // 5. ... + // + { + // Reserve two slots at the top of the block. + int32_t topLoc = blockTopLoc(true); + insertOp(topLoc); + + // Emit the STO_SP + int32_t stoLoc = allocateData(1); // Reserve the data location for storing save stack ptr. + int32_t op = buildOp(URX_STO_SP, stoLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc); + + // Emit the SAVE_STATE + int32_t continueLoc = fRXPat->fCompiledPat->size()+1; + op = buildOp(URX_STATE_SAVE, continueLoc); + fRXPat->fCompiledPat->setElementAt(op, topLoc+1); + + // Emit the LD_SP + appendOp(URX_LD_SP, stoLoc); + } + break; + + + case doBeginMatchMode: + fNewModeFlags = fModeFlags; + fSetModeFlag = true; + break; + + case doMatchMode: // (?i) and similar + { + int32_t bit = 0; + switch (fC.fChar) { + case 0x69: /* 'i' */ bit = UREGEX_CASE_INSENSITIVE; break; + case 0x64: /* 'd' */ bit = UREGEX_UNIX_LINES; break; + case 0x6d: /* 'm' */ bit = UREGEX_MULTILINE; break; + case 0x73: /* 's' */ bit = UREGEX_DOTALL; break; + case 0x75: /* 'u' */ bit = 0; /* Unicode casing */ break; + case 0x77: /* 'w' */ bit = UREGEX_UWORD; break; + case 0x78: /* 'x' */ bit = UREGEX_COMMENTS; break; + case 0x2d: /* '-' */ fSetModeFlag = false; break; + default: + UPRV_UNREACHABLE_EXIT; // Should never happen. Other chars are filtered out + // by the scanner. + } + if (fSetModeFlag) { + fNewModeFlags |= bit; + } else { + fNewModeFlags &= ~bit; + } + } + break; + + case doSetMatchMode: + // Emit code to match any pending literals, using the not-yet changed match mode. + fixLiterals(); + + // We've got a (?i) or similar. The match mode is being changed, but + // the change is not scoped to a parenthesized block. + U_ASSERT(fNewModeFlags < 0); + fModeFlags = fNewModeFlags; + + break; + + + case doMatchModeParen: + // We've got a (?i: or similar. Begin a parenthesized block, save old + // mode flags so they can be restored at the close of the block. + // + // Compile to a + // - NOP, which later may be replaced by a save-state if the + // parenthesized group gets a * quantifier, followed by + // - NOP, which may later be replaced by a save-state if there + // is an '|' alternation within the parens. + { + fixLiterals(false); + appendOp(URX_NOP, 0); + appendOp(URX_NOP, 0); + + // On the Parentheses stack, start a new frame and add the positions + // of the two NOPs (a normal non-capturing () frame, except for the + // saving of the original mode flags.) + fParenStack.push(fModeFlags, *fStatus); + fParenStack.push(flags, *fStatus); // Frame Marker + fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP + fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP + + // Set the current mode flags to the new values. + U_ASSERT(fNewModeFlags < 0); + fModeFlags = fNewModeFlags; + } + break; + + case doBadModeFlag: + error(U_REGEX_INVALID_FLAG); + break; + + case doSuppressComments: + // We have just scanned a '(?'. We now need to prevent the character scanner from + // treating a '#' as a to-the-end-of-line comment. + // (This Perl compatibility just gets uglier and uglier to do...) + fEOLComments = false; + break; + + + case doSetAddAmp: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + set->add(chAmp); + } + break; + + case doSetAddDash: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + set->add(chDash); + } + break; + + case doSetBackslash_s: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + set->addAll(RegexStaticSets::gStaticSets->fPropSets[URX_ISSPACE_SET]); + break; + } + + case doSetBackslash_S: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet SSet; + SSet.addAll(RegexStaticSets::gStaticSets->fPropSets[URX_ISSPACE_SET]).complement(); + set->addAll(SSet); + break; + } + + case doSetBackslash_d: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + // TODO - make a static set, ticket 6058. + addCategory(set, U_GC_ND_MASK, *fStatus); + break; + } + + case doSetBackslash_D: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet digits; + // TODO - make a static set, ticket 6058. + digits.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ND_MASK, *fStatus); + digits.complement(); + set->addAll(digits); + break; + } + + case doSetBackslash_h: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet h; + h.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ZS_MASK, *fStatus); + h.add((UChar32)9); // Tab + set->addAll(h); + break; + } + + case doSetBackslash_H: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet h; + h.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ZS_MASK, *fStatus); + h.add((UChar32)9); // Tab + h.complement(); + set->addAll(h); + break; + } + + case doSetBackslash_v: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + set->add((UChar32)0x0a, (UChar32)0x0d); // add range + set->add((UChar32)0x85); + set->add((UChar32)0x2028, (UChar32)0x2029); + break; + } + + case doSetBackslash_V: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet v; + v.add((UChar32)0x0a, (UChar32)0x0d); // add range + v.add((UChar32)0x85); + v.add((UChar32)0x2028, (UChar32)0x2029); + v.complement(); + set->addAll(v); + break; + } + + case doSetBackslash_w: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + set->addAll(RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET]); + break; + } + + case doSetBackslash_W: + { + UnicodeSet *set = (UnicodeSet *)fSetStack.peek(); + UnicodeSet SSet; + SSet.addAll(RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET]).complement(); + set->addAll(SSet); + break; + } + + case doSetBegin: + { + fixLiterals(false); + LocalPointer lpSet(new UnicodeSet(), *fStatus); + fSetStack.push(lpSet.orphan(), *fStatus); + fSetOpStack.push(setStart, *fStatus); + if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) { + fSetOpStack.push(setCaseClose, *fStatus); + } + break; + } + + case doSetBeginDifference1: + // We have scanned something like [[abc]-[ + // Set up a new UnicodeSet for the set beginning with the just-scanned '[' + // Push a Difference operator, which will cause the new set to be subtracted from what + // went before once it is created. + setPushOp(setDifference1); + fSetOpStack.push(setStart, *fStatus); + if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) { + fSetOpStack.push(setCaseClose, *fStatus); + } + break; + + case doSetBeginIntersection1: + // We have scanned something like [[abc]&[ + // Need both the '&' operator and the open '[' operator. + setPushOp(setIntersection1); + fSetOpStack.push(setStart, *fStatus); + if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) { + fSetOpStack.push(setCaseClose, *fStatus); + } + break; + + case doSetBeginUnion: + // We have scanned something like [[abc][ + // Need to handle the union operation explicitly [[abc] | [ + setPushOp(setUnion); + fSetOpStack.push(setStart, *fStatus); + if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) { + fSetOpStack.push(setCaseClose, *fStatus); + } + break; + + case doSetDifference2: + // We have scanned something like [abc-- + // Consider this to unambiguously be a set difference operator. + setPushOp(setDifference2); + break; + + case doSetEnd: + // Have encountered the ']' that closes a set. + // Force the evaluation of any pending operations within this set, + // leave the completed set on the top of the set stack. + setEval(setEnd); + U_ASSERT(fSetOpStack.peeki()==setStart); + fSetOpStack.popi(); + break; + + case doSetFinish: + { + // Finished a complete set expression, including all nested sets. + // The close bracket has already triggered clearing out pending set operators, + // the operator stack should be empty and the operand stack should have just + // one entry, the result set. + U_ASSERT(fSetOpStack.empty()); + UnicodeSet *theSet = (UnicodeSet *)fSetStack.pop(); + U_ASSERT(fSetStack.empty()); + compileSet(theSet); + break; + } + + case doSetIntersection2: + // Have scanned something like [abc&& + setPushOp(setIntersection2); + break; + + case doSetLiteral: + // Union the just-scanned literal character into the set being built. + // This operation is the highest precedence set operation, so we can always do + // it immediately, without waiting to see what follows. It is necessary to perform + // any pending '-' or '&' operation first, because these have the same precedence + // as union-ing in a literal' + { + setEval(setUnion); + UnicodeSet *s = (UnicodeSet *)fSetStack.peek(); + s->add(fC.fChar); + fLastSetLiteral = fC.fChar; + break; + } + + case doSetLiteralEscaped: + // A back-slash escaped literal character was encountered. + // Processing is the same as with setLiteral, above, with the addition of + // the optional check for errors on escaped ASCII letters. + { + if ((fModeFlags & UREGEX_ERROR_ON_UNKNOWN_ESCAPES) != 0 && + ((fC.fChar >= 0x41 && fC.fChar<= 0x5A) || // in [A-Z] + (fC.fChar >= 0x61 && fC.fChar <= 0x7a))) { // in [a-z] + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + } + setEval(setUnion); + UnicodeSet *s = (UnicodeSet *)fSetStack.peek(); + s->add(fC.fChar); + fLastSetLiteral = fC.fChar; + break; + } + + case doSetNamedChar: + // Scanning a \N{UNICODE CHARACTER NAME} + // Aside from the source of the character, the processing is identical to doSetLiteral, + // above. + { + UChar32 c = scanNamedChar(); + setEval(setUnion); + UnicodeSet *s = (UnicodeSet *)fSetStack.peek(); + s->add(c); + fLastSetLiteral = c; + break; + } + + case doSetNamedRange: + // We have scanned literal-\N{CHAR NAME}. Add the range to the set. + // The left character is already in the set, and is saved in fLastSetLiteral. + // The right side needs to be picked up, the scan is at the 'N'. + // Lower Limit > Upper limit being an error matches both Java + // and ICU UnicodeSet behavior. + { + UChar32 c = scanNamedChar(); + if (U_SUCCESS(*fStatus) && (fLastSetLiteral == U_SENTINEL || fLastSetLiteral > c)) { + error(U_REGEX_INVALID_RANGE); + } + UnicodeSet *s = (UnicodeSet *)fSetStack.peek(); + s->add(fLastSetLiteral, c); + fLastSetLiteral = c; + break; + } + + + case doSetNegate: + // Scanned a '^' at the start of a set. + // Push the negation operator onto the set op stack. + // A twist for case-insensitive matching: + // the case closure operation must happen _before_ negation. + // But the case closure operation will already be on the stack if it's required. + // This requires checking for case closure, and swapping the stack order + // if it is present. + { + int32_t tosOp = fSetOpStack.peeki(); + if (tosOp == setCaseClose) { + fSetOpStack.popi(); + fSetOpStack.push(setNegation, *fStatus); + fSetOpStack.push(setCaseClose, *fStatus); + } else { + fSetOpStack.push(setNegation, *fStatus); + } + } + break; + + case doSetNoCloseError: + error(U_REGEX_MISSING_CLOSE_BRACKET); + break; + + case doSetOpError: + error(U_REGEX_RULE_SYNTAX); // -- or && at the end of a set. Illegal. + break; + + case doSetPosixProp: + { + UnicodeSet *s = scanPosixProp(); + if (s != nullptr) { + UnicodeSet *tos = (UnicodeSet *)fSetStack.peek(); + tos->addAll(*s); + delete s; + } // else error. scanProp() reported the error status already. + } + break; + + case doSetProp: + // Scanned a \p \P within [brackets]. + { + UnicodeSet *s = scanProp(); + if (s != nullptr) { + UnicodeSet *tos = (UnicodeSet *)fSetStack.peek(); + tos->addAll(*s); + delete s; + } // else error. scanProp() reported the error status already. + } + break; + + + case doSetRange: + // We have scanned literal-literal. Add the range to the set. + // The left character is already in the set, and is saved in fLastSetLiteral. + // The right side is the current character. + // Lower Limit > Upper limit being an error matches both Java + // and ICU UnicodeSet behavior. + { + + if (fLastSetLiteral == U_SENTINEL || fLastSetLiteral > fC.fChar) { + error(U_REGEX_INVALID_RANGE); + } + UnicodeSet *s = (UnicodeSet *)fSetStack.peek(); + s->add(fLastSetLiteral, fC.fChar); + break; + } + + default: + UPRV_UNREACHABLE_EXIT; + } + + if (U_FAILURE(*fStatus)) { + returnVal = false; + } + + return returnVal; +} + + + +//------------------------------------------------------------------------------ +// +// literalChar We've encountered a literal character from the pattern, +// or an escape sequence that reduces to a character. +// Add it to the string containing all literal chars/strings from +// the pattern. +// +//------------------------------------------------------------------------------ +void RegexCompile::literalChar(UChar32 c) { + fLiteralChars.append(c); +} + + +//------------------------------------------------------------------------------ +// +// fixLiterals When compiling something that can follow a literal +// string in a pattern, emit the code to match the +// accumulated literal string. +// +// Optionally, split the last char of the string off into +// a single "ONE_CHAR" operation, so that quantifiers can +// apply to that char alone. Example: abc* +// The * must apply to the 'c' only. +// +//------------------------------------------------------------------------------ +void RegexCompile::fixLiterals(UBool split) { + + // If no literal characters have been scanned but not yet had code generated + // for them, nothing needs to be done. + if (fLiteralChars.length() == 0) { + return; + } + + int32_t indexOfLastCodePoint = fLiteralChars.moveIndex32(fLiteralChars.length(), -1); + UChar32 lastCodePoint = fLiteralChars.char32At(indexOfLastCodePoint); + + // Split: We need to ensure that the last item in the compiled pattern + // refers only to the last literal scanned in the pattern, so that + // quantifiers (*, +, etc.) affect only it, and not a longer string. + // Split before case folding for case insensitive matches. + + if (split) { + fLiteralChars.truncate(indexOfLastCodePoint); + fixLiterals(false); // Recursive call, emit code to match the first part of the string. + // Note that the truncated literal string may be empty, in which case + // nothing will be emitted. + + literalChar(lastCodePoint); // Re-add the last code point as if it were a new literal. + fixLiterals(false); // Second recursive call, code for the final code point. + return; + } + + // If we are doing case-insensitive matching, case fold the string. This may expand + // the string, e.g. the German sharp-s turns into "ss" + if (fModeFlags & UREGEX_CASE_INSENSITIVE) { + fLiteralChars.foldCase(); + indexOfLastCodePoint = fLiteralChars.moveIndex32(fLiteralChars.length(), -1); + lastCodePoint = fLiteralChars.char32At(indexOfLastCodePoint); + } + + if (indexOfLastCodePoint == 0) { + // Single character, emit a URX_ONECHAR op to match it. + if ((fModeFlags & UREGEX_CASE_INSENSITIVE) && + u_hasBinaryProperty(lastCodePoint, UCHAR_CASE_SENSITIVE)) { + appendOp(URX_ONECHAR_I, lastCodePoint); + } else { + appendOp(URX_ONECHAR, lastCodePoint); + } + } else { + // Two or more chars, emit a URX_STRING to match them. + if (fLiteralChars.length() > 0x00ffffff || fRXPat->fLiteralText.length() > 0x00ffffff) { + error(U_REGEX_PATTERN_TOO_BIG); + } + if (fModeFlags & UREGEX_CASE_INSENSITIVE) { + appendOp(URX_STRING_I, fRXPat->fLiteralText.length()); + } else { + // TODO here: add optimization to split case sensitive strings of length two + // into two single char ops, for efficiency. + appendOp(URX_STRING, fRXPat->fLiteralText.length()); + } + appendOp(URX_STRING_LEN, fLiteralChars.length()); + + // Add this string into the accumulated strings of the compiled pattern. + fRXPat->fLiteralText.append(fLiteralChars); + } + + fLiteralChars.remove(); +} + + +int32_t RegexCompile::buildOp(int32_t type, int32_t val) { + if (U_FAILURE(*fStatus)) { + return 0; + } + if (type < 0 || type > 255) { + UPRV_UNREACHABLE_EXIT; + } + if (val > 0x00ffffff) { + UPRV_UNREACHABLE_EXIT; + } + if (val < 0) { + if (!(type == URX_RESERVED_OP_N || type == URX_RESERVED_OP)) { + UPRV_UNREACHABLE_EXIT; + } + if (URX_TYPE(val) != 0xff) { + UPRV_UNREACHABLE_EXIT; + } + type = URX_RESERVED_OP_N; + } + return (type << 24) | val; +} + + +//------------------------------------------------------------------------------ +// +// appendOp() Append a new instruction onto the compiled pattern +// Includes error checking, limiting the size of the +// pattern to lengths that can be represented in the +// 24 bit operand field of an instruction. +// +//------------------------------------------------------------------------------ +void RegexCompile::appendOp(int32_t op) { + if (U_FAILURE(*fStatus)) { + return; + } + fRXPat->fCompiledPat->addElement(op, *fStatus); + if ((fRXPat->fCompiledPat->size() > 0x00fffff0) && U_SUCCESS(*fStatus)) { + error(U_REGEX_PATTERN_TOO_BIG); + } +} + +void RegexCompile::appendOp(int32_t type, int32_t val) { + appendOp(buildOp(type, val)); +} + + +//------------------------------------------------------------------------------ +// +// insertOp() Insert a slot for a new opcode into the already +// compiled pattern code. +// +// Fill the slot with a NOP. Our caller will replace it +// with what they really wanted. +// +//------------------------------------------------------------------------------ +void RegexCompile::insertOp(int32_t where) { + UVector64 *code = fRXPat->fCompiledPat; + U_ASSERT(where>0 && where < code->size()); + + int32_t nop = buildOp(URX_NOP, 0); + code->insertElementAt(nop, where, *fStatus); + + // Walk through the pattern, looking for any ops with targets that + // were moved down by the insert. Fix them. + int32_t loc; + for (loc=0; locsize(); loc++) { + int32_t op = (int32_t)code->elementAti(loc); + int32_t opType = URX_TYPE(op); + int32_t opValue = URX_VAL(op); + if ((opType == URX_JMP || + opType == URX_JMPX || + opType == URX_STATE_SAVE || + opType == URX_CTR_LOOP || + opType == URX_CTR_LOOP_NG || + opType == URX_JMP_SAV || + opType == URX_JMP_SAV_X || + opType == URX_RELOC_OPRND) && opValue > where) { + // Target location for this opcode is after the insertion point and + // needs to be incremented to adjust for the insertion. + opValue++; + op = buildOp(opType, opValue); + code->setElementAt(op, loc); + } + } + + // Now fix up the parentheses stack. All positive values in it are locations in + // the compiled pattern. (Negative values are frame boundaries, and don't need fixing.) + for (loc=0; locsize()); + if (x>where) { + x++; + fParenStack.setElementAt(x, loc); + } + } + + if (fMatchCloseParen > where) { + fMatchCloseParen++; + } + if (fMatchOpenParen > where) { + fMatchOpenParen++; + } +} + + +//------------------------------------------------------------------------------ +// +// allocateData() Allocate storage in the matcher's static data area. +// Return the index for the newly allocated data. +// The storage won't actually exist until we are running a match +// operation, but the storage indexes are inserted into various +// opcodes while compiling the pattern. +// +//------------------------------------------------------------------------------ +int32_t RegexCompile::allocateData(int32_t size) { + if (U_FAILURE(*fStatus)) { + return 0; + } + if (size <= 0 || size > 0x100 || fRXPat->fDataSize < 0) { + error(U_REGEX_INTERNAL_ERROR); + return 0; + } + int32_t dataIndex = fRXPat->fDataSize; + fRXPat->fDataSize += size; + if (fRXPat->fDataSize >= 0x00fffff0) { + error(U_REGEX_INTERNAL_ERROR); + } + return dataIndex; +} + + +//------------------------------------------------------------------------------ +// +// allocateStackData() Allocate space in the back-tracking stack frame. +// Return the index for the newly allocated data. +// The frame indexes are inserted into various +// opcodes while compiling the pattern, meaning that frame +// size must be restricted to the size that will fit +// as an operand (24 bits). +// +//------------------------------------------------------------------------------ +int32_t RegexCompile::allocateStackData(int32_t size) { + if (U_FAILURE(*fStatus)) { + return 0; + } + if (size <= 0 || size > 0x100 || fRXPat->fFrameSize < 0) { + error(U_REGEX_INTERNAL_ERROR); + return 0; + } + int32_t dataIndex = fRXPat->fFrameSize; + fRXPat->fFrameSize += size; + if (fRXPat->fFrameSize >= 0x00fffff0) { + error(U_REGEX_PATTERN_TOO_BIG); + } + return dataIndex; +} + + +//------------------------------------------------------------------------------ +// +// blockTopLoc() Find or create a location in the compiled pattern +// at the start of the operation or block that has +// just been compiled. Needed when a quantifier (* or +// whatever) appears, and we need to add an operation +// at the start of the thing being quantified. +// +// (Parenthesized Blocks) have a slot with a NOP that +// is reserved for this purpose. .* or similar don't +// and a slot needs to be added. +// +// parameter reserveLoc : true - ensure that there is space to add an opcode +// at the returned location. +// false - just return the address, +// do not reserve a location there. +// +//------------------------------------------------------------------------------ +int32_t RegexCompile::blockTopLoc(UBool reserveLoc) { + int32_t theLoc; + fixLiterals(true); // Emit code for any pending literals. + // If last item was a string, emit separate op for the its last char. + if (fRXPat->fCompiledPat->size() == fMatchCloseParen) + { + // The item just processed is a parenthesized block. + theLoc = fMatchOpenParen; // A slot is already reserved for us. + U_ASSERT(theLoc > 0); + U_ASSERT(URX_TYPE(((uint32_t)fRXPat->fCompiledPat->elementAti(theLoc))) == URX_NOP); + } + else { + // Item just compiled is a single thing, a ".", or a single char, a string or a set reference. + // No slot for STATE_SAVE was pre-reserved in the compiled code. + // We need to make space now. + theLoc = fRXPat->fCompiledPat->size()-1; + int32_t opAtTheLoc = (int32_t)fRXPat->fCompiledPat->elementAti(theLoc); + if (URX_TYPE(opAtTheLoc) == URX_STRING_LEN) { + // Strings take two opcode, we want the position of the first one. + // We can have a string at this point if a single character case-folded to two. + theLoc--; + } + if (reserveLoc) { + int32_t nop = buildOp(URX_NOP, 0); + fRXPat->fCompiledPat->insertElementAt(nop, theLoc, *fStatus); + } + } + return theLoc; +} + + + +//------------------------------------------------------------------------------ +// +// handleCloseParen When compiling a close paren, we need to go back +// and fix up any JMP or SAVE operations within the +// parenthesized block that need to target the end +// of the block. The locations of these are kept on +// the paretheses stack. +// +// This function is called both when encountering a +// real ) and at the end of the pattern. +// +//------------------------------------------------------------------------------ +void RegexCompile::handleCloseParen() { + int32_t patIdx; + int32_t patOp; + if (fParenStack.size() <= 0) { + error(U_REGEX_MISMATCHED_PAREN); + return; + } + + // Emit code for any pending literals. + fixLiterals(false); + + // Fixup any operations within the just-closed parenthesized group + // that need to reference the end of the (block). + // (The first one popped from the stack is an unused slot for + // alternation (OR) state save, but applying the fixup to it does no harm.) + for (;;) { + patIdx = fParenStack.popi(); + if (patIdx < 0) { + // value < 0 flags the start of the frame on the paren stack. + break; + } + U_ASSERT(patIdx>0 && patIdx <= fRXPat->fCompiledPat->size()); + patOp = (int32_t)fRXPat->fCompiledPat->elementAti(patIdx); + U_ASSERT(URX_VAL(patOp) == 0); // Branch target for JMP should not be set. + patOp |= fRXPat->fCompiledPat->size(); // Set it now. + fRXPat->fCompiledPat->setElementAt(patOp, patIdx); + fMatchOpenParen = patIdx; + } + + // At the close of any parenthesized block, restore the match mode flags to + // the value they had at the open paren. Saved value is + // at the top of the paren stack. + fModeFlags = fParenStack.popi(); + U_ASSERT(fModeFlags < 0); + + // DO any additional fixups, depending on the specific kind of + // parentesized grouping this is + + switch (patIdx) { + case plain: + case flags: + // No additional fixups required. + // (Grouping-only parentheses) + break; + case capturing: + // Capturing Parentheses. + // Insert a End Capture op into the pattern. + // The frame offset of the variables for this cg is obtained from the + // start capture op and put it into the end-capture op. + { + int32_t captureOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen+1); + U_ASSERT(URX_TYPE(captureOp) == URX_START_CAPTURE); + + int32_t frameVarLocation = URX_VAL(captureOp); + appendOp(URX_END_CAPTURE, frameVarLocation); + } + break; + case atomic: + // Atomic Parenthesis. + // Insert a LD_SP operation to restore the state stack to the position + // it was when the atomic parens were entered. + { + int32_t stoOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen+1); + U_ASSERT(URX_TYPE(stoOp) == URX_STO_SP); + int32_t stoLoc = URX_VAL(stoOp); + appendOp(URX_LD_SP, stoLoc); + } + break; + + case lookAhead: + { + int32_t startOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen-5); + U_ASSERT(URX_TYPE(startOp) == URX_LA_START); + int32_t dataLoc = URX_VAL(startOp); + appendOp(URX_LA_END, dataLoc); + } + break; + + case negLookAhead: + { + // See comment at doOpenLookAheadNeg + int32_t startOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen-1); + U_ASSERT(URX_TYPE(startOp) == URX_LA_START); + int32_t dataLoc = URX_VAL(startOp); + appendOp(URX_LA_END, dataLoc); + appendOp(URX_BACKTRACK, 0); + appendOp(URX_LA_END, dataLoc); + + // Patch the URX_SAVE near the top of the block. + // The destination of the SAVE is the final LA_END that was just added. + int32_t saveOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen); + U_ASSERT(URX_TYPE(saveOp) == URX_STATE_SAVE); + int32_t dest = fRXPat->fCompiledPat->size()-1; + saveOp = buildOp(URX_STATE_SAVE, dest); + fRXPat->fCompiledPat->setElementAt(saveOp, fMatchOpenParen); + } + break; + + case lookBehind: + { + // See comment at doOpenLookBehind. + + // Append the URX_LB_END and URX_LA_END to the compiled pattern. + int32_t startOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen-4); + U_ASSERT(URX_TYPE(startOp) == URX_LB_START); + int32_t dataLoc = URX_VAL(startOp); + appendOp(URX_LB_END, dataLoc); + appendOp(URX_LA_END, dataLoc); + + // Determine the min and max bounds for the length of the + // string that the pattern can match. + // An unbounded upper limit is an error. + int32_t patEnd = fRXPat->fCompiledPat->size() - 1; + int32_t minML = minMatchLength(fMatchOpenParen, patEnd); + int32_t maxML = maxMatchLength(fMatchOpenParen, patEnd); + if (URX_TYPE(maxML) != 0) { + error(U_REGEX_LOOK_BEHIND_LIMIT); + break; + } + if (maxML == INT32_MAX) { + error(U_REGEX_LOOK_BEHIND_LIMIT); + break; + } + if (minML == INT32_MAX) { + // This condition happens when no match is possible, such as with a + // [set] expression containing no elements. + // In principle, the generated code to evaluate the expression could be deleted, + // but it's probably not worth the complication. + minML = 0; + } + U_ASSERT(minML <= maxML); + + // Insert the min and max match len bounds into the URX_LB_CONT op that + // appears at the top of the look-behind block, at location fMatchOpenParen+1 + fRXPat->fCompiledPat->setElementAt(minML, fMatchOpenParen-2); + fRXPat->fCompiledPat->setElementAt(maxML, fMatchOpenParen-1); + + } + break; + + + + case lookBehindN: + { + // See comment at doOpenLookBehindNeg. + + // Append the URX_LBN_END to the compiled pattern. + int32_t startOp = (int32_t)fRXPat->fCompiledPat->elementAti(fMatchOpenParen-5); + U_ASSERT(URX_TYPE(startOp) == URX_LB_START); + int32_t dataLoc = URX_VAL(startOp); + appendOp(URX_LBN_END, dataLoc); + + // Determine the min and max bounds for the length of the + // string that the pattern can match. + // An unbounded upper limit is an error. + int32_t patEnd = fRXPat->fCompiledPat->size() - 1; + int32_t minML = minMatchLength(fMatchOpenParen, patEnd); + int32_t maxML = maxMatchLength(fMatchOpenParen, patEnd); + if (URX_TYPE(maxML) != 0) { + error(U_REGEX_LOOK_BEHIND_LIMIT); + break; + } + if (maxML == INT32_MAX) { + error(U_REGEX_LOOK_BEHIND_LIMIT); + break; + } + if (minML == INT32_MAX) { + // This condition happens when no match is possible, such as with a + // [set] expression containing no elements. + // In principle, the generated code to evaluate the expression could be deleted, + // but it's probably not worth the complication. + minML = 0; + } + + U_ASSERT(minML <= maxML); + + // Insert the min and max match len bounds into the URX_LB_CONT op that + // appears at the top of the look-behind block, at location fMatchOpenParen+1 + fRXPat->fCompiledPat->setElementAt(minML, fMatchOpenParen-3); + fRXPat->fCompiledPat->setElementAt(maxML, fMatchOpenParen-2); + + // Insert the pattern location to continue at after a successful match + // as the last operand of the URX_LBN_CONT + int32_t op = buildOp(URX_RELOC_OPRND, fRXPat->fCompiledPat->size()); + fRXPat->fCompiledPat->setElementAt(op, fMatchOpenParen-1); + } + break; + + + + default: + UPRV_UNREACHABLE_EXIT; + } + + // remember the next location in the compiled pattern. + // The compilation of Quantifiers will look at this to see whether its looping + // over a parenthesized block or a single item + fMatchCloseParen = fRXPat->fCompiledPat->size(); +} + + + +//------------------------------------------------------------------------------ +// +// compileSet Compile the pattern operations for a reference to a +// UnicodeSet. +// +//------------------------------------------------------------------------------ +void RegexCompile::compileSet(UnicodeSet *theSet) +{ + if (theSet == nullptr) { + return; + } + // Remove any strings from the set. + // There shouldn't be any, but just in case. + // (Case Closure can add them; if we had a simple case closure available that + // ignored strings, that would be better.) + theSet->removeAllStrings(); + int32_t setSize = theSet->size(); + + switch (setSize) { + case 0: + { + // Set of no elements. Always fails to match. + appendOp(URX_BACKTRACK, 0); + delete theSet; + } + break; + + case 1: + { + // The set contains only a single code point. Put it into + // the compiled pattern as a single char operation rather + // than a set, and discard the set itself. + literalChar(theSet->charAt(0)); + delete theSet; + } + break; + + default: + { + // The set contains two or more chars. (the normal case) + // Put it into the compiled pattern as a set. + theSet->freeze(); + int32_t setNumber = fRXPat->fSets->size(); + fRXPat->fSets->addElement(theSet, *fStatus); + if (U_SUCCESS(*fStatus)) { + appendOp(URX_SETREF, setNumber); + } else { + delete theSet; + } + } + } +} + + +//------------------------------------------------------------------------------ +// +// compileInterval Generate the code for a {min, max} style interval quantifier. +// Except for the specific opcodes used, the code is the same +// for all three types (greedy, non-greedy, possessive) of +// intervals. The opcodes are supplied as parameters. +// (There are two sets of opcodes - greedy & possessive use the +// same ones, while non-greedy has it's own.) +// +// The code for interval loops has this form: +// 0 CTR_INIT counter loc (in stack frame) +// 1 5 patt address of CTR_LOOP at bottom of block +// 2 min count +// 3 max count (-1 for unbounded) +// 4 ... block to be iterated over +// 5 CTR_LOOP +// +// In +//------------------------------------------------------------------------------ +void RegexCompile::compileInterval(int32_t InitOp, int32_t LoopOp) +{ + // The CTR_INIT op at the top of the block with the {n,m} quantifier takes + // four slots in the compiled code. Reserve them. + int32_t topOfBlock = blockTopLoc(true); + insertOp(topOfBlock); + insertOp(topOfBlock); + insertOp(topOfBlock); + + // The operands for the CTR_INIT opcode include the index in the matcher data + // of the counter. Allocate it now. There are two data items + // counterLoc --> Loop counter + // +1 --> Input index (for breaking non-progressing loops) + // (Only present if unbounded upper limit on loop) + int32_t dataSize = fIntervalUpper < 0 ? 2 : 1; + int32_t counterLoc = allocateStackData(dataSize); + + int32_t op = buildOp(InitOp, counterLoc); + fRXPat->fCompiledPat->setElementAt(op, topOfBlock); + + // The second operand of CTR_INIT is the location following the end of the loop. + // Must put in as a URX_RELOC_OPRND so that the value will be adjusted if the + // compilation of something later on causes the code to grow and the target + // position to move. + int32_t loopEnd = fRXPat->fCompiledPat->size(); + op = buildOp(URX_RELOC_OPRND, loopEnd); + fRXPat->fCompiledPat->setElementAt(op, topOfBlock+1); + + // Followed by the min and max counts. + fRXPat->fCompiledPat->setElementAt(fIntervalLow, topOfBlock+2); + fRXPat->fCompiledPat->setElementAt(fIntervalUpper, topOfBlock+3); + + // Append the CTR_LOOP op. The operand is the location of the CTR_INIT op. + // Goes at end of the block being looped over, so just append to the code so far. + appendOp(LoopOp, topOfBlock); + + if ((fIntervalLow & 0xff000000) != 0 || + (fIntervalUpper > 0 && (fIntervalUpper & 0xff000000) != 0)) { + error(U_REGEX_NUMBER_TOO_BIG); + } + + if (fIntervalLow > fIntervalUpper && fIntervalUpper != -1) { + error(U_REGEX_MAX_LT_MIN); + } +} + + + +UBool RegexCompile::compileInlineInterval() { + if (fIntervalUpper > 10 || fIntervalUpper < fIntervalLow) { + // Too big to inline. Fail, which will cause looping code to be generated. + // (Upper < Lower picks up unbounded upper and errors, both.) + return false; + } + + int32_t topOfBlock = blockTopLoc(false); + if (fIntervalUpper == 0) { + // Pathological case. Attempt no matches, as if the block doesn't exist. + // Discard the generated code for the block. + // If the block included parens, discard the info pertaining to them as well. + fRXPat->fCompiledPat->setSize(topOfBlock); + if (fMatchOpenParen >= topOfBlock) { + fMatchOpenParen = -1; + } + if (fMatchCloseParen >= topOfBlock) { + fMatchCloseParen = -1; + } + return true; + } + + if (topOfBlock != fRXPat->fCompiledPat->size()-1 && fIntervalUpper != 1) { + // The thing being repeated is not a single op, but some + // more complex block. Do it as a loop, not inlines. + // Note that things "repeated" a max of once are handled as inline, because + // the one copy of the code already generated is just fine. + return false; + } + + // Pick up the opcode that is to be repeated + // + int32_t op = (int32_t)fRXPat->fCompiledPat->elementAti(topOfBlock); + + // Compute the pattern location where the inline sequence + // will end, and set up the state save op that will be needed. + // + int32_t endOfSequenceLoc = fRXPat->fCompiledPat->size()-1 + + fIntervalUpper + (fIntervalUpper-fIntervalLow); + int32_t saveOp = buildOp(URX_STATE_SAVE, endOfSequenceLoc); + if (fIntervalLow == 0) { + insertOp(topOfBlock); + fRXPat->fCompiledPat->setElementAt(saveOp, topOfBlock); + } + + + + // Loop, emitting the op for the thing being repeated each time. + // Loop starts at 1 because one instance of the op already exists in the pattern, + // it was put there when it was originally encountered. + int32_t i; + for (i=1; i= fIntervalLow) { + appendOp(saveOp); + } + appendOp(op); + } + return true; +} + + + +//------------------------------------------------------------------------------ +// +// caseInsensitiveStart given a single code point from a pattern string, determine the +// set of characters that could potentially begin a case-insensitive +// match of a string beginning with that character, using full Unicode +// case insensitive matching. +// +// This is used in optimizing find(). +// +// closeOver(USET_CASE_INSENSITIVE) does most of what is needed, but +// misses cases like this: +// A string from the pattern begins with 'ss' (although all we know +// in this context is that it begins with 's') +// The pattern could match a string beginning with a German sharp-s +// +// To the ordinary case closure for a character c, we add all other +// characters cx where the case closure of cx includes a string form that begins +// with the original character c. +// +// This function could be made smarter. The full pattern string is available +// and it would be possible to verify that the extra characters being added +// to the starting set fully match, rather than having just a first-char of the +// folded form match. +// +//------------------------------------------------------------------------------ +void RegexCompile::findCaseInsensitiveStarters(UChar32 c, UnicodeSet *starterChars) { + +// Machine Generated below. +// It may need updating with new versions of Unicode. +// Intltest test RegexTest::TestCaseInsensitiveStarters will fail if an update is needed. +// The update tool is here: +// https://github.com/unicode-org/icu/tree/main/tools/unicode/c/genregexcasing + +// Machine Generated Data. Do not hand edit. + static const UChar32 RECaseFixCodePoints[] = { + 0x61, 0x66, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x77, 0x79, 0x2bc, + 0x3ac, 0x3ae, 0x3b1, 0x3b7, 0x3b9, 0x3c1, 0x3c5, 0x3c9, 0x3ce, 0x565, + 0x574, 0x57e, 0x1f00, 0x1f01, 0x1f02, 0x1f03, 0x1f04, 0x1f05, 0x1f06, 0x1f07, + 0x1f20, 0x1f21, 0x1f22, 0x1f23, 0x1f24, 0x1f25, 0x1f26, 0x1f27, 0x1f60, 0x1f61, + 0x1f62, 0x1f63, 0x1f64, 0x1f65, 0x1f66, 0x1f67, 0x1f70, 0x1f74, 0x1f7c, 0x110000}; + + static const int16_t RECaseFixStringOffsets[] = { + 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xd, 0xe, 0xf, 0x10, + 0x11, 0x12, 0x13, 0x17, 0x1b, 0x20, 0x21, 0x2a, 0x2e, 0x2f, + 0x30, 0x34, 0x35, 0x37, 0x39, 0x3b, 0x3d, 0x3f, 0x41, 0x43, + 0x45, 0x47, 0x49, 0x4b, 0x4d, 0x4f, 0x51, 0x53, 0x55, 0x57, + 0x59, 0x5b, 0x5d, 0x5f, 0x61, 0x63, 0x65, 0x66, 0x67, 0}; + + static const int16_t RECaseFixCounts[] = { + 0x1, 0x5, 0x1, 0x1, 0x1, 0x4, 0x1, 0x1, 0x1, 0x1, + 0x1, 0x1, 0x4, 0x4, 0x5, 0x1, 0x9, 0x4, 0x1, 0x1, + 0x4, 0x1, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, + 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, + 0x2, 0x2, 0x2, 0x2, 0x2, 0x2, 0x1, 0x1, 0x1, 0}; + + static const char16_t RECaseFixData[] = { + 0x1e9a, 0xfb00, 0xfb01, 0xfb02, 0xfb03, 0xfb04, 0x1e96, 0x130, 0x1f0, 0xdf, + 0x1e9e, 0xfb05, 0xfb06, 0x1e97, 0x1e98, 0x1e99, 0x149, 0x1fb4, 0x1fc4, 0x1fb3, + 0x1fb6, 0x1fb7, 0x1fbc, 0x1fc3, 0x1fc6, 0x1fc7, 0x1fcc, 0x390, 0x1fd2, 0x1fd3, + 0x1fd6, 0x1fd7, 0x1fe4, 0x3b0, 0x1f50, 0x1f52, 0x1f54, 0x1f56, 0x1fe2, 0x1fe3, + 0x1fe6, 0x1fe7, 0x1ff3, 0x1ff6, 0x1ff7, 0x1ffc, 0x1ff4, 0x587, 0xfb13, 0xfb14, + 0xfb15, 0xfb17, 0xfb16, 0x1f80, 0x1f88, 0x1f81, 0x1f89, 0x1f82, 0x1f8a, 0x1f83, + 0x1f8b, 0x1f84, 0x1f8c, 0x1f85, 0x1f8d, 0x1f86, 0x1f8e, 0x1f87, 0x1f8f, 0x1f90, + 0x1f98, 0x1f91, 0x1f99, 0x1f92, 0x1f9a, 0x1f93, 0x1f9b, 0x1f94, 0x1f9c, 0x1f95, + 0x1f9d, 0x1f96, 0x1f9e, 0x1f97, 0x1f9f, 0x1fa0, 0x1fa8, 0x1fa1, 0x1fa9, 0x1fa2, + 0x1faa, 0x1fa3, 0x1fab, 0x1fa4, 0x1fac, 0x1fa5, 0x1fad, 0x1fa6, 0x1fae, 0x1fa7, + 0x1faf, 0x1fb2, 0x1fc2, 0x1ff2, 0}; + +// End of machine generated data. + + if (c < UCHAR_MIN_VALUE || c > UCHAR_MAX_VALUE) { + // This function should never be called with an invalid input character. + UPRV_UNREACHABLE_EXIT; + } else if (u_hasBinaryProperty(c, UCHAR_CASE_SENSITIVE)) { + UChar32 caseFoldedC = u_foldCase(c, U_FOLD_CASE_DEFAULT); + starterChars->set(caseFoldedC, caseFoldedC); + + int32_t i; + for (i=0; RECaseFixCodePoints[i]add(cpToAdd); + } + } + + starterChars->closeOver(USET_CASE_INSENSITIVE); + starterChars->removeAllStrings(); + } else { + // Not a cased character. Just return it alone. + starterChars->set(c, c); + } +} + + +// Increment with overflow check. +// val and delta will both be positive. + +static int32_t safeIncrement(int32_t val, int32_t delta) { + if (INT32_MAX - val > delta) { + return val + delta; + } else { + return INT32_MAX; + } +} + + +//------------------------------------------------------------------------------ +// +// matchStartType Determine how a match can start. +// Used to optimize find() operations. +// +// Operation is very similar to minMatchLength(). Walk the compiled +// pattern, keeping an on-going minimum-match-length. For any +// op where the min match coming in is zero, add that ops possible +// starting matches to the possible starts for the overall pattern. +// +//------------------------------------------------------------------------------ +void RegexCompile::matchStartType() { + if (U_FAILURE(*fStatus)) { + return; + } + + + int32_t loc; // Location in the pattern of the current op being processed. + int32_t op; // The op being processed + int32_t opType; // The opcode type of the op + int32_t currentLen = 0; // Minimum length of a match to this point (loc) in the pattern + int32_t numInitialStrings = 0; // Number of strings encountered that could match at start. + + UBool atStart = true; // True if no part of the pattern yet encountered + // could have advanced the position in a match. + // (Maximum match length so far == 0) + + // forwardedLength is a vector holding minimum-match-length values that + // are propagated forward in the pattern by JMP or STATE_SAVE operations. + // It must be one longer than the pattern being checked because some ops + // will jmp to a end-of-block+1 location from within a block, and we must + // count those when checking the block. + int32_t end = fRXPat->fCompiledPat->size(); + UVector32 forwardedLength(end+1, *fStatus); + forwardedLength.setSize(end+1); + for (loc=3; locfCompiledPat->elementAti(loc); + opType = URX_TYPE(op); + + // The loop is advancing linearly through the pattern. + // If the op we are now at was the destination of a branch in the pattern, + // and that path has a shorter minimum length than the current accumulated value, + // replace the current accumulated value. + if (forwardedLength.elementAti(loc) < currentLen) { + currentLen = forwardedLength.elementAti(loc); + U_ASSERT(currentLen>=0 && currentLen < INT32_MAX); + } + + switch (opType) { + // Ops that don't change the total length matched + case URX_RESERVED_OP: + case URX_END: + case URX_FAIL: + case URX_STRING_LEN: + case URX_NOP: + case URX_START_CAPTURE: + case URX_END_CAPTURE: + case URX_BACKSLASH_B: + case URX_BACKSLASH_BU: + case URX_BACKSLASH_G: + case URX_BACKSLASH_Z: + case URX_DOLLAR: + case URX_DOLLAR_M: + case URX_DOLLAR_D: + case URX_DOLLAR_MD: + case URX_RELOC_OPRND: + case URX_STO_INP_LOC: + case URX_BACKREF: // BackRef. Must assume that it might be a zero length match + case URX_BACKREF_I: + + case URX_STO_SP: // Setup for atomic or possessive blocks. Doesn't change what can match. + case URX_LD_SP: + break; + + case URX_CARET: + if (atStart) { + fRXPat->fStartType = START_START; + } + break; + + case URX_CARET_M: + case URX_CARET_M_UNIX: + if (atStart) { + fRXPat->fStartType = START_LINE; + } + break; + + case URX_ONECHAR: + if (currentLen == 0) { + // This character could appear at the start of a match. + // Add it to the set of possible starting characters. + fRXPat->fInitialChars->add(URX_VAL(op)); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + case URX_SETREF: + if (currentLen == 0) { + int32_t sn = URX_VAL(op); + U_ASSERT(sn > 0 && sn < fRXPat->fSets->size()); + const UnicodeSet *s = (UnicodeSet *)fRXPat->fSets->elementAt(sn); + fRXPat->fInitialChars->addAll(*s); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + case URX_LOOP_SR_I: + // [Set]*, like a SETREF, above, in what it can match, + // but may not match at all, so currentLen is not incremented. + if (currentLen == 0) { + int32_t sn = URX_VAL(op); + U_ASSERT(sn > 0 && sn < fRXPat->fSets->size()); + const UnicodeSet *s = (UnicodeSet *)fRXPat->fSets->elementAt(sn); + fRXPat->fInitialChars->addAll(*s); + numInitialStrings += 2; + } + atStart = false; + break; + + case URX_LOOP_DOT_I: + if (currentLen == 0) { + // .* at the start of a pattern. + // Any character can begin the match. + fRXPat->fInitialChars->clear(); + fRXPat->fInitialChars->complement(); + numInitialStrings += 2; + } + atStart = false; + break; + + + case URX_STATIC_SETREF: + if (currentLen == 0) { + int32_t sn = URX_VAL(op); + U_ASSERT(sn>0 && snfPropSets[sn]; + fRXPat->fInitialChars->addAll(s); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + + case URX_STAT_SETREF_N: + if (currentLen == 0) { + int32_t sn = URX_VAL(op); + UnicodeSet sc; + sc.addAll(RegexStaticSets::gStaticSets->fPropSets[sn]).complement(); + fRXPat->fInitialChars->addAll(sc); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + + case URX_BACKSLASH_D: + // Digit Char + if (currentLen == 0) { + UnicodeSet s; + s.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ND_MASK, *fStatus); + if (URX_VAL(op) != 0) { + s.complement(); + } + fRXPat->fInitialChars->addAll(s); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + case URX_BACKSLASH_H: + // Horiz white space + if (currentLen == 0) { + UnicodeSet s; + s.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ZS_MASK, *fStatus); + s.add((UChar32)9); // Tab + if (URX_VAL(op) != 0) { + s.complement(); + } + fRXPat->fInitialChars->addAll(s); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + case URX_BACKSLASH_R: // Any line ending sequence + case URX_BACKSLASH_V: // Any line ending code point, with optional negation + if (currentLen == 0) { + UnicodeSet s; + s.add((UChar32)0x0a, (UChar32)0x0d); // add range + s.add((UChar32)0x85); + s.add((UChar32)0x2028, (UChar32)0x2029); + if (URX_VAL(op) != 0) { + // Complement option applies to URX_BACKSLASH_V only. + s.complement(); + } + fRXPat->fInitialChars->addAll(s); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + + case URX_ONECHAR_I: + // Case Insensitive Single Character. + if (currentLen == 0) { + UChar32 c = URX_VAL(op); + if (u_hasBinaryProperty(c, UCHAR_CASE_SENSITIVE)) { + UnicodeSet starters(c, c); + starters.closeOver(USET_CASE_INSENSITIVE); + // findCaseInsensitiveStarters(c, &starters); + // For ONECHAR_I, no need to worry about text chars that expand on folding into strings. + // The expanded folding can't match the pattern. + fRXPat->fInitialChars->addAll(starters); + } else { + // Char has no case variants. Just add it as-is to the + // set of possible starting chars. + fRXPat->fInitialChars->add(c); + } + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + case URX_BACKSLASH_X: // Grapheme Cluster. Minimum is 1, max unbounded. + case URX_DOTANY_ALL: // . matches one or two. + case URX_DOTANY: + case URX_DOTANY_UNIX: + if (currentLen == 0) { + // These constructs are all bad news when they appear at the start + // of a match. Any character can begin the match. + fRXPat->fInitialChars->clear(); + fRXPat->fInitialChars->complement(); + numInitialStrings += 2; + } + currentLen = safeIncrement(currentLen, 1); + atStart = false; + break; + + + case URX_JMPX: + loc++; // Except for extra operand on URX_JMPX, same as URX_JMP. + U_FALLTHROUGH; + case URX_JMP: + { + int32_t jmpDest = URX_VAL(op); + if (jmpDest < loc) { + // Loop of some kind. Can safely ignore, the worst that will happen + // is that we understate the true minimum length + currentLen = forwardedLength.elementAti(loc+1); + + } else { + // Forward jump. Propagate the current min length to the target loc of the jump. + U_ASSERT(jmpDest <= end+1); + if (forwardedLength.elementAti(jmpDest) > currentLen) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + atStart = false; + break; + + case URX_JMP_SAV: + case URX_JMP_SAV_X: + // Combo of state save to the next loc, + jmp backwards. + // Net effect on min. length computation is nothing. + atStart = false; + break; + + case URX_BACKTRACK: + // Fails are kind of like a branch, except that the min length was + // propagated already, by the state save. + currentLen = forwardedLength.elementAti(loc+1); + atStart = false; + break; + + + case URX_STATE_SAVE: + { + // State Save, for forward jumps, propagate the current minimum. + // of the state save. + int32_t jmpDest = URX_VAL(op); + if (jmpDest > loc) { + if (currentLen < forwardedLength.elementAti(jmpDest)) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + atStart = false; + break; + + + + + case URX_STRING: + { + loc++; + int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + int32_t stringLen = URX_VAL(stringLenOp); + U_ASSERT(URX_TYPE(stringLenOp) == URX_STRING_LEN); + U_ASSERT(stringLenOp >= 2); + if (currentLen == 0) { + // Add the starting character of this string to the set of possible starting + // characters for this pattern. + int32_t stringStartIdx = URX_VAL(op); + UChar32 c = fRXPat->fLiteralText.char32At(stringStartIdx); + fRXPat->fInitialChars->add(c); + + // Remember this string. After the entire pattern has been checked, + // if nothing else is identified that can start a match, we'll use it. + numInitialStrings++; + fRXPat->fInitialStringIdx = stringStartIdx; + fRXPat->fInitialStringLen = stringLen; + } + + currentLen = safeIncrement(currentLen, stringLen); + atStart = false; + } + break; + + case URX_STRING_I: + { + // Case-insensitive string. Unlike exact-match strings, we won't + // attempt a string search for possible match positions. But we + // do update the set of possible starting characters. + loc++; + int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + int32_t stringLen = URX_VAL(stringLenOp); + U_ASSERT(URX_TYPE(stringLenOp) == URX_STRING_LEN); + U_ASSERT(stringLenOp >= 2); + if (currentLen == 0) { + // Add the starting character of this string to the set of possible starting + // characters for this pattern. + int32_t stringStartIdx = URX_VAL(op); + UChar32 c = fRXPat->fLiteralText.char32At(stringStartIdx); + UnicodeSet s; + findCaseInsensitiveStarters(c, &s); + fRXPat->fInitialChars->addAll(s); + numInitialStrings += 2; // Matching on an initial string not possible. + } + currentLen = safeIncrement(currentLen, stringLen); + atStart = false; + } + break; + + case URX_CTR_INIT: + case URX_CTR_INIT_NG: + { + // Loop Init Ops. These don't change the min length, but they are 4 word ops + // so location must be updated accordingly. + // Loop Init Ops. + // If the min loop count == 0 + // move loc forwards to the end of the loop, skipping over the body. + // If the min count is > 0, + // continue normal processing of the body of the loop. + int32_t loopEndLoc = (int32_t)fRXPat->fCompiledPat->elementAti(loc+1); + loopEndLoc = URX_VAL(loopEndLoc); + int32_t minLoopCount = (int32_t)fRXPat->fCompiledPat->elementAti(loc+2); + if (minLoopCount == 0) { + // Min Loop Count of 0, treat like a forward branch and + // move the current minimum length up to the target + // (end of loop) location. + U_ASSERT(loopEndLoc <= end+1); + if (forwardedLength.elementAti(loopEndLoc) > currentLen) { + forwardedLength.setElementAt(currentLen, loopEndLoc); + } + } + loc+=3; // Skips over operands of CTR_INIT + } + atStart = false; + break; + + + case URX_CTR_LOOP: + case URX_CTR_LOOP_NG: + // Loop ops. + // The jump is conditional, backwards only. + atStart = false; + break; + + case URX_LOOP_C: + // More loop ops. These state-save to themselves. + // don't change the minimum match + atStart = false; + break; + + + case URX_LA_START: + case URX_LB_START: + { + // Look-around. Scan forward until the matching look-ahead end, + // without processing the look-around block. This is overly pessimistic. + + // Keep track of the nesting depth of look-around blocks. Boilerplate code for + // lookahead contains two LA_END instructions, so count goes up by two + // for each LA_START. + int32_t depth = (opType == URX_LA_START? 2: 1); + for (;;) { + loc++; + op = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + if (URX_TYPE(op) == URX_LA_START) { + depth+=2; + } + if (URX_TYPE(op) == URX_LB_START) { + depth++; + } + if (URX_TYPE(op) == URX_LA_END || URX_TYPE(op)==URX_LBN_END) { + depth--; + if (depth == 0) { + break; + } + } + if (URX_TYPE(op) == URX_STATE_SAVE) { + // Need this because neg lookahead blocks will FAIL to outside + // of the block. + int32_t jmpDest = URX_VAL(op); + if (jmpDest > loc) { + if (currentLen < forwardedLength.elementAti(jmpDest)) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + U_ASSERT(loc <= end); + } + } + break; + + case URX_LA_END: + case URX_LB_CONT: + case URX_LB_END: + case URX_LBN_CONT: + case URX_LBN_END: + UPRV_UNREACHABLE_EXIT; // Shouldn't get here. These ops should be + // consumed by the scan in URX_LA_START and LB_START + default: + UPRV_UNREACHABLE_EXIT; + } + + } + + + // We have finished walking through the ops. Check whether some forward jump + // propagated a shorter length to location end+1. + if (forwardedLength.elementAti(end+1) < currentLen) { + currentLen = forwardedLength.elementAti(end+1); + } + + + fRXPat->fInitialChars8->init(fRXPat->fInitialChars); + + + // Sort out what we should check for when looking for candidate match start positions. + // In order of preference, + // 1. Start of input text buffer. + // 2. A literal string. + // 3. Start of line in multi-line mode. + // 4. A single literal character. + // 5. A character from a set of characters. + // + if (fRXPat->fStartType == START_START) { + // Match only at the start of an input text string. + // start type is already set. We're done. + } else if (numInitialStrings == 1 && fRXPat->fMinMatchLen > 0) { + // Match beginning only with a literal string. + UChar32 c = fRXPat->fLiteralText.char32At(fRXPat->fInitialStringIdx); + U_ASSERT(fRXPat->fInitialChars->contains(c)); + fRXPat->fStartType = START_STRING; + fRXPat->fInitialChar = c; + } else if (fRXPat->fStartType == START_LINE) { + // Match at start of line in Multi-Line mode. + // Nothing to do here; everything is already set. + } else if (fRXPat->fMinMatchLen == 0) { + // Zero length match possible. We could start anywhere. + fRXPat->fStartType = START_NO_INFO; + } else if (fRXPat->fInitialChars->size() == 1) { + // All matches begin with the same char. + fRXPat->fStartType = START_CHAR; + fRXPat->fInitialChar = fRXPat->fInitialChars->charAt(0); + U_ASSERT(fRXPat->fInitialChar != (UChar32)-1); + } else if (fRXPat->fInitialChars->contains((UChar32)0, (UChar32)0x10ffff) == false && + fRXPat->fMinMatchLen > 0) { + // Matches start with a set of character smaller than the set of all chars. + fRXPat->fStartType = START_SET; + } else { + // Matches can start with anything + fRXPat->fStartType = START_NO_INFO; + } + + return; +} + + + +//------------------------------------------------------------------------------ +// +// minMatchLength Calculate the length of the shortest string that could +// match the specified pattern. +// Length is in 16 bit code units, not code points. +// +// The calculated length may not be exact. The returned +// value may be shorter than the actual minimum; it must +// never be longer. +// +// start and end are the range of p-code operations to be +// examined. The endpoints are included in the range. +// +//------------------------------------------------------------------------------ +int32_t RegexCompile::minMatchLength(int32_t start, int32_t end) { + if (U_FAILURE(*fStatus)) { + return 0; + } + + U_ASSERT(start <= end); + U_ASSERT(end < fRXPat->fCompiledPat->size()); + + + int32_t loc; + int32_t op; + int32_t opType; + int32_t currentLen = 0; + + + // forwardedLength is a vector holding minimum-match-length values that + // are propagated forward in the pattern by JMP or STATE_SAVE operations. + // It must be one longer than the pattern being checked because some ops + // will jmp to a end-of-block+1 location from within a block, and we must + // count those when checking the block. + UVector32 forwardedLength(end+2, *fStatus); + forwardedLength.setSize(end+2); + for (loc=start; loc<=end+1; loc++) { + forwardedLength.setElementAt(INT32_MAX, loc); + } + + for (loc = start; loc<=end; loc++) { + op = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + opType = URX_TYPE(op); + + // The loop is advancing linearly through the pattern. + // If the op we are now at was the destination of a branch in the pattern, + // and that path has a shorter minimum length than the current accumulated value, + // replace the current accumulated value. + // U_ASSERT(currentLen>=0 && currentLen < INT32_MAX); // MinLength == INT32_MAX for some + // no-match-possible cases. + if (forwardedLength.elementAti(loc) < currentLen) { + currentLen = forwardedLength.elementAti(loc); + U_ASSERT(currentLen>=0 && currentLen < INT32_MAX); + } + + switch (opType) { + // Ops that don't change the total length matched + case URX_RESERVED_OP: + case URX_END: + case URX_STRING_LEN: + case URX_NOP: + case URX_START_CAPTURE: + case URX_END_CAPTURE: + case URX_BACKSLASH_B: + case URX_BACKSLASH_BU: + case URX_BACKSLASH_G: + case URX_BACKSLASH_Z: + case URX_CARET: + case URX_DOLLAR: + case URX_DOLLAR_M: + case URX_DOLLAR_D: + case URX_DOLLAR_MD: + case URX_RELOC_OPRND: + case URX_STO_INP_LOC: + case URX_CARET_M: + case URX_CARET_M_UNIX: + case URX_BACKREF: // BackRef. Must assume that it might be a zero length match + case URX_BACKREF_I: + + case URX_STO_SP: // Setup for atomic or possessive blocks. Doesn't change what can match. + case URX_LD_SP: + + case URX_JMP_SAV: + case URX_JMP_SAV_X: + break; + + + // Ops that match a minimum of one character (one or two 16 bit code units.) + // + case URX_ONECHAR: + case URX_STATIC_SETREF: + case URX_STAT_SETREF_N: + case URX_SETREF: + case URX_BACKSLASH_D: + case URX_BACKSLASH_H: + case URX_BACKSLASH_R: + case URX_BACKSLASH_V: + case URX_ONECHAR_I: + case URX_BACKSLASH_X: // Grapheme Cluster. Minimum is 1, max unbounded. + case URX_DOTANY_ALL: // . matches one or two. + case URX_DOTANY: + case URX_DOTANY_UNIX: + currentLen = safeIncrement(currentLen, 1); + break; + + + case URX_JMPX: + loc++; // URX_JMPX has an extra operand, ignored here, + // otherwise processed identically to URX_JMP. + U_FALLTHROUGH; + case URX_JMP: + { + int32_t jmpDest = URX_VAL(op); + if (jmpDest < loc) { + // Loop of some kind. Can safely ignore, the worst that will happen + // is that we understate the true minimum length + currentLen = forwardedLength.elementAti(loc+1); + } else { + // Forward jump. Propagate the current min length to the target loc of the jump. + U_ASSERT(jmpDest <= end+1); + if (forwardedLength.elementAti(jmpDest) > currentLen) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + break; + + case URX_BACKTRACK: + { + // Back-tracks are kind of like a branch, except that the min length was + // propagated already, by the state save. + currentLen = forwardedLength.elementAti(loc+1); + } + break; + + + case URX_STATE_SAVE: + { + // State Save, for forward jumps, propagate the current minimum. + // of the state save. + int32_t jmpDest = URX_VAL(op); + if (jmpDest > loc) { + if (currentLen < forwardedLength.elementAti(jmpDest)) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + break; + + + case URX_STRING: + { + loc++; + int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + currentLen = safeIncrement(currentLen, URX_VAL(stringLenOp)); + } + break; + + + case URX_STRING_I: + { + loc++; + // TODO: with full case folding, matching input text may be shorter than + // the string we have here. More smarts could put some bounds on it. + // Assume a min length of one for now. A min length of zero causes + // optimization failures for a pattern like "string"+ + // currentLen += URX_VAL(stringLenOp); + currentLen = safeIncrement(currentLen, 1); + } + break; + + case URX_CTR_INIT: + case URX_CTR_INIT_NG: + { + // Loop Init Ops. + // If the min loop count == 0 + // move loc forwards to the end of the loop, skipping over the body. + // If the min count is > 0, + // continue normal processing of the body of the loop. + int32_t loopEndLoc = (int32_t)fRXPat->fCompiledPat->elementAti(loc+1); + loopEndLoc = URX_VAL(loopEndLoc); + int32_t minLoopCount = (int32_t)fRXPat->fCompiledPat->elementAti(loc+2); + if (minLoopCount == 0) { + loc = loopEndLoc; + } else { + loc+=3; // Skips over operands of CTR_INIT + } + } + break; + + + case URX_CTR_LOOP: + case URX_CTR_LOOP_NG: + // Loop ops. + // The jump is conditional, backwards only. + break; + + case URX_LOOP_SR_I: + case URX_LOOP_DOT_I: + case URX_LOOP_C: + // More loop ops. These state-save to themselves. + // don't change the minimum match - could match nothing at all. + break; + + + case URX_LA_START: + case URX_LB_START: + { + // Look-around. Scan forward until the matching look-ahead end, + // without processing the look-around block. This is overly pessimistic for look-ahead, + // it assumes that the look-ahead match might be zero-length. + // TODO: Positive lookahead could recursively do the block, then continue + // with the longer of the block or the value coming in. Ticket 6060 + int32_t depth = (opType == URX_LA_START? 2: 1); + for (;;) { + loc++; + op = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + if (URX_TYPE(op) == URX_LA_START) { + // The boilerplate for look-ahead includes two LA_END instructions, + // Depth will be decremented by each one when it is seen. + depth += 2; + } + if (URX_TYPE(op) == URX_LB_START) { + depth++; + } + if (URX_TYPE(op) == URX_LA_END) { + depth--; + if (depth == 0) { + break; + } + } + if (URX_TYPE(op)==URX_LBN_END) { + depth--; + if (depth == 0) { + break; + } + } + if (URX_TYPE(op) == URX_STATE_SAVE) { + // Need this because neg lookahead blocks will FAIL to outside + // of the block. + int32_t jmpDest = URX_VAL(op); + if (jmpDest > loc) { + if (currentLen < forwardedLength.elementAti(jmpDest)) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } + } + U_ASSERT(loc <= end); + } + } + break; + + case URX_LA_END: + case URX_LB_CONT: + case URX_LB_END: + case URX_LBN_CONT: + case URX_LBN_END: + // Only come here if the matching URX_LA_START or URX_LB_START was not in the + // range being sized, which happens when measuring size of look-behind blocks. + break; + + default: + UPRV_UNREACHABLE_EXIT; + } + + } + + // We have finished walking through the ops. Check whether some forward jump + // propagated a shorter length to location end+1. + if (forwardedLength.elementAti(end+1) < currentLen) { + currentLen = forwardedLength.elementAti(end+1); + U_ASSERT(currentLen>=0 && currentLen < INT32_MAX); + } + + return currentLen; +} + +//------------------------------------------------------------------------------ +// +// maxMatchLength Calculate the length of the longest string that could +// match the specified pattern. +// Length is in 16 bit code units, not code points. +// +// The calculated length may not be exact. The returned +// value may be longer than the actual maximum; it must +// never be shorter. +// +// start, end: the range of the pattern to check. +// end is inclusive. +// +//------------------------------------------------------------------------------ +int32_t RegexCompile::maxMatchLength(int32_t start, int32_t end) { + if (U_FAILURE(*fStatus)) { + return 0; + } + U_ASSERT(start <= end); + U_ASSERT(end < fRXPat->fCompiledPat->size()); + + int32_t loc; + int32_t op; + int32_t opType; + int32_t currentLen = 0; + UVector32 forwardedLength(end+1, *fStatus); + forwardedLength.setSize(end+1); + + for (loc=start; loc<=end; loc++) { + forwardedLength.setElementAt(0, loc); + } + + for (loc = start; loc<=end; loc++) { + op = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + opType = URX_TYPE(op); + + // The loop is advancing linearly through the pattern. + // If the op we are now at was the destination of a branch in the pattern, + // and that path has a longer maximum length than the current accumulated value, + // replace the current accumulated value. + if (forwardedLength.elementAti(loc) > currentLen) { + currentLen = forwardedLength.elementAti(loc); + } + + switch (opType) { + // Ops that don't change the total length matched + case URX_RESERVED_OP: + case URX_END: + case URX_STRING_LEN: + case URX_NOP: + case URX_START_CAPTURE: + case URX_END_CAPTURE: + case URX_BACKSLASH_B: + case URX_BACKSLASH_BU: + case URX_BACKSLASH_G: + case URX_BACKSLASH_Z: + case URX_CARET: + case URX_DOLLAR: + case URX_DOLLAR_M: + case URX_DOLLAR_D: + case URX_DOLLAR_MD: + case URX_RELOC_OPRND: + case URX_STO_INP_LOC: + case URX_CARET_M: + case URX_CARET_M_UNIX: + + case URX_STO_SP: // Setup for atomic or possessive blocks. Doesn't change what can match. + case URX_LD_SP: + + case URX_LB_END: + case URX_LB_CONT: + case URX_LBN_CONT: + case URX_LBN_END: + break; + + + // Ops that increase that cause an unbounded increase in the length + // of a matched string, or that increase it a hard to characterize way. + // Call the max length unbounded, and stop further checking. + case URX_BACKREF: // BackRef. Must assume that it might be a zero length match + case URX_BACKREF_I: + case URX_BACKSLASH_X: // Grapheme Cluster. Minimum is 1, max unbounded. + currentLen = INT32_MAX; + break; + + + // Ops that match a max of one character (possibly two 16 bit code units.) + // + case URX_STATIC_SETREF: + case URX_STAT_SETREF_N: + case URX_SETREF: + case URX_BACKSLASH_D: + case URX_BACKSLASH_H: + case URX_BACKSLASH_R: + case URX_BACKSLASH_V: + case URX_ONECHAR_I: + case URX_DOTANY_ALL: + case URX_DOTANY: + case URX_DOTANY_UNIX: + currentLen = safeIncrement(currentLen, 2); + break; + + // Single literal character. Increase current max length by one or two, + // depending on whether the char is in the supplementary range. + case URX_ONECHAR: + currentLen = safeIncrement(currentLen, 1); + if (URX_VAL(op) > 0x10000) { + currentLen = safeIncrement(currentLen, 1); + } + break; + + // Jumps. + // + case URX_JMP: + case URX_JMPX: + case URX_JMP_SAV: + case URX_JMP_SAV_X: + { + int32_t jmpDest = URX_VAL(op); + if (jmpDest < loc) { + // Loop of some kind. Max match length is unbounded. + currentLen = INT32_MAX; + } else { + // Forward jump. Propagate the current min length to the target loc of the jump. + if (forwardedLength.elementAti(jmpDest) < currentLen) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + currentLen = 0; + } + } + break; + + case URX_BACKTRACK: + // back-tracks are kind of like a branch, except that the max length was + // propagated already, by the state save. + currentLen = forwardedLength.elementAti(loc+1); + break; + + + case URX_STATE_SAVE: + { + // State Save, for forward jumps, propagate the current minimum. + // of the state save. + // For backwards jumps, they create a loop, maximum + // match length is unbounded. + int32_t jmpDest = URX_VAL(op); + if (jmpDest > loc) { + if (currentLen > forwardedLength.elementAti(jmpDest)) { + forwardedLength.setElementAt(currentLen, jmpDest); + } + } else { + currentLen = INT32_MAX; + } + } + break; + + + + + case URX_STRING: + { + loc++; + int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + currentLen = safeIncrement(currentLen, URX_VAL(stringLenOp)); + break; + } + + case URX_STRING_I: + // TODO: This code assumes that any user string that matches will be no longer + // than our compiled string, with case insensitive matching. + // Our compiled string has been case-folded already. + // + // Any matching user string will have no more code points than our + // compiled (folded) string. Folding may add code points, but + // not remove them. + // + // There is a potential problem if a supplemental code point + // case-folds to a BMP code point. In this case our compiled string + // could be shorter (in code units) than a matching user string. + // + // At this time (Unicode 6.1) there are no such characters, and this case + // is not being handled. A test, intltest regex/Bug9283, will fail if + // any problematic characters are added to Unicode. + // + // If this happens, we can make a set of the BMP chars that the + // troublesome supplementals fold to, scan our string, and bump the + // currentLen one extra for each that is found. + // + { + loc++; + int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + currentLen = safeIncrement(currentLen, URX_VAL(stringLenOp)); + } + break; + + case URX_CTR_INIT: + case URX_CTR_INIT_NG: + // For Loops, recursively call this function on the pattern for the loop body, + // then multiply the result by the maximum loop count. + { + int32_t loopEndLoc = URX_VAL(fRXPat->fCompiledPat->elementAti(loc+1)); + if (loopEndLoc == loc+4) { + // Loop has an empty body. No affect on max match length. + // Continue processing with code after the loop end. + loc = loopEndLoc; + break; + } + + int32_t maxLoopCount = static_cast(fRXPat->fCompiledPat->elementAti(loc+3)); + if (maxLoopCount == -1) { + // Unbounded Loop. No upper bound on match length. + currentLen = INT32_MAX; + break; + } + + U_ASSERT(loopEndLoc >= loc+4); + int64_t blockLen = maxMatchLength(loc+4, loopEndLoc-1); // Recursive call. + int64_t updatedLen = (int64_t)currentLen + blockLen * maxLoopCount; + if (updatedLen >= INT32_MAX) { + currentLen = INT32_MAX; + break; + } + currentLen = (int32_t)updatedLen; + loc = loopEndLoc; + break; + } + + case URX_CTR_LOOP: + case URX_CTR_LOOP_NG: + // These opcodes will be skipped over by code for URX_CTR_INIT. + // We shouldn't encounter them here. + UPRV_UNREACHABLE_EXIT; + + case URX_LOOP_SR_I: + case URX_LOOP_DOT_I: + case URX_LOOP_C: + // For anything to do with loops, make the match length unbounded. + currentLen = INT32_MAX; + break; + + + + case URX_LA_START: + case URX_LA_END: + // Look-ahead. Just ignore, treat the look-ahead block as if + // it were normal pattern. Gives a too-long match length, + // but good enough for now. + break; + + // End of look-ahead ops should always be consumed by the processing at + // the URX_LA_START op. + // UPRV_UNREACHABLE_EXIT; + + case URX_LB_START: + { + // Look-behind. Scan forward until the matching look-around end, + // without processing the look-behind block. + int32_t dataLoc = URX_VAL(op); + for (loc = loc + 1; loc <= end; ++loc) { + op = (int32_t)fRXPat->fCompiledPat->elementAti(loc); + int32_t opType = URX_TYPE(op); + if ((opType == URX_LA_END || opType == URX_LBN_END) && (URX_VAL(op) == dataLoc)) { + break; + } + } + U_ASSERT(loc <= end); + } + break; + + default: + UPRV_UNREACHABLE_EXIT; + } + + + if (currentLen == INT32_MAX) { + // The maximum length is unbounded. + // Stop further processing of the pattern. + break; + } + + } + return currentLen; + +} + + +//------------------------------------------------------------------------------ +// +// stripNOPs Remove any NOP operations from the compiled pattern code. +// Extra NOPs are inserted for some constructs during the initial +// code generation to provide locations that may be patched later. +// Many end up unneeded, and are removed by this function. +// +// In order to minimize the number of passes through the pattern, +// back-reference fixup is also performed here (adjusting +// back-reference operands to point to the correct frame offsets). +// +//------------------------------------------------------------------------------ +void RegexCompile::stripNOPs() { + + if (U_FAILURE(*fStatus)) { + return; + } + + int32_t end = fRXPat->fCompiledPat->size(); + UVector32 deltas(end, *fStatus); + + // Make a first pass over the code, computing the amount that things + // will be offset at each location in the original code. + int32_t loc; + int32_t d = 0; + for (loc=0; locfCompiledPat->elementAti(loc); + if (URX_TYPE(op) == URX_NOP) { + d++; + } + } + + UnicodeString caseStringBuffer; + + // Make a second pass over the code, removing the NOPs by moving following + // code up, and patching operands that refer to code locations that + // are being moved. The array of offsets from the first step is used + // to compute the new operand values. + int32_t src; + int32_t dst = 0; + for (src=0; srcfCompiledPat->elementAti(src); + int32_t opType = URX_TYPE(op); + switch (opType) { + case URX_NOP: + break; + + case URX_STATE_SAVE: + case URX_JMP: + case URX_CTR_LOOP: + case URX_CTR_LOOP_NG: + case URX_RELOC_OPRND: + case URX_JMPX: + case URX_JMP_SAV: + case URX_JMP_SAV_X: + // These are instructions with operands that refer to code locations. + { + int32_t operandAddress = URX_VAL(op); + U_ASSERT(operandAddress>=0 && operandAddressfCompiledPat->setElementAt(op, dst); + dst++; + break; + } + + case URX_BACKREF: + case URX_BACKREF_I: + { + int32_t where = URX_VAL(op); + if (where > fRXPat->fGroupMap->size()) { + error(U_REGEX_INVALID_BACK_REF); + break; + } + where = fRXPat->fGroupMap->elementAti(where-1); + op = buildOp(opType, where); + fRXPat->fCompiledPat->setElementAt(op, dst); + dst++; + + fRXPat->fNeedsAltInput = true; + break; + } + case URX_RESERVED_OP: + case URX_RESERVED_OP_N: + case URX_BACKTRACK: + case URX_END: + case URX_ONECHAR: + case URX_STRING: + case URX_STRING_LEN: + case URX_START_CAPTURE: + case URX_END_CAPTURE: + case URX_STATIC_SETREF: + case URX_STAT_SETREF_N: + case URX_SETREF: + case URX_DOTANY: + case URX_FAIL: + case URX_BACKSLASH_B: + case URX_BACKSLASH_BU: + case URX_BACKSLASH_G: + case URX_BACKSLASH_X: + case URX_BACKSLASH_Z: + case URX_DOTANY_ALL: + case URX_BACKSLASH_D: + case URX_CARET: + case URX_DOLLAR: + case URX_CTR_INIT: + case URX_CTR_INIT_NG: + case URX_DOTANY_UNIX: + case URX_STO_SP: + case URX_LD_SP: + case URX_STO_INP_LOC: + case URX_LA_START: + case URX_LA_END: + case URX_ONECHAR_I: + case URX_STRING_I: + case URX_DOLLAR_M: + case URX_CARET_M: + case URX_CARET_M_UNIX: + case URX_LB_START: + case URX_LB_CONT: + case URX_LB_END: + case URX_LBN_CONT: + case URX_LBN_END: + case URX_LOOP_SR_I: + case URX_LOOP_DOT_I: + case URX_LOOP_C: + case URX_DOLLAR_D: + case URX_DOLLAR_MD: + case URX_BACKSLASH_H: + case URX_BACKSLASH_R: + case URX_BACKSLASH_V: + // These instructions are unaltered by the relocation. + fRXPat->fCompiledPat->setElementAt(op, dst); + dst++; + break; + + default: + // Some op is unaccounted for. + UPRV_UNREACHABLE_EXIT; + } + } + + fRXPat->fCompiledPat->setSize(dst); +} + + + + +//------------------------------------------------------------------------------ +// +// Error Report a rule parse error. +// Only report it if no previous error has been recorded. +// +//------------------------------------------------------------------------------ +void RegexCompile::error(UErrorCode e) { + if (U_SUCCESS(*fStatus) || e == U_MEMORY_ALLOCATION_ERROR) { + *fStatus = e; + // Hmm. fParseErr (UParseError) line & offset fields are int32_t in public + // API (see common/unicode/parseerr.h), while fLineNum and fCharNum are + // int64_t. If the values of the latter are out of range for the former, + // set them to the appropriate "field not supported" values. + if (fLineNum > 0x7FFFFFFF) { + fParseErr->line = 0; + fParseErr->offset = -1; + } else if (fCharNum > 0x7FFFFFFF) { + fParseErr->line = (int32_t)fLineNum; + fParseErr->offset = -1; + } else { + fParseErr->line = (int32_t)fLineNum; + fParseErr->offset = (int32_t)fCharNum; + } + + UErrorCode status = U_ZERO_ERROR; // throwaway status for extracting context + + // Fill in the context. + // Note: extractBetween() pins supplied indices to the string bounds. + uprv_memset(fParseErr->preContext, 0, sizeof(fParseErr->preContext)); + uprv_memset(fParseErr->postContext, 0, sizeof(fParseErr->postContext)); + utext_extract(fRXPat->fPattern, fScanIndex-U_PARSE_CONTEXT_LEN+1, fScanIndex, fParseErr->preContext, U_PARSE_CONTEXT_LEN, &status); + utext_extract(fRXPat->fPattern, fScanIndex, fScanIndex+U_PARSE_CONTEXT_LEN-1, fParseErr->postContext, U_PARSE_CONTEXT_LEN, &status); + } +} + + +// +// Assorted Unicode character constants. +// Numeric because there is no portable way to enter them as literals. +// (Think EBCDIC). +// +static const char16_t chCR = 0x0d; // New lines, for terminating comments. +static const char16_t chLF = 0x0a; // Line Feed +static const char16_t chPound = 0x23; // '#', introduces a comment. +static const char16_t chDigit0 = 0x30; // '0' +static const char16_t chDigit7 = 0x37; // '9' +static const char16_t chColon = 0x3A; // ':' +static const char16_t chE = 0x45; // 'E' +static const char16_t chQ = 0x51; // 'Q' +//static const char16_t chN = 0x4E; // 'N' +static const char16_t chP = 0x50; // 'P' +static const char16_t chBackSlash = 0x5c; // '\' introduces a char escape +//static const char16_t chLBracket = 0x5b; // '[' +static const char16_t chRBracket = 0x5d; // ']' +static const char16_t chUp = 0x5e; // '^' +static const char16_t chLowerP = 0x70; +static const char16_t chLBrace = 0x7b; // '{' +static const char16_t chRBrace = 0x7d; // '}' +static const char16_t chNEL = 0x85; // NEL newline variant +static const char16_t chLS = 0x2028; // Unicode Line Separator + + +//------------------------------------------------------------------------------ +// +// nextCharLL Low Level Next Char from the regex pattern. +// Get a char from the string, keep track of input position +// for error reporting. +// +//------------------------------------------------------------------------------ +UChar32 RegexCompile::nextCharLL() { + UChar32 ch; + + if (fPeekChar != -1) { + ch = fPeekChar; + fPeekChar = -1; + return ch; + } + + // assume we're already in the right place + ch = UTEXT_NEXT32(fRXPat->fPattern); + if (ch == U_SENTINEL) { + return ch; + } + + if (ch == chCR || + ch == chNEL || + ch == chLS || + (ch == chLF && fLastChar != chCR)) { + // Character is starting a new line. Bump up the line number, and + // reset the column to 0. + fLineNum++; + fCharNum=0; + } + else { + // Character is not starting a new line. Except in the case of a + // LF following a CR, increment the column position. + if (ch != chLF) { + fCharNum++; + } + } + fLastChar = ch; + return ch; +} + +//------------------------------------------------------------------------------ +// +// peekCharLL Low Level Character Scanning, sneak a peek at the next +// character without actually getting it. +// +//------------------------------------------------------------------------------ +UChar32 RegexCompile::peekCharLL() { + if (fPeekChar == -1) { + fPeekChar = nextCharLL(); + } + return fPeekChar; +} + + +//------------------------------------------------------------------------------ +// +// nextChar for pattern scanning. At this level, we handle stripping +// out comments and processing some backslash character escapes. +// The rest of the pattern grammar is handled at the next level up. +// +//------------------------------------------------------------------------------ +void RegexCompile::nextChar(RegexPatternChar &c) { + tailRecursion: + fScanIndex = UTEXT_GETNATIVEINDEX(fRXPat->fPattern); + c.fChar = nextCharLL(); + c.fQuoted = false; + + if (fQuoteMode) { + c.fQuoted = true; + if ((c.fChar==chBackSlash && peekCharLL()==chE && ((fModeFlags & UREGEX_LITERAL) == 0)) || + c.fChar == (UChar32)-1) { + fQuoteMode = false; // Exit quote mode, + nextCharLL(); // discard the E + // nextChar(c); // recurse to get the real next char + goto tailRecursion; // Note: fuzz testing produced testcases that + // resulted in stack overflow here. + } + } + else if (fInBackslashQuote) { + // The current character immediately follows a '\' + // Don't check for any further escapes, just return it as-is. + // Don't set c.fQuoted, because that would prevent the state machine from + // dispatching on the character. + fInBackslashQuote = false; + } + else + { + // We are not in a \Q quoted region \E of the source. + // + if (fModeFlags & UREGEX_COMMENTS) { + // + // We are in free-spacing and comments mode. + // Scan through any white space and comments, until we + // reach a significant character or the end of input. + for (;;) { + if (c.fChar == (UChar32)-1) { + break; // End of Input + } + if (c.fChar == chPound && fEOLComments) { + // Start of a comment. Consume the rest of it, until EOF or a new line + for (;;) { + c.fChar = nextCharLL(); + if (c.fChar == (UChar32)-1 || // EOF + c.fChar == chCR || + c.fChar == chLF || + c.fChar == chNEL || + c.fChar == chLS) { + break; + } + } + } + // TODO: check what Java & Perl do with non-ASCII white spaces. Ticket 6061. + if (PatternProps::isWhiteSpace(c.fChar) == false) { + break; + } + c.fChar = nextCharLL(); + } + } + + // + // check for backslash escaped characters. + // + if (c.fChar == chBackSlash) { + int64_t pos = UTEXT_GETNATIVEINDEX(fRXPat->fPattern); + if (RegexStaticSets::gStaticSets->fUnescapeCharSet.contains(peekCharLL())) { + // + // A '\' sequence that is handled by ICU's standard unescapeAt function. + // Includes \uxxxx, \n, \r, many others. + // Return the single equivalent character. + // + nextCharLL(); // get & discard the peeked char. + c.fQuoted = true; + + if (UTEXT_FULL_TEXT_IN_CHUNK(fRXPat->fPattern, fPatternLength)) { + int32_t endIndex = (int32_t)pos; + c.fChar = u_unescapeAt(uregex_ucstr_unescape_charAt, &endIndex, (int32_t)fPatternLength, (void *)fRXPat->fPattern->chunkContents); + + if (endIndex == pos) { + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + } + fCharNum += endIndex - pos; + UTEXT_SETNATIVEINDEX(fRXPat->fPattern, endIndex); + } else { + int32_t offset = 0; + struct URegexUTextUnescapeCharContext context = U_REGEX_UTEXT_UNESCAPE_CONTEXT(fRXPat->fPattern); + + UTEXT_SETNATIVEINDEX(fRXPat->fPattern, pos); + c.fChar = u_unescapeAt(uregex_utext_unescape_charAt, &offset, INT32_MAX, &context); + + if (offset == 0) { + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + } else if (context.lastOffset == offset) { + UTEXT_PREVIOUS32(fRXPat->fPattern); + } else if (context.lastOffset != offset-1) { + utext_moveIndex32(fRXPat->fPattern, offset - context.lastOffset - 1); + } + fCharNum += offset; + } + } + else if (peekCharLL() == chDigit0) { + // Octal Escape, using Java Regexp Conventions + // which are \0 followed by 1-3 octal digits. + // Different from ICU Unescape handling of Octal, which does not + // require the leading 0. + // Java also has the convention of only consuming 2 octal digits if + // the three digit number would be > 0xff + // + c.fChar = 0; + nextCharLL(); // Consume the initial 0. + int index; + for (index=0; index<3; index++) { + int32_t ch = peekCharLL(); + if (chchDigit7) { + if (index==0) { + // \0 is not followed by any octal digits. + error(U_REGEX_BAD_ESCAPE_SEQUENCE); + } + break; + } + c.fChar <<= 3; + c.fChar += ch&7; + if (c.fChar <= 255) { + nextCharLL(); + } else { + // The last digit made the number too big. Forget we saw it. + c.fChar >>= 3; + } + } + c.fQuoted = true; + } + else if (peekCharLL() == chQ) { + // "\Q" enter quote mode, which will continue until "\E" + fQuoteMode = true; + nextCharLL(); // discard the 'Q'. + // nextChar(c); // recurse to get the real next char. + goto tailRecursion; // Note: fuzz testing produced test cases that + // resulted in stack overflow here. + } + else + { + // We are in a '\' escape that will be handled by the state table scanner. + // Just return the backslash, but remember that the following char is to + // be taken literally. + fInBackslashQuote = true; + } + } + } + + // re-enable # to end-of-line comments, in case they were disabled. + // They are disabled by the parser upon seeing '(?', but this lasts for + // the fetching of the next character only. + fEOLComments = true; + + // putc(c.fChar, stdout); +} + + + +//------------------------------------------------------------------------------ +// +// scanNamedChar +// Get a UChar32 from a \N{UNICODE CHARACTER NAME} in the pattern. +// +// The scan position will be at the 'N'. On return +// the scan position should be just after the '}' +// +// Return the UChar32 +// +//------------------------------------------------------------------------------ +UChar32 RegexCompile::scanNamedChar() { + if (U_FAILURE(*fStatus)) { + return 0; + } + + nextChar(fC); + if (fC.fChar != chLBrace) { + error(U_REGEX_PROPERTY_SYNTAX); + return 0; + } + + UnicodeString charName; + for (;;) { + nextChar(fC); + if (fC.fChar == chRBrace) { + break; + } + if (fC.fChar == -1) { + error(U_REGEX_PROPERTY_SYNTAX); + return 0; + } + charName.append(fC.fChar); + } + + char name[100]; + if (!uprv_isInvariantUString(charName.getBuffer(), charName.length()) || + (uint32_t)charName.length()>=sizeof(name)) { + // All Unicode character names have only invariant characters. + // The API to get a character, given a name, accepts only char *, forcing us to convert, + // which requires this error check + error(U_REGEX_PROPERTY_SYNTAX); + return 0; + } + charName.extract(0, charName.length(), name, sizeof(name), US_INV); + + UChar32 theChar = u_charFromName(U_UNICODE_CHAR_NAME, name, fStatus); + if (U_FAILURE(*fStatus)) { + error(U_REGEX_PROPERTY_SYNTAX); + } + + nextChar(fC); // Continue overall regex pattern processing with char after the '}' + return theChar; +} + +//------------------------------------------------------------------------------ +// +// scanProp Construct a UnicodeSet from the text at the current scan +// position, which will be of the form \p{whaterver} +// +// The scan position will be at the 'p' or 'P'. On return +// the scan position should be just after the '}' +// +// Return a UnicodeSet, constructed from the \P pattern, +// or nullptr if the pattern is invalid. +// +//------------------------------------------------------------------------------ +UnicodeSet *RegexCompile::scanProp() { + UnicodeSet *uset = nullptr; + + if (U_FAILURE(*fStatus)) { + return nullptr; + } + (void)chLowerP; // Suppress compiler unused variable warning. + U_ASSERT(fC.fChar == chLowerP || fC.fChar == chP); + UBool negated = (fC.fChar == chP); + + UnicodeString propertyName; + nextChar(fC); + if (fC.fChar != chLBrace) { + error(U_REGEX_PROPERTY_SYNTAX); + return nullptr; + } + for (;;) { + nextChar(fC); + if (fC.fChar == chRBrace) { + break; + } + if (fC.fChar == -1) { + // Hit the end of the input string without finding the closing '}' + error(U_REGEX_PROPERTY_SYNTAX); + return nullptr; + } + propertyName.append(fC.fChar); + } + uset = createSetForProperty(propertyName, negated); + nextChar(fC); // Move input scan to position following the closing '}' + return uset; +} + +//------------------------------------------------------------------------------ +// +// scanPosixProp Construct a UnicodeSet from the text at the current scan +// position, which is expected be of the form [:property expression:] +// +// The scan position will be at the opening ':'. On return +// the scan position must be on the closing ']' +// +// Return a UnicodeSet constructed from the pattern, +// or nullptr if this is not a valid POSIX-style set expression. +// If not a property expression, restore the initial scan position +// (to the opening ':') +// +// Note: the opening '[:' is not sufficient to guarantee that +// this is a [:property:] expression. +// [:'+=,] is a perfectly good ordinary set expression that +// happens to include ':' as one of its characters. +// +//------------------------------------------------------------------------------ +UnicodeSet *RegexCompile::scanPosixProp() { + UnicodeSet *uset = nullptr; + + if (U_FAILURE(*fStatus)) { + return nullptr; + } + + U_ASSERT(fC.fChar == chColon); + + // Save the scanner state. + // TODO: move this into the scanner, with the state encapsulated in some way. Ticket 6062 + int64_t savedScanIndex = fScanIndex; + int64_t savedNextIndex = UTEXT_GETNATIVEINDEX(fRXPat->fPattern); + UBool savedQuoteMode = fQuoteMode; + UBool savedInBackslashQuote = fInBackslashQuote; + UBool savedEOLComments = fEOLComments; + int64_t savedLineNum = fLineNum; + int64_t savedCharNum = fCharNum; + UChar32 savedLastChar = fLastChar; + UChar32 savedPeekChar = fPeekChar; + RegexPatternChar savedfC = fC; + + // Scan for a closing ]. A little tricky because there are some perverse + // edge cases possible. "[:abc\Qdef:] \E]" is a valid non-property expression, + // ending on the second closing ]. + + UnicodeString propName; + UBool negated = false; + + // Check for and consume the '^' in a negated POSIX property, e.g. [:^Letter:] + nextChar(fC); + if (fC.fChar == chUp) { + negated = true; + nextChar(fC); + } + + // Scan for the closing ":]", collecting the property name along the way. + UBool sawPropSetTerminator = false; + for (;;) { + propName.append(fC.fChar); + nextChar(fC); + if (fC.fQuoted || fC.fChar == -1) { + // Escaped characters or end of input - either says this isn't a [:Property:] + break; + } + if (fC.fChar == chColon) { + nextChar(fC); + if (fC.fChar == chRBracket) { + sawPropSetTerminator = true; + } + break; + } + } + + if (sawPropSetTerminator) { + uset = createSetForProperty(propName, negated); + } + else + { + // No closing ":]". + // Restore the original scan position. + // The main scanner will retry the input as a normal set expression, + // not a [:Property:] expression. + fScanIndex = savedScanIndex; + fQuoteMode = savedQuoteMode; + fInBackslashQuote = savedInBackslashQuote; + fEOLComments = savedEOLComments; + fLineNum = savedLineNum; + fCharNum = savedCharNum; + fLastChar = savedLastChar; + fPeekChar = savedPeekChar; + fC = savedfC; + UTEXT_SETNATIVEINDEX(fRXPat->fPattern, savedNextIndex); + } + return uset; +} + +static inline void addIdentifierIgnorable(UnicodeSet *set, UErrorCode& ec) { + set->add(0, 8).add(0x0e, 0x1b).add(0x7f, 0x9f); + addCategory(set, U_GC_CF_MASK, ec); +} + +// +// Create a Unicode Set from a Unicode Property expression. +// This is common code underlying both \p{...} and [:...:] expressions. +// Includes trying the Java "properties" that aren't supported as +// normal ICU UnicodeSet properties +// +UnicodeSet *RegexCompile::createSetForProperty(const UnicodeString &propName, UBool negated) { + + if (U_FAILURE(*fStatus)) { + return nullptr; + } + LocalPointer set; + UErrorCode status = U_ZERO_ERROR; + + do { // non-loop, exists to allow breaks from the block. + // + // First try the property as we received it + // + UnicodeString setExpr; + uint32_t usetFlags = 0; + setExpr.append(u"[\\p{", -1); + setExpr.append(propName); + setExpr.append(u"}]", -1); + if (fModeFlags & UREGEX_CASE_INSENSITIVE) { + usetFlags |= USET_CASE_INSENSITIVE; + } + set.adoptInsteadAndCheckErrorCode(new UnicodeSet(setExpr, usetFlags, nullptr, status), status); + if (U_SUCCESS(status) || status == U_MEMORY_ALLOCATION_ERROR) { + break; + } + + // + // The incoming property wasn't directly recognized by ICU. + + // Check [:word:] and [:all:]. These are not recognized as a properties by ICU UnicodeSet. + // Java accepts 'word' with mixed case. + // Java accepts 'all' only in all lower case. + + status = U_ZERO_ERROR; + if (propName.caseCompare(u"word", -1, 0) == 0) { + set.adoptInsteadAndCheckErrorCode( + RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET].cloneAsThawed(), status); + break; + } + if (propName.compare(u"all", -1) == 0) { + set.adoptInsteadAndCheckErrorCode(new UnicodeSet(0, 0x10ffff), status); + break; + } + + + // Do Java InBlock expressions + // + UnicodeString mPropName = propName; + if (mPropName.startsWith(u"In", 2) && mPropName.length() >= 3) { + status = U_ZERO_ERROR; + set.adoptInsteadAndCheckErrorCode(new UnicodeSet(), status); + if (U_FAILURE(status)) { + break; + } + UnicodeString blockName(mPropName, 2); // Property with the leading "In" removed. + set->applyPropertyAlias(UnicodeString(u"Block"), blockName, status); + break; + } + + // Check for the Java form "IsBooleanPropertyValue", which we will recast + // as "BooleanPropertyValue". The property value can be either a + // a General Category or a Script Name. + + if (propName.startsWith(u"Is", 2) && propName.length()>=3) { + mPropName.remove(0, 2); // Strip the "Is" + if (mPropName.indexOf(u'=') >= 0) { + // Reject any "Is..." property expression containing an '=', that is, + // any non-binary property expression. + status = U_REGEX_PROPERTY_SYNTAX; + break; + } + + if (mPropName.caseCompare(u"assigned", -1, 0) == 0) { + mPropName.setTo(u"unassigned", -1); + negated = !negated; + } else if (mPropName.caseCompare(u"TitleCase", -1, 0) == 0) { + mPropName.setTo(u"Titlecase_Letter", -1); + } + + mPropName.insert(0, u"[\\p{", -1); + mPropName.append(u"}]", -1); + set.adoptInsteadAndCheckErrorCode(new UnicodeSet(mPropName, *fStatus), status); + + if (U_SUCCESS(status) && !set->isEmpty() && (usetFlags & USET_CASE_INSENSITIVE)) { + set->closeOver(USET_CASE_INSENSITIVE); + } + break; + + } + + if (propName.startsWith(u"java", -1)) { + status = U_ZERO_ERROR; + set.adoptInsteadAndCheckErrorCode(new UnicodeSet(), status); + if (U_FAILURE(status)) { + break; + } + // + // Try the various Java specific properties. + // These all begin with "java" + // + if (propName.compare(u"javaDefined", -1) == 0) { + addCategory(set.getAlias(), U_GC_CN_MASK, status); + set->complement(); + } + else if (propName.compare(u"javaDigit", -1) == 0) { + addCategory(set.getAlias(), U_GC_ND_MASK, status); + } + else if (propName.compare(u"javaIdentifierIgnorable", -1) == 0) { + addIdentifierIgnorable(set.getAlias(), status); + } + else if (propName.compare(u"javaISOControl", -1) == 0) { + set->add(0, 0x1F).add(0x7F, 0x9F); + } + else if (propName.compare(u"javaJavaIdentifierPart", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + addCategory(set.getAlias(), U_GC_SC_MASK, status); + addCategory(set.getAlias(), U_GC_PC_MASK, status); + addCategory(set.getAlias(), U_GC_ND_MASK, status); + addCategory(set.getAlias(), U_GC_NL_MASK, status); + addCategory(set.getAlias(), U_GC_MC_MASK, status); + addCategory(set.getAlias(), U_GC_MN_MASK, status); + addIdentifierIgnorable(set.getAlias(), status); + } + else if (propName.compare(u"javaJavaIdentifierStart", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + addCategory(set.getAlias(), U_GC_NL_MASK, status); + addCategory(set.getAlias(), U_GC_SC_MASK, status); + addCategory(set.getAlias(), U_GC_PC_MASK, status); + } + else if (propName.compare(u"javaLetter", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + } + else if (propName.compare(u"javaLetterOrDigit", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + addCategory(set.getAlias(), U_GC_ND_MASK, status); + } + else if (propName.compare(u"javaLowerCase", -1) == 0) { + addCategory(set.getAlias(), U_GC_LL_MASK, status); + } + else if (propName.compare(u"javaMirrored", -1) == 0) { + set->applyIntPropertyValue(UCHAR_BIDI_MIRRORED, 1, status); + } + else if (propName.compare(u"javaSpaceChar", -1) == 0) { + addCategory(set.getAlias(), U_GC_Z_MASK, status); + } + else if (propName.compare(u"javaSupplementaryCodePoint", -1) == 0) { + set->add(0x10000, UnicodeSet::MAX_VALUE); + } + else if (propName.compare(u"javaTitleCase", -1) == 0) { + addCategory(set.getAlias(), U_GC_LT_MASK, status); + } + else if (propName.compare(u"javaUnicodeIdentifierStart", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + addCategory(set.getAlias(), U_GC_NL_MASK, status); + } + else if (propName.compare(u"javaUnicodeIdentifierPart", -1) == 0) { + addCategory(set.getAlias(), U_GC_L_MASK, status); + addCategory(set.getAlias(), U_GC_PC_MASK, status); + addCategory(set.getAlias(), U_GC_ND_MASK, status); + addCategory(set.getAlias(), U_GC_NL_MASK, status); + addCategory(set.getAlias(), U_GC_MC_MASK, status); + addCategory(set.getAlias(), U_GC_MN_MASK, status); + addIdentifierIgnorable(set.getAlias(), status); + } + else if (propName.compare(u"javaUpperCase", -1) == 0) { + addCategory(set.getAlias(), U_GC_LU_MASK, status); + } + else if (propName.compare(u"javaValidCodePoint", -1) == 0) { + set->add(0, UnicodeSet::MAX_VALUE); + } + else if (propName.compare(u"javaWhitespace", -1) == 0) { + addCategory(set.getAlias(), U_GC_Z_MASK, status); + set->removeAll(UnicodeSet().add(0xa0).add(0x2007).add(0x202f)); + set->add(9, 0x0d).add(0x1c, 0x1f); + } else { + status = U_REGEX_PROPERTY_SYNTAX; + } + + if (U_SUCCESS(status) && !set->isEmpty() && (usetFlags & USET_CASE_INSENSITIVE)) { + set->closeOver(USET_CASE_INSENSITIVE); + } + break; + } + + // Unrecognized property. ICU didn't like it as it was, and none of the Java compatibility + // extensions matched it. + status = U_REGEX_PROPERTY_SYNTAX; + } while (false); // End of do loop block. Code above breaks out of the block on success or hard failure. + + if (U_SUCCESS(status)) { + // ICU 70 adds emoji properties of strings, but as long as Java does not say how to + // deal with properties of strings and character classes with strings, we ignore them. + // Just in case something downstream might stumble over the strings, + // we remove them from the set. + // Note that when we support strings, the complement of a property (as with \P) + // should be implemented as .complement().removeAllStrings() (code point complement). + set->removeAllStrings(); + U_ASSERT(set.isValid()); + if (negated) { + set->complement(); + } + return set.orphan(); + } else { + if (status == U_ILLEGAL_ARGUMENT_ERROR) { + status = U_REGEX_PROPERTY_SYNTAX; + } + error(status); + return nullptr; + } +} + + +// +// SetEval Part of the evaluation of [set expressions]. +// Perform any pending (stacked) operations with precedence +// equal or greater to that of the next operator encountered +// in the expression. +// +void RegexCompile::setEval(int32_t nextOp) { + UnicodeSet *rightOperand = nullptr; + UnicodeSet *leftOperand = nullptr; + for (;;) { + U_ASSERT(fSetOpStack.empty()==false); + int32_t pendingSetOperation = fSetOpStack.peeki(); + if ((pendingSetOperation&0xffff0000) < (nextOp&0xffff0000)) { + break; + } + fSetOpStack.popi(); + U_ASSERT(fSetStack.empty() == false); + rightOperand = (UnicodeSet *)fSetStack.peek(); + // ICU 70 adds emoji properties of strings, but createSetForProperty() removes all strings + // (see comments there). + // We also do not yet support string literals in character classes, + // so there should not be any strings. + // Note that when we support strings, the complement of a set (as with ^ or \P) + // should be implemented as .complement().removeAllStrings() (code point complement). + U_ASSERT(!rightOperand->hasStrings()); + switch (pendingSetOperation) { + case setNegation: + rightOperand->complement(); + break; + case setCaseClose: + // TODO: need a simple close function. Ticket 6065 + rightOperand->closeOver(USET_CASE_INSENSITIVE); + rightOperand->removeAllStrings(); + break; + case setDifference1: + case setDifference2: + fSetStack.pop(); + leftOperand = (UnicodeSet *)fSetStack.peek(); + leftOperand->removeAll(*rightOperand); + delete rightOperand; + break; + case setIntersection1: + case setIntersection2: + fSetStack.pop(); + leftOperand = (UnicodeSet *)fSetStack.peek(); + leftOperand->retainAll(*rightOperand); + delete rightOperand; + break; + case setUnion: + fSetStack.pop(); + leftOperand = (UnicodeSet *)fSetStack.peek(); + leftOperand->addAll(*rightOperand); + delete rightOperand; + break; + default: + UPRV_UNREACHABLE_EXIT; + } + } + } + +void RegexCompile::setPushOp(int32_t op) { + setEval(op); + fSetOpStack.push(op, *fStatus); + LocalPointer lpSet(new UnicodeSet(), *fStatus); + fSetStack.push(lpSet.orphan(), *fStatus); +} + +U_NAMESPACE_END +#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS + diff --git a/intl/icu/source/i18n/regexcmp.h b/intl/icu/source/i18n/regexcmp.h new file mode 100644 index 0000000000..81ac9e5178 --- /dev/null +++ b/intl/icu/source/i18n/regexcmp.h @@ -0,0 +1,234 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// regexcmp.h +// +// Copyright (C) 2002-2016, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains declarations for the class RegexCompile +// +// This class is internal to the regular expression implementation. +// For the public Regular Expression API, see the file "unicode/regex.h" +// + + +#ifndef RBBISCAN_H +#define RBBISCAN_H + +#include "unicode/utypes.h" +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + +#include "unicode/parseerr.h" +#include "unicode/uniset.h" +#include "unicode/uobject.h" +#include "unicode/utext.h" +#include "uhash.h" +#include "uvector.h" +#include "uvectr32.h" + + + +U_NAMESPACE_BEGIN + + +//-------------------------------------------------------------------------------- +// +// class RegexCompile Contains the regular expression compiler. +// +//-------------------------------------------------------------------------------- +class RegexPattern; + + +class U_I18N_API RegexCompile : public UMemory { +public: + + enum { + kStackSize = 100 // The size of the state stack for + }; // pattern parsing. Corresponds roughly + // to the depth of parentheses nesting + // that is allowed in the rules. + + struct RegexPatternChar { + UChar32 fChar; + UBool fQuoted; + }; + + RegexCompile(RegexPattern *rp, UErrorCode &e); + + void compile(const UnicodeString &pat, UParseError &pp, UErrorCode &e); + void compile(UText *pat, UParseError &pp, UErrorCode &e); + + + virtual ~RegexCompile(); + + void nextChar(RegexPatternChar &c); // Get the next char from the input stream. + + + // Categories of parentheses in pattern. + // The category is saved in the compile-time parentheses stack frame, and + // determines the code to be generated when the matching close ) is encountered. + enum EParenClass { + plain = -1, // No special handling + capturing = -2, + atomic = -3, + lookAhead = -4, + negLookAhead = -5, + flags = -6, + lookBehind = -7, + lookBehindN = -8 + }; + +private: + + + UBool doParseActions(int32_t a); + void error(UErrorCode e); // error reporting convenience function. + + UChar32 nextCharLL(); + UChar32 peekCharLL(); + UnicodeSet *scanProp(); + UnicodeSet *scanPosixProp(); + void handleCloseParen(); + int32_t blockTopLoc(UBool reserve); // Locate a position in the compiled pattern + // at the top of the just completed block + // or operation, and optionally ensure that + // there is space to add an opcode there. + void compileSet(UnicodeSet *theSet); // Generate the compiled pattern for + // a reference to a UnicodeSet. + void compileInterval(int32_t InitOp, // Generate the code for a {min,max} quantifier. + int32_t LoopOp); + UBool compileInlineInterval(); // Generate inline code for a {min,max} quantifier + void literalChar(UChar32 c); // Compile a literal char + void fixLiterals(UBool split=false); // Generate code for pending literal characters. + void insertOp(int32_t where); // Open up a slot for a new op in the + // generated code at the specified location. + void appendOp(int32_t op); // Append a new op to the compiled pattern. + void appendOp(int32_t type, int32_t val); // Build & append a new op to the compiled pattern. + int32_t buildOp(int32_t type, int32_t val); // Construct a new pcode instruction. + int32_t allocateData(int32_t size); // Allocate space in the matcher data area. + // Return index of the newly allocated data. + int32_t allocateStackData(int32_t size); // Allocate space in the match back-track stack frame. + // Return offset index in the frame. + int32_t minMatchLength(int32_t start, + int32_t end); + int32_t maxMatchLength(int32_t start, + int32_t end); + void matchStartType(); + void stripNOPs(); + + void setEval(int32_t op); + void setPushOp(int32_t op); + UChar32 scanNamedChar(); + UnicodeSet *createSetForProperty(const UnicodeString &propName, UBool negated); + +public: // Public for testing only. + static void U_EXPORT2 findCaseInsensitiveStarters(UChar32 c, UnicodeSet *starterChars); +private: + + + UErrorCode *fStatus; + RegexPattern *fRXPat; + UParseError *fParseErr; + + // + // Data associated with low level character scanning + // + int64_t fScanIndex; // Index of current character being processed + // in the rule input string. + UBool fQuoteMode; // Scan is in a \Q...\E quoted region + UBool fInBackslashQuote; // Scan is between a '\' and the following char. + UBool fEOLComments; // When scan is just after '(?', inhibit #... to + // end of line comments, in favor of (?#...) comments. + int64_t fLineNum; // Line number in input file. + int64_t fCharNum; // Char position within the line. + UChar32 fLastChar; // Previous char, needed to count CR-LF + // as a single line, not two. + UChar32 fPeekChar; // Saved char, if we've scanned ahead. + + + RegexPatternChar fC; // Current char for parse state machine + // processing. + + uint16_t fStack[kStackSize]; // State stack, holds state pushes + int32_t fStackPtr; // and pops as specified in the state + // transition rules. + + // + // Data associated with the generation of the pcode for the match engine + // + int32_t fModeFlags; // Match Flags. (Case Insensitive, etc.) + // Always has high bit (31) set so that flag values + // on the paren stack are distinguished from relocatable + // pcode addresses. + int32_t fNewModeFlags; // New flags, while compiling (?i, holds state + // until last flag is scanned. + UBool fSetModeFlag; // true for (?ismx, false for (?-ismx + + UnicodeString fLiteralChars; // Literal chars or strings from the pattern are accumulated here. + // Once completed, meaning that some non-literal pattern + // construct is encountered, the appropriate opcodes + // to match the literal will be generated, and this + // string will be cleared. + + int64_t fPatternLength; // Length of the input pattern string. + + UVector32 fParenStack; // parentheses stack. Each frame consists of + // the positions of compiled pattern operations + // needing fixup, followed by negative value. The + // first entry in each frame is the position of the + // spot reserved for use when a quantifier + // needs to add a SAVE at the start of a (block) + // The negative value (-1, -2,...) indicates + // the kind of paren that opened the frame. Some + // need special handling on close. + + + int32_t fMatchOpenParen; // The position in the compiled pattern + // of the slot reserved for a state save + // at the start of the most recently processed + // parenthesized block. Updated when processing + // a close to the location for the corresponding open. + + int32_t fMatchCloseParen; // The position in the pattern of the first + // location after the most recently processed + // parenthesized block. + + int32_t fIntervalLow; // {lower, upper} interval quantifier values. + int32_t fIntervalUpper; // Placed here temporarily, when pattern is + // initially scanned. Each new interval + // encountered overwrites these values. + // -1 for the upper interval value means none + // was specified (unlimited occurrences.) + + UStack fSetStack; // Stack of UnicodeSets, used while evaluating + // (at compile time) set expressions within + // the pattern. + UStack fSetOpStack; // Stack of pending set operators (&&, --, union) + + UChar32 fLastSetLiteral; // The last single code point added to a set. + // needed when "-y" is scanned, and we need + // to turn "x-y" into a range. + + UnicodeString *fCaptureName; // Named Capture, the group name is built up + // in this string while being scanned. +}; + +// Constant values to be pushed onto fSetOpStack while scanning & evaluating [set expressions] +// The high 16 bits are the operator precedence, and the low 16 are a code for the operation itself. + +enum SetOperations { + setStart = 0 << 16 | 1, + setEnd = 1 << 16 | 2, + setNegation = 2 << 16 | 3, + setCaseClose = 2 << 16 | 9, + setDifference2 = 3 << 16 | 4, // '--' set difference operator + setIntersection2 = 3 << 16 | 5, // '&&' set intersection operator + setUnion = 4 << 16 | 6, // implicit union of adjacent items + setDifference1 = 4 << 16 | 7, // '-', single dash difference op, for compatibility with old UnicodeSet. + setIntersection1 = 4 << 16 | 8 // '&', single amp intersection op, for compatibility with old UnicodeSet. + }; + +U_NAMESPACE_END +#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS +#endif // RBBISCAN_H diff --git a/intl/icu/source/i18n/regexcst.h b/intl/icu/source/i18n/regexcst.h new file mode 100644 index 0000000000..a475b6b363 --- /dev/null +++ b/intl/icu/source/i18n/regexcst.h @@ -0,0 +1,570 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +//--------------------------------------------------------------------------------- +// +// Generated Header File. Do not edit by hand. +// This file contains the state table for the ICU Regular Expression Pattern Parser +// It is generated by the Perl script "regexcst.pl" from +// the rule parser state definitions file "regexcst.txt". +// +// Copyright (C) 2002-2016 International Business Machines Corporation +// and others. All rights reserved. +// +//--------------------------------------------------------------------------------- +#ifndef RBBIRPT_H +#define RBBIRPT_H + +#include "unicode/utypes.h" + +U_NAMESPACE_BEGIN +// +// Character classes for regex pattern scanning. +// + static const uint8_t kRuleSet_digit_char = 128; + static const uint8_t kRuleSet_ascii_letter = 129; + static const uint8_t kRuleSet_rule_char = 130; + constexpr uint32_t kRuleSet_count = 131-128; + +enum Regex_PatternParseAction { + doSetBackslash_D, + doBackslashh, + doBackslashH, + doSetLiteralEscaped, + doOpenLookAheadNeg, + doCompleteNamedBackRef, + doPatStart, + doBackslashS, + doBackslashD, + doNGStar, + doNOP, + doBackslashX, + doSetLiteral, + doContinueNamedCapture, + doBackslashG, + doBackslashR, + doSetBegin, + doSetBackslash_v, + doPossessivePlus, + doPerlInline, + doBackslashZ, + doSetAddAmp, + doSetBeginDifference1, + doIntervalError, + doSetNegate, + doIntervalInit, + doSetIntersection2, + doPossessiveInterval, + doRuleError, + doBackslashW, + doContinueNamedBackRef, + doOpenNonCaptureParen, + doExit, + doSetNamedChar, + doSetBackslash_V, + doConditionalExpr, + doEscapeError, + doBadOpenParenType, + doPossessiveStar, + doSetAddDash, + doEscapedLiteralChar, + doSetBackslash_w, + doIntervalUpperDigit, + doBackslashv, + doSetBackslash_S, + doSetNoCloseError, + doSetProp, + doBackslashB, + doSetEnd, + doSetRange, + doMatchModeParen, + doPlus, + doBackslashV, + doSetMatchMode, + doBackslashz, + doSetNamedRange, + doOpenLookBehindNeg, + doInterval, + doBadNamedCapture, + doBeginMatchMode, + doBackslashd, + doPatFinish, + doNamedChar, + doNGPlus, + doSetDifference2, + doSetBackslash_H, + doCloseParen, + doDotAny, + doOpenCaptureParen, + doEnterQuoteMode, + doOpenAtomicParen, + doBadModeFlag, + doSetBackslash_d, + doSetFinish, + doProperty, + doBeginNamedBackRef, + doBackRef, + doOpt, + doDollar, + doBeginNamedCapture, + doNGInterval, + doSetOpError, + doSetPosixProp, + doSetBeginIntersection1, + doBackslashb, + doSetBeginUnion, + doIntevalLowerDigit, + doSetBackslash_h, + doStar, + doMatchMode, + doBackslashA, + doOpenLookBehind, + doPossessiveOpt, + doOrOperator, + doBackslashw, + doBackslashs, + doLiteralChar, + doSuppressComments, + doCaret, + doIntervalSame, + doNGOpt, + doOpenLookAhead, + doSetBackslash_W, + doMismatchedParenErr, + doSetBackslash_s, + rbbiLastAction}; + +//------------------------------------------------------------------------------- +// +// RegexTableEl represents the structure of a row in the transition table +// for the pattern parser state machine. +//------------------------------------------------------------------------------- +struct RegexTableEl { + Regex_PatternParseAction fAction; + uint8_t fCharClass; // 0-127: an individual ASCII character + // 128-255: character class index + uint8_t fNextState; // 0-250: normal next-state numbers + // 255: pop next-state from stack. + uint8_t fPushState; + UBool fNextChar; +}; + +static const struct RegexTableEl gRuleParseStateTable[] = { + {doNOP, 0, 0, 0, true} + , {doPatStart, 255, 2,0, false} // 1 start + , {doLiteralChar, 254, 14,0, true} // 2 term + , {doLiteralChar, 130, 14,0, true} // 3 + , {doSetBegin, 91 /* [ */, 123, 205, true} // 4 + , {doNOP, 40 /* ( */, 27,0, true} // 5 + , {doDotAny, 46 /* . */, 14,0, true} // 6 + , {doCaret, 94 /* ^ */, 14,0, true} // 7 + , {doDollar, 36 /* $ */, 14,0, true} // 8 + , {doNOP, 92 /* \ */, 89,0, true} // 9 + , {doOrOperator, 124 /* | */, 2,0, true} // 10 + , {doCloseParen, 41 /* ) */, 255,0, true} // 11 + , {doPatFinish, 253, 2,0, false} // 12 + , {doRuleError, 255, 206,0, false} // 13 + , {doNOP, 42 /* * */, 68,0, true} // 14 expr-quant + , {doNOP, 43 /* + */, 71,0, true} // 15 + , {doNOP, 63 /* ? */, 74,0, true} // 16 + , {doIntervalInit, 123 /* { */, 77,0, true} // 17 + , {doNOP, 40 /* ( */, 23,0, true} // 18 + , {doNOP, 255, 20,0, false} // 19 + , {doOrOperator, 124 /* | */, 2,0, true} // 20 expr-cont + , {doCloseParen, 41 /* ) */, 255,0, true} // 21 + , {doNOP, 255, 2,0, false} // 22 + , {doSuppressComments, 63 /* ? */, 25,0, true} // 23 open-paren-quant + , {doNOP, 255, 27,0, false} // 24 + , {doNOP, 35 /* # */, 50, 14, true} // 25 open-paren-quant2 + , {doNOP, 255, 29,0, false} // 26 + , {doSuppressComments, 63 /* ? */, 29,0, true} // 27 open-paren + , {doOpenCaptureParen, 255, 2, 14, false} // 28 + , {doOpenNonCaptureParen, 58 /* : */, 2, 14, true} // 29 open-paren-extended + , {doOpenAtomicParen, 62 /* > */, 2, 14, true} // 30 + , {doOpenLookAhead, 61 /* = */, 2, 20, true} // 31 + , {doOpenLookAheadNeg, 33 /* ! */, 2, 20, true} // 32 + , {doNOP, 60 /* < */, 46,0, true} // 33 + , {doNOP, 35 /* # */, 50, 2, true} // 34 + , {doBeginMatchMode, 105 /* i */, 53,0, false} // 35 + , {doBeginMatchMode, 100 /* d */, 53,0, false} // 36 + , {doBeginMatchMode, 109 /* m */, 53,0, false} // 37 + , {doBeginMatchMode, 115 /* s */, 53,0, false} // 38 + , {doBeginMatchMode, 117 /* u */, 53,0, false} // 39 + , {doBeginMatchMode, 119 /* w */, 53,0, false} // 40 + , {doBeginMatchMode, 120 /* x */, 53,0, false} // 41 + , {doBeginMatchMode, 45 /* - */, 53,0, false} // 42 + , {doConditionalExpr, 40 /* ( */, 206,0, true} // 43 + , {doPerlInline, 123 /* { */, 206,0, true} // 44 + , {doBadOpenParenType, 255, 206,0, false} // 45 + , {doOpenLookBehind, 61 /* = */, 2, 20, true} // 46 open-paren-lookbehind + , {doOpenLookBehindNeg, 33 /* ! */, 2, 20, true} // 47 + , {doBeginNamedCapture, 129, 64,0, false} // 48 + , {doBadOpenParenType, 255, 206,0, false} // 49 + , {doNOP, 41 /* ) */, 255,0, true} // 50 paren-comment + , {doMismatchedParenErr, 253, 206,0, false} // 51 + , {doNOP, 255, 50,0, true} // 52 + , {doMatchMode, 105 /* i */, 53,0, true} // 53 paren-flag + , {doMatchMode, 100 /* d */, 53,0, true} // 54 + , {doMatchMode, 109 /* m */, 53,0, true} // 55 + , {doMatchMode, 115 /* s */, 53,0, true} // 56 + , {doMatchMode, 117 /* u */, 53,0, true} // 57 + , {doMatchMode, 119 /* w */, 53,0, true} // 58 + , {doMatchMode, 120 /* x */, 53,0, true} // 59 + , {doMatchMode, 45 /* - */, 53,0, true} // 60 + , {doSetMatchMode, 41 /* ) */, 2,0, true} // 61 + , {doMatchModeParen, 58 /* : */, 2, 14, true} // 62 + , {doBadModeFlag, 255, 206,0, false} // 63 + , {doContinueNamedCapture, 129, 64,0, true} // 64 named-capture + , {doContinueNamedCapture, 128, 64,0, true} // 65 + , {doOpenCaptureParen, 62 /* > */, 2, 14, true} // 66 + , {doBadNamedCapture, 255, 206,0, false} // 67 + , {doNGStar, 63 /* ? */, 20,0, true} // 68 quant-star + , {doPossessiveStar, 43 /* + */, 20,0, true} // 69 + , {doStar, 255, 20,0, false} // 70 + , {doNGPlus, 63 /* ? */, 20,0, true} // 71 quant-plus + , {doPossessivePlus, 43 /* + */, 20,0, true} // 72 + , {doPlus, 255, 20,0, false} // 73 + , {doNGOpt, 63 /* ? */, 20,0, true} // 74 quant-opt + , {doPossessiveOpt, 43 /* + */, 20,0, true} // 75 + , {doOpt, 255, 20,0, false} // 76 + , {doNOP, 128, 79,0, false} // 77 interval-open + , {doIntervalError, 255, 206,0, false} // 78 + , {doIntevalLowerDigit, 128, 79,0, true} // 79 interval-lower + , {doNOP, 44 /* , */, 83,0, true} // 80 + , {doIntervalSame, 125 /* } */, 86,0, true} // 81 + , {doIntervalError, 255, 206,0, false} // 82 + , {doIntervalUpperDigit, 128, 83,0, true} // 83 interval-upper + , {doNOP, 125 /* } */, 86,0, true} // 84 + , {doIntervalError, 255, 206,0, false} // 85 + , {doNGInterval, 63 /* ? */, 20,0, true} // 86 interval-type + , {doPossessiveInterval, 43 /* + */, 20,0, true} // 87 + , {doInterval, 255, 20,0, false} // 88 + , {doBackslashA, 65 /* A */, 2,0, true} // 89 backslash + , {doBackslashB, 66 /* B */, 2,0, true} // 90 + , {doBackslashb, 98 /* b */, 2,0, true} // 91 + , {doBackslashd, 100 /* d */, 14,0, true} // 92 + , {doBackslashD, 68 /* D */, 14,0, true} // 93 + , {doBackslashG, 71 /* G */, 2,0, true} // 94 + , {doBackslashh, 104 /* h */, 14,0, true} // 95 + , {doBackslashH, 72 /* H */, 14,0, true} // 96 + , {doNOP, 107 /* k */, 115,0, true} // 97 + , {doNamedChar, 78 /* N */, 14,0, false} // 98 + , {doProperty, 112 /* p */, 14,0, false} // 99 + , {doProperty, 80 /* P */, 14,0, false} // 100 + , {doBackslashR, 82 /* R */, 14,0, true} // 101 + , {doEnterQuoteMode, 81 /* Q */, 2,0, true} // 102 + , {doBackslashS, 83 /* S */, 14,0, true} // 103 + , {doBackslashs, 115 /* s */, 14,0, true} // 104 + , {doBackslashv, 118 /* v */, 14,0, true} // 105 + , {doBackslashV, 86 /* V */, 14,0, true} // 106 + , {doBackslashW, 87 /* W */, 14,0, true} // 107 + , {doBackslashw, 119 /* w */, 14,0, true} // 108 + , {doBackslashX, 88 /* X */, 14,0, true} // 109 + , {doBackslashZ, 90 /* Z */, 2,0, true} // 110 + , {doBackslashz, 122 /* z */, 2,0, true} // 111 + , {doBackRef, 128, 14,0, true} // 112 + , {doEscapeError, 253, 206,0, false} // 113 + , {doEscapedLiteralChar, 255, 14,0, true} // 114 + , {doBeginNamedBackRef, 60 /* < */, 117,0, true} // 115 named-backref + , {doBadNamedCapture, 255, 206,0, false} // 116 + , {doContinueNamedBackRef, 129, 119,0, true} // 117 named-backref-2 + , {doBadNamedCapture, 255, 206,0, false} // 118 + , {doContinueNamedBackRef, 129, 119,0, true} // 119 named-backref-3 + , {doContinueNamedBackRef, 128, 119,0, true} // 120 + , {doCompleteNamedBackRef, 62 /* > */, 14,0, true} // 121 + , {doBadNamedCapture, 255, 206,0, false} // 122 + , {doSetNegate, 94 /* ^ */, 126,0, true} // 123 set-open + , {doSetPosixProp, 58 /* : */, 128,0, false} // 124 + , {doNOP, 255, 126,0, false} // 125 + , {doSetLiteral, 93 /* ] */, 141,0, true} // 126 set-open2 + , {doNOP, 255, 131,0, false} // 127 + , {doSetEnd, 93 /* ] */, 255,0, true} // 128 set-posix + , {doNOP, 58 /* : */, 131,0, false} // 129 + , {doRuleError, 255, 206,0, false} // 130 + , {doSetEnd, 93 /* ] */, 255,0, true} // 131 set-start + , {doSetBeginUnion, 91 /* [ */, 123, 148, true} // 132 + , {doNOP, 92 /* \ */, 191,0, true} // 133 + , {doNOP, 45 /* - */, 137,0, true} // 134 + , {doNOP, 38 /* & */, 139,0, true} // 135 + , {doSetLiteral, 255, 141,0, true} // 136 + , {doRuleError, 45 /* - */, 206,0, false} // 137 set-start-dash + , {doSetAddDash, 255, 141,0, false} // 138 + , {doRuleError, 38 /* & */, 206,0, false} // 139 set-start-amp + , {doSetAddAmp, 255, 141,0, false} // 140 + , {doSetEnd, 93 /* ] */, 255,0, true} // 141 set-after-lit + , {doSetBeginUnion, 91 /* [ */, 123, 148, true} // 142 + , {doNOP, 45 /* - */, 178,0, true} // 143 + , {doNOP, 38 /* & */, 169,0, true} // 144 + , {doNOP, 92 /* \ */, 191,0, true} // 145 + , {doSetNoCloseError, 253, 206,0, false} // 146 + , {doSetLiteral, 255, 141,0, true} // 147 + , {doSetEnd, 93 /* ] */, 255,0, true} // 148 set-after-set + , {doSetBeginUnion, 91 /* [ */, 123, 148, true} // 149 + , {doNOP, 45 /* - */, 171,0, true} // 150 + , {doNOP, 38 /* & */, 166,0, true} // 151 + , {doNOP, 92 /* \ */, 191,0, true} // 152 + , {doSetNoCloseError, 253, 206,0, false} // 153 + , {doSetLiteral, 255, 141,0, true} // 154 + , {doSetEnd, 93 /* ] */, 255,0, true} // 155 set-after-range + , {doSetBeginUnion, 91 /* [ */, 123, 148, true} // 156 + , {doNOP, 45 /* - */, 174,0, true} // 157 + , {doNOP, 38 /* & */, 176,0, true} // 158 + , {doNOP, 92 /* \ */, 191,0, true} // 159 + , {doSetNoCloseError, 253, 206,0, false} // 160 + , {doSetLiteral, 255, 141,0, true} // 161 + , {doSetBeginUnion, 91 /* [ */, 123, 148, true} // 162 set-after-op + , {doSetOpError, 93 /* ] */, 206,0, false} // 163 + , {doNOP, 92 /* \ */, 191,0, true} // 164 + , {doSetLiteral, 255, 141,0, true} // 165 + , {doSetBeginIntersection1, 91 /* [ */, 123, 148, true} // 166 set-set-amp + , {doSetIntersection2, 38 /* & */, 162,0, true} // 167 + , {doSetAddAmp, 255, 141,0, false} // 168 + , {doSetIntersection2, 38 /* & */, 162,0, true} // 169 set-lit-amp + , {doSetAddAmp, 255, 141,0, false} // 170 + , {doSetBeginDifference1, 91 /* [ */, 123, 148, true} // 171 set-set-dash + , {doSetDifference2, 45 /* - */, 162,0, true} // 172 + , {doSetAddDash, 255, 141,0, false} // 173 + , {doSetDifference2, 45 /* - */, 162,0, true} // 174 set-range-dash + , {doSetAddDash, 255, 141,0, false} // 175 + , {doSetIntersection2, 38 /* & */, 162,0, true} // 176 set-range-amp + , {doSetAddAmp, 255, 141,0, false} // 177 + , {doSetDifference2, 45 /* - */, 162,0, true} // 178 set-lit-dash + , {doSetAddDash, 91 /* [ */, 141,0, false} // 179 + , {doSetAddDash, 93 /* ] */, 141,0, false} // 180 + , {doNOP, 92 /* \ */, 183,0, true} // 181 + , {doSetRange, 255, 155,0, true} // 182 + , {doSetOpError, 115 /* s */, 206,0, false} // 183 set-lit-dash-escape + , {doSetOpError, 83 /* S */, 206,0, false} // 184 + , {doSetOpError, 119 /* w */, 206,0, false} // 185 + , {doSetOpError, 87 /* W */, 206,0, false} // 186 + , {doSetOpError, 100 /* d */, 206,0, false} // 187 + , {doSetOpError, 68 /* D */, 206,0, false} // 188 + , {doSetNamedRange, 78 /* N */, 155,0, false} // 189 + , {doSetRange, 255, 155,0, true} // 190 + , {doSetProp, 112 /* p */, 148,0, false} // 191 set-escape + , {doSetProp, 80 /* P */, 148,0, false} // 192 + , {doSetNamedChar, 78 /* N */, 141,0, false} // 193 + , {doSetBackslash_s, 115 /* s */, 155,0, true} // 194 + , {doSetBackslash_S, 83 /* S */, 155,0, true} // 195 + , {doSetBackslash_w, 119 /* w */, 155,0, true} // 196 + , {doSetBackslash_W, 87 /* W */, 155,0, true} // 197 + , {doSetBackslash_d, 100 /* d */, 155,0, true} // 198 + , {doSetBackslash_D, 68 /* D */, 155,0, true} // 199 + , {doSetBackslash_h, 104 /* h */, 155,0, true} // 200 + , {doSetBackslash_H, 72 /* H */, 155,0, true} // 201 + , {doSetBackslash_v, 118 /* v */, 155,0, true} // 202 + , {doSetBackslash_V, 86 /* V */, 155,0, true} // 203 + , {doSetLiteralEscaped, 255, 141,0, true} // 204 + , {doSetFinish, 255, 14,0, false} // 205 set-finish + , {doExit, 255, 206,0, true} // 206 errorDeath + }; +static const char * const RegexStateNames[] = { 0, + "start", + "term", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "expr-quant", + 0, + 0, + 0, + 0, + 0, + "expr-cont", + 0, + 0, + "open-paren-quant", + 0, + "open-paren-quant2", + 0, + "open-paren", + 0, + "open-paren-extended", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "open-paren-lookbehind", + 0, + 0, + 0, + "paren-comment", + 0, + 0, + "paren-flag", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "named-capture", + 0, + 0, + 0, + "quant-star", + 0, + 0, + "quant-plus", + 0, + 0, + "quant-opt", + 0, + 0, + "interval-open", + 0, + "interval-lower", + 0, + 0, + 0, + "interval-upper", + 0, + 0, + "interval-type", + 0, + 0, + "backslash", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "named-backref", + 0, + "named-backref-2", + 0, + "named-backref-3", + 0, + 0, + 0, + "set-open", + 0, + 0, + "set-open2", + 0, + "set-posix", + 0, + 0, + "set-start", + 0, + 0, + 0, + 0, + 0, + "set-start-dash", + 0, + "set-start-amp", + 0, + "set-after-lit", + 0, + 0, + 0, + 0, + 0, + 0, + "set-after-set", + 0, + 0, + 0, + 0, + 0, + 0, + "set-after-range", + 0, + 0, + 0, + 0, + 0, + 0, + "set-after-op", + 0, + 0, + 0, + "set-set-amp", + 0, + 0, + "set-lit-amp", + 0, + "set-set-dash", + 0, + 0, + "set-range-dash", + 0, + "set-range-amp", + 0, + "set-lit-dash", + 0, + 0, + 0, + 0, + "set-lit-dash-escape", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "set-escape", + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "set-finish", + "errorDeath", + 0}; + +U_NAMESPACE_END +#endif diff --git a/intl/icu/source/i18n/regexcst.pl b/intl/icu/source/i18n/regexcst.pl new file mode 100755 index 0000000000..24596d4122 --- /dev/null +++ b/intl/icu/source/i18n/regexcst.pl @@ -0,0 +1,335 @@ +#!/usr/bin/perl +# Copyright (C) 2016 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html +# ******************************************************************** +# * COPYRIGHT: +# * Copyright (c) 2002-2016, International Business Machines Corporation and +# * others. All Rights Reserved. +# ******************************************************************** +# +# regexcst.pl +# Compile the regular expression parser state table data into initialized C data. +# Usage: +# cd icu4c/source/i18n +# perl regexcst.pl < regexcst.txt > regexcst.h +# +# The output file, regexcst.h, is included by some of the .cpp regex +# implementation files. This perl script is NOT run as part +# of a normal ICU build. It is run by hand when needed, and the +# regexcst.h generated file is put back into the source code repository. +# +# See regexcst.txt for a description of the input format for this script. +# +# This script is derived from rbbicst.pl, which peforms the same function +# for the Rule Based Break Iterator Rule Parser. Perhaps they could be +# merged? +# + + +$num_states = 1; # Always the state number for the line being compiled. +$line_num = 0; # The line number in the input file. + +$states{"pop"} = 255; # Add the "pop" to the list of defined state names. + # This prevents any state from being labelled with "pop", + # and resolves references to "pop" in the next state field. + +line_loop: while (<>) { + chomp(); + $line = $_; + @fields = split(); + $line_num++; + + # Remove # comments, which are any fields beginning with a #, plus all + # that follow on the line. + for ($i=0; $i<@fields; $i++) { + if ($fields[$i] =~ /^#/) { + @fields = @fields[0 .. $i-1]; + last; + } + } + # ignore blank lines, and those with no fields left after stripping comments.. + if (@fields == 0) { + next; + } + + # + # State Label: handling. + # Does the first token end with a ":"? If so, it's the name of a state. + # Put in a hash, together with the current state number, + # so that we can later look up the number from the name. + # + if (@fields[0] =~ /.*:$/) { + $state_name = @fields[0]; + $state_name =~ s/://; # strip off the colon from the state name. + + if ($states{$state_name} != 0) { + print " rbbicst: at line $line-num duplicate definition of state $state_name\n"; + } + $states{$state_name} = $num_states; + $stateNames[$num_states] = $state_name; + + # if the label was the only thing on this line, go on to the next line, + # otherwise assume that a state definition is on the same line and fall through. + if (@fields == 1) { + next line_loop; + } + shift @fields; # shift off label field in preparation + # for handling the rest of the line. + } + + # + # State Transition line. + # syntax is this, + # character [n] target-state [^push-state] [function-name] + # where + # [something] is an optional something + # character is either a single quoted character e.g. '[' + # or a name of a character class, e.g. white_space + # + + $state_line_num[$num_states] = $line_num; # remember line number with each state + # so we can make better error messages later. + # + # First field, character class or literal character for this transition. + # + if ($fields[0] =~ /^'.'$/) { + # We've got a quoted literal character. + $state_literal_chars[$num_states] = $fields[0]; + $state_literal_chars[$num_states] =~ s/'//g; + } else { + # We've got the name of a character class. + $state_char_class[$num_states] = $fields[0]; + if ($fields[0] =~ /[\W]/) { + print " rbbicsts: at line $line_num, bad character literal or character class name.\n"; + print " scanning $fields[0]\n"; + exit(-1); + } + } + shift @fields; + + # + # do the 'n' flag + # + $state_flag[$num_states] = "false"; + if ($fields[0] eq "n") { + $state_flag[$num_states] = "true"; + shift @fields; + } + + # + # do the destination state. + # + $state_dest_state[$num_states] = $fields[0]; + if ($fields[0] eq "") { + print " rbbicsts: at line $line_num, destination state missing.\n"; + exit(-1); + } + shift @fields; + + # + # do the push state, if present. + # + if ($fields[0] =~ /^\^/) { + $fields[0] =~ s/^\^//; + $state_push_state[$num_states] = $fields[0]; + if ($fields[0] eq "" ) { + print " rbbicsts: at line $line_num, expected state after ^ (no spaces).\n"; + exit(-1); + } + shift @fields; + } + + # + # Lastly, do the optional action name. + # + if ($fields[0] ne "") { + $state_func_name[$num_states] = $fields[0]; + shift @fields; + } + + # + # There should be no fields left on the line at this point. + # + if (@fields > 0) { + print " rbbicsts: at line $line_num, unexpected extra stuff on input line.\n"; + print " scanning $fields[0]\n"; + } + $num_states++; +} + +# +# We've read in the whole file, now go back and output the +# C source code for the state transition table. +# +# We read all states first, before writing anything, so that the state numbers +# for the destination states are all available to be written. +# + +# +# Make hashes for the names of the character classes and +# for the names of the actions that appeared. +# +for ($state=1; $state < $num_states; $state++) { + if ($state_char_class[$state] ne "") { + if ($charClasses{$state_char_class[$state]} == 0) { + $charClasses{$state_char_class[$state]} = 1; + } + } + if ($state_func_name[$state] eq "") { + $state_func_name[$state] = "doNOP"; + } + if ($actions{$state_action_name[$state]} == 0) { + $actions{$state_func_name[$state]} = 1; + } +} + +# +# Check that all of the destination states have been defined +# +# +$states{"exit"} = 0; # Predefined state name, terminates state machine. +for ($state=1; $state<$num_states; $state++) { + if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") { + print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n"; + $errors++; + } + if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) { + print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n"; + $errors++; + } +} + +die if ($errors>0); + +print "// © 2016 and later: Unicode, Inc. and others.\n"; +print "// License & terms of use: http://www.unicode.org/copyright.html\n"; +print "//---------------------------------------------------------------------------------\n"; +print "//\n"; +print "// Generated Header File. Do not edit by hand.\n"; +print "// This file contains the state table for the ICU Regular Expression Pattern Parser\n"; +print "// It is generated by the Perl script \"regexcst.pl\" from\n"; +print "// the rule parser state definitions file \"regexcst.txt\".\n"; +print "//\n"; +print "// Copyright (C) 2002-2016 International Business Machines Corporation \n"; +print "// and others. All rights reserved. \n"; +print "//\n"; +print "//---------------------------------------------------------------------------------\n"; +print "#ifndef RBBIRPT_H\n"; +print "#define RBBIRPT_H\n"; +print "\n"; +print "#include \"unicode/utypes.h\"\n"; +print "\n"; +print "U_NAMESPACE_BEGIN\n"; + +# +# Emit the constants for indices of Unicode Sets +# Define one constant for each of the character classes encountered. +# At the same time, store the index corresponding to the set name back into hash. +# +print "//\n"; +print "// Character classes for regex pattern scanning.\n"; +print "//\n"; +$i = 128; # State Table values for Unicode char sets range from 128-250. + # Sets "default", "quoted", etc. get special handling. + # They have no corresponding UnicodeSet object in the state machine, + # but are handled by special case code. So we emit no reference + # to a UnicodeSet object to them here. +foreach $setName (keys %charClasses) { + if ($setName eq "default") { + $charClasses{$setName} = 255;} + elsif ($setName eq "quoted") { + $charClasses{$setName} = 254;} + elsif ($setName eq "eof") { + $charClasses{$setName} = 253;} + else { + # Normal character class. Fill in array with a ptr to the corresponding UnicodeSet in the state machine. + print " static const uint8_t kRuleSet_$setName = $i;\n"; + $charClasses{$setName} = $i; + $i++; + } +} +print " constexpr uint32_t kRuleSet_count = $i-128;"; +print "\n\n"; + +# +# Emit the enum for the actions to be performed. +# +print "enum Regex_PatternParseAction {\n"; +foreach $act (keys %actions) { + print " $act,\n"; +} +print " rbbiLastAction};\n\n"; + +# +# Emit the struct definition for transition table elements. +# +print "//-------------------------------------------------------------------------------\n"; +print "//\n"; +print "// RegexTableEl represents the structure of a row in the transition table\n"; +print "// for the pattern parser state machine.\n"; +print "//-------------------------------------------------------------------------------\n"; +print "struct RegexTableEl {\n"; +print " Regex_PatternParseAction fAction;\n"; +print " uint8_t fCharClass; // 0-127: an individual ASCII character\n"; +print " // 128-255: character class index\n"; +print " uint8_t fNextState; // 0-250: normal next-state numbers\n"; +print " // 255: pop next-state from stack.\n"; +print " uint8_t fPushState;\n"; +print " UBool fNextChar;\n"; +print "};\n\n"; + +# +# emit the state transition table +# +print "static const struct RegexTableEl gRuleParseStateTable[] = {\n"; +print " {doNOP, 0, 0, 0, true}\n"; # State 0 is a dummy. Real states start with index = 1. +for ($state=1; $state < $num_states; $state++) { + print " , {$state_func_name[$state],"; + if ($state_literal_chars[$state] ne "") { + $c = $state_literal_chars[$state]; + printf(" %d /* $c */,", ord($c)); # use numeric value, so EBCDIC machines are ok. + }else { + print " $charClasses{$state_char_class[$state]},"; + } + print " $states{$state_dest_state[$state]},"; + + # The push-state field is optional. If omitted, fill field with a zero, which flags + # the state machine that there is no push state. + if ($state_push_state[$state] eq "") { + print "0, "; + } else { + print " $states{$state_push_state[$state]},"; + } + print " $state_flag[$state]} "; + + # Put out a C++ comment showing the number (index) of this state row, + # and, if this is the first row of the table for this state, the state name. + print " // $state "; + if ($stateNames[$state] ne "") { + print " $stateNames[$state]"; + } + print "\n"; +}; +print " };\n"; + + +# +# emit a mapping array from state numbers to state names. +# +# This array is used for producing debugging output from the pattern parser. +# +print "static const char * const RegexStateNames[] = {"; +for ($state=0; $state<$num_states; $state++) { + if ($stateNames[$state] ne "") { + print " \"$stateNames[$state]\",\n"; + } else { + print " 0,\n"; + } +} +print " 0};\n\n"; + +print "U_NAMESPACE_END\n"; +print "#endif\n"; + + + diff --git a/intl/icu/source/i18n/regexcst.txt b/intl/icu/source/i18n/regexcst.txt new file mode 100644 index 0000000000..7e53578e24 --- /dev/null +++ b/intl/icu/source/i18n/regexcst.txt @@ -0,0 +1,505 @@ +# Copyright (C) 2016 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html +#***************************************************************************** +# +# Copyright (C) 2002-2015, International Business Machines Corporation and others. +# All Rights Reserved. +# +#***************************************************************************** +# +# file: regexcst.txt +# ICU Regular Expression Parser State Table +# +# This state table is used when reading and parsing a regular expression pattern +# The pattern parser uses a state machine; the data in this file define the +# state transitions that occur for each input character. +# +# *** This file defines the regex pattern grammar. This is it. +# *** The determination of what is accepted is here. +# +# This file is processed by a perl script "regexcst.pl" to produce initialized C arrays +# that are then built with the rule parser. +# + +# +# Here is the syntax of the state definitions in this file: +# +# +#StateName: +# input-char n next-state ^push-state action +# input-char n next-state ^push-state action +# | | | | | +# | | | | |--- action to be performed by state machine +# | | | | See function RBBIRuleScanner::doParseActions() +# | | | | +# | | | |--- Push this named state onto the state stack. +# | | | Later, when next state is specified as "pop", +# | | | the pushed state will become the current state. +# | | | +# | | |--- Transition to this state if the current input character matches the input +# | | character or char class in the left hand column. "pop" causes the next +# | | state to be popped from the state stack. +# | | +# | |--- When making the state transition specified on this line, advance to the next +# | character from the input only if 'n' appears here. +# | +# |--- Character or named character classes to test for. If the current character being scanned +# matches, perform the actions and go to the state specified on this line. +# The input character is tested sequentally, in the order written. The characters and +# character classes tested for do not need to be mutually exclusive. The first match wins. +# + + + + +# +# start state, scan position is at the beginning of the pattern. +# +start: + default term doPatStart + + + + +# +# term. At a position where we can accept the start most items in a pattern. +# +term: + quoted n expr-quant doLiteralChar + rule_char n expr-quant doLiteralChar + '[' n set-open ^set-finish doSetBegin + '(' n open-paren + '.' n expr-quant doDotAny + '^' n expr-quant doCaret + '$' n expr-quant doDollar + '\' n backslash + '|' n term doOrOperator + ')' n pop doCloseParen + eof term doPatFinish + default errorDeath doRuleError + + + +# +# expr-quant We've just finished scanning a term, now look for the optional +# trailing quantifier - *, +, ?, *?, etc. +# +expr-quant: + '*' n quant-star + '+' n quant-plus + '?' n quant-opt + '{' n interval-open doIntervalInit + '(' n open-paren-quant + default expr-cont + + +# +# expr-cont Expression, continuation. At a point where additional terms are +# allowed, but not required. No Quantifiers +# +expr-cont: + '|' n term doOrOperator + ')' n pop doCloseParen + default term + + +# +# open-paren-quant Special case handling for comments appearing before a quantifier, +# e.g. x(?#comment )* +# Open parens from expr-quant come here; anything but a (?# comment +# branches into the normal parenthesis sequence as quickly as possible. +# +open-paren-quant: + '?' n open-paren-quant2 doSuppressComments + default open-paren + +open-paren-quant2: + '#' n paren-comment ^expr-quant + default open-paren-extended + + +# +# open-paren We've got an open paren. We need to scan further to +# determine what kind of quantifier it is - plain (, (?:, (?>, or whatever. +# +open-paren: + '?' n open-paren-extended doSuppressComments + default term ^expr-quant doOpenCaptureParen + +open-paren-extended: + ':' n term ^expr-quant doOpenNonCaptureParen # (?: + '>' n term ^expr-quant doOpenAtomicParen # (?> + '=' n term ^expr-cont doOpenLookAhead # (?= + '!' n term ^expr-cont doOpenLookAheadNeg # (?! + '<' n open-paren-lookbehind + '#' n paren-comment ^term + 'i' paren-flag doBeginMatchMode + 'd' paren-flag doBeginMatchMode + 'm' paren-flag doBeginMatchMode + 's' paren-flag doBeginMatchMode + 'u' paren-flag doBeginMatchMode + 'w' paren-flag doBeginMatchMode + 'x' paren-flag doBeginMatchMode + '-' paren-flag doBeginMatchMode + '(' n errorDeath doConditionalExpr + '{' n errorDeath doPerlInline + default errorDeath doBadOpenParenType + +open-paren-lookbehind: + '=' n term ^expr-cont doOpenLookBehind # (?<= + '!' n term ^expr-cont doOpenLookBehindNeg # (? ... ), position currently on the name. +# +named-capture: + ascii_letter n named-capture doContinueNamedCapture + digit_char n named-capture doContinueNamedCapture + '>' n term ^expr-quant doOpenCaptureParen # common w non-named capture. + default errorDeath doBadNamedCapture + +# +# quant-star Scanning a '*' quantifier. Need to look ahead to decide +# between plain '*', '*?', '*+' +# +quant-star: + '?' n expr-cont doNGStar # *? + '+' n expr-cont doPossessiveStar # *+ + default expr-cont doStar + + +# +# quant-plus Scanning a '+' quantifier. Need to look ahead to decide +# between plain '+', '+?', '++' +# +quant-plus: + '?' n expr-cont doNGPlus # *? + '+' n expr-cont doPossessivePlus # *+ + default expr-cont doPlus + + +# +# quant-opt Scanning a '?' quantifier. Need to look ahead to decide +# between plain '?', '??', '?+' +# +quant-opt: + '?' n expr-cont doNGOpt # ?? + '+' n expr-cont doPossessiveOpt # ?+ + default expr-cont doOpt # ? + + +# +# Interval scanning a '{', the opening delimiter for an interval specification +# {number} or {min, max} or {min,} +# +interval-open: + digit_char interval-lower + default errorDeath doIntervalError + +interval-lower: + digit_char n interval-lower doIntevalLowerDigit + ',' n interval-upper + '}' n interval-type doIntervalSame # {n} + default errorDeath doIntervalError + +interval-upper: + digit_char n interval-upper doIntervalUpperDigit + '}' n interval-type + default errorDeath doIntervalError + +interval-type: + '?' n expr-cont doNGInterval # {n,m}? + '+' n expr-cont doPossessiveInterval # {n,m}+ + default expr-cont doInterval # {m,n} + + +# +# backslash # Backslash. Figure out which of the \thingies we have encountered. +# The low level next-char function will have preprocessed +# some of them already; those won't come here. +backslash: + 'A' n term doBackslashA + 'B' n term doBackslashB + 'b' n term doBackslashb + 'd' n expr-quant doBackslashd + 'D' n expr-quant doBackslashD + 'G' n term doBackslashG + 'h' n expr-quant doBackslashh + 'H' n expr-quant doBackslashH + 'k' n named-backref + 'N' expr-quant doNamedChar # \N{NAME} named char + 'p' expr-quant doProperty # \p{Lu} style property + 'P' expr-quant doProperty + 'R' n expr-quant doBackslashR + 'Q' n term doEnterQuoteMode + 'S' n expr-quant doBackslashS + 's' n expr-quant doBackslashs + 'v' n expr-quant doBackslashv + 'V' n expr-quant doBackslashV + 'W' n expr-quant doBackslashW + 'w' n expr-quant doBackslashw + 'X' n expr-quant doBackslashX + 'Z' n term doBackslashZ + 'z' n term doBackslashz + digit_char n expr-quant doBackRef # Will scan multiple digits + eof errorDeath doEscapeError + default n expr-quant doEscapedLiteralChar + + +# named-backref Scanned \k +# Leading to \k +# Failure to get the full sequence is an error. +# +named-backref: + '<' n named-backref-2 doBeginNamedBackRef + default errorDeath doBadNamedCapture + +named-backref-2: + ascii_letter n named-backref-3 doContinueNamedBackRef + default errorDeath doBadNamedCapture + +named-backref-3: + ascii_letter n named-backref-3 doContinueNamedBackRef + digit_char n named-backref-3 doContinueNamedBackRef + '>' n expr-quant doCompleteNamedBackRef + default errorDeath doBadNamedCapture + + +# +# [set expression] parsing, +# All states involved in parsing set expressions have names beginning with "set-" +# + +set-open: + '^' n set-open2 doSetNegate + ':' set-posix doSetPosixProp + default set-open2 + +set-open2: + ']' n set-after-lit doSetLiteral + default set-start + +# set-posix: +# scanned a '[:' If it really is a [:property:], doSetPosixProp will have +# moved the scan to the closing ']'. If it wasn't a property +# expression, the scan will still be at the opening ':', which should +# be interpreted as a normal set expression. +set-posix: + ']' n pop doSetEnd + ':' set-start + default errorDeath doRuleError # should not be possible. + +# +# set-start after the [ and special case leading characters (^ and/or ]) but before +# everything else. A '-' is literal at this point. +# +set-start: + ']' n pop doSetEnd + '[' n set-open ^set-after-set doSetBeginUnion + '\' n set-escape + '-' n set-start-dash + '&' n set-start-amp + default n set-after-lit doSetLiteral + +# set-start-dash Turn "[--" into a syntax error. +# "[-x" is good, - and x are literals. +# +set-start-dash: + '-' errorDeath doRuleError + default set-after-lit doSetAddDash + +# set-start-amp Turn "[&&" into a syntax error. +# "[&x" is good, & and x are literals. +# +set-start-amp: + '&' errorDeath doRuleError + default set-after-lit doSetAddAmp + +# +# set-after-lit The last thing scanned was a literal character within a set. +# Can be followed by anything. Single '-' or '&' are +# literals in this context, not operators. +set-after-lit: + ']' n pop doSetEnd + '[' n set-open ^set-after-set doSetBeginUnion + '-' n set-lit-dash + '&' n set-lit-amp + '\' n set-escape + eof errorDeath doSetNoCloseError + default n set-after-lit doSetLiteral + +set-after-set: + ']' n pop doSetEnd + '[' n set-open ^set-after-set doSetBeginUnion + '-' n set-set-dash + '&' n set-set-amp + '\' n set-escape + eof errorDeath doSetNoCloseError + default n set-after-lit doSetLiteral + +set-after-range: + ']' n pop doSetEnd + '[' n set-open ^set-after-set doSetBeginUnion + '-' n set-range-dash + '&' n set-range-amp + '\' n set-escape + eof errorDeath doSetNoCloseError + default n set-after-lit doSetLiteral + + +# set-after-op +# After a -- or && +# It is an error to close a set at this point. +# +set-after-op: + '[' n set-open ^set-after-set doSetBeginUnion + ']' errorDeath doSetOpError + '\' n set-escape + default n set-after-lit doSetLiteral + +# +# set-set-amp +# Have scanned [[set]& +# Could be a '&' intersection operator, if a set follows. +# Could be the start of a '&&' operator. +# Otherwise is a literal. +set-set-amp: + '[' n set-open ^set-after-set doSetBeginIntersection1 + '&' n set-after-op doSetIntersection2 + default set-after-lit doSetAddAmp + + +# set-lit-amp Have scanned "[literals&" +# Could be a start of "&&" operator or a literal +# In [abc&[def]], the '&' is a literal +# +set-lit-amp: + '&' n set-after-op doSetIntersection2 + default set-after-lit doSetAddAmp + + +# +# set-set-dash +# Have scanned [set]- +# Could be a '-' difference operator, if a [set] follows. +# Could be the start of a '--' operator. +# Otherwise is a literal. +set-set-dash: + '[' n set-open ^set-after-set doSetBeginDifference1 + '-' n set-after-op doSetDifference2 + default set-after-lit doSetAddDash + + +# +# set-range-dash +# scanned a-b- or \w- +# any set or range like item where the trailing single '-' should +# be literal, not a set difference operation. +# A trailing "--" is still a difference operator. +set-range-dash: + '-' n set-after-op doSetDifference2 + default set-after-lit doSetAddDash + + +set-range-amp: + '&' n set-after-op doSetIntersection2 + default set-after-lit doSetAddAmp + + +# set-lit-dash +# Have scanned "[literals-" Could be a range or a -- operator or a literal +# In [abc-[def]], the '-' is a literal (confirmed with a Java test) +# [abc-\p{xx} the '-' is an error +# [abc-] the '-' is a literal +# [ab-xy] the '-' is a range +# +set-lit-dash: + '-' n set-after-op doSetDifference2 + '[' set-after-lit doSetAddDash + ']' set-after-lit doSetAddDash + '\' n set-lit-dash-escape + default n set-after-range doSetRange + +# set-lit-dash-escape +# +# scanned "[literal-\" +# Could be a range, if the \ introduces an escaped literal char or a named char. +# Otherwise it is an error. +# +set-lit-dash-escape: + 's' errorDeath doSetOpError + 'S' errorDeath doSetOpError + 'w' errorDeath doSetOpError + 'W' errorDeath doSetOpError + 'd' errorDeath doSetOpError + 'D' errorDeath doSetOpError + 'N' set-after-range doSetNamedRange + default n set-after-range doSetRange + + +# +# set-escape +# Common back-slash escape processing within set expressions +# +set-escape: + 'p' set-after-set doSetProp + 'P' set-after-set doSetProp + 'N' set-after-lit doSetNamedChar + 's' n set-after-range doSetBackslash_s + 'S' n set-after-range doSetBackslash_S + 'w' n set-after-range doSetBackslash_w + 'W' n set-after-range doSetBackslash_W + 'd' n set-after-range doSetBackslash_d + 'D' n set-after-range doSetBackslash_D + 'h' n set-after-range doSetBackslash_h + 'H' n set-after-range doSetBackslash_H + 'v' n set-after-range doSetBackslash_v + 'V' n set-after-range doSetBackslash_V + default n set-after-lit doSetLiteralEscaped + +# +# set-finish +# Have just encountered the final ']' that completes a [set], and +# arrived here via a pop. From here, we exit the set parsing world, and go +# back to generic regular expression parsing. +# +set-finish: + default expr-quant doSetFinish + + +# +# errorDeath. This state is specified as the next state whenever a syntax error +# in the source rules is detected. Barring bugs, the state machine will never +# actually get here, but will stop because of the action associated with the error. +# But, just in case, this state asks the state machine to exit. +errorDeath: + default n errorDeath doExit + + diff --git a/intl/icu/source/i18n/regeximp.cpp b/intl/icu/source/i18n/regeximp.cpp new file mode 100644 index 0000000000..86e238c0f7 --- /dev/null +++ b/intl/icu/source/i18n/regeximp.cpp @@ -0,0 +1,120 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// Copyright (C) 2012 International Business Machines Corporation +// and others. All rights reserved. +// +// file: regeximp.cpp +// +// ICU Regular Expressions, +// miscellaneous implementation functions. +// + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS +#include "regeximp.h" +#include "unicode/utf16.h" + +U_NAMESPACE_BEGIN + +CaseFoldingUTextIterator::CaseFoldingUTextIterator(UText &text) : + fUText(text), fFoldChars(nullptr), fFoldLength(0) { +} + +CaseFoldingUTextIterator::~CaseFoldingUTextIterator() {} + +UChar32 CaseFoldingUTextIterator::next() { + UChar32 foldedC; + UChar32 originalC; + if (fFoldChars == nullptr) { + // We are not in a string folding of an earlier character. + // Start handling the next char from the input UText. + originalC = UTEXT_NEXT32(&fUText); + if (originalC == U_SENTINEL) { + return originalC; + } + fFoldLength = ucase_toFullFolding(originalC, &fFoldChars, U_FOLD_CASE_DEFAULT); + if (fFoldLength >= UCASE_MAX_STRING_LENGTH || fFoldLength < 0) { + // input code point folds to a single code point, possibly itself. + // See comment in ucase.h for explanation of return values from ucase_toFullFoldings. + if (fFoldLength < 0) { + fFoldLength = ~fFoldLength; + } + foldedC = (UChar32)fFoldLength; + fFoldChars = nullptr; + return foldedC; + } + // String foldings fall through here. + fFoldIndex = 0; + } + + U16_NEXT(fFoldChars, fFoldIndex, fFoldLength, foldedC); + if (fFoldIndex >= fFoldLength) { + fFoldChars = nullptr; + } + return foldedC; +} + + +UBool CaseFoldingUTextIterator::inExpansion() { + return fFoldChars != nullptr; +} + + + +CaseFoldingUCharIterator::CaseFoldingUCharIterator(const char16_t *chars, int64_t start, int64_t limit) : + fChars(chars), fIndex(start), fLimit(limit), fFoldChars(nullptr), fFoldLength(0) { +} + + +CaseFoldingUCharIterator::~CaseFoldingUCharIterator() {} + + +UChar32 CaseFoldingUCharIterator::next() { + UChar32 foldedC; + UChar32 originalC; + if (fFoldChars == nullptr) { + // We are not in a string folding of an earlier character. + // Start handling the next char from the input UText. + if (fIndex >= fLimit) { + return U_SENTINEL; + } + U16_NEXT(fChars, fIndex, fLimit, originalC); + + fFoldLength = ucase_toFullFolding(originalC, &fFoldChars, U_FOLD_CASE_DEFAULT); + if (fFoldLength >= UCASE_MAX_STRING_LENGTH || fFoldLength < 0) { + // input code point folds to a single code point, possibly itself. + // See comment in ucase.h for explanation of return values from ucase_toFullFoldings. + if (fFoldLength < 0) { + fFoldLength = ~fFoldLength; + } + foldedC = (UChar32)fFoldLength; + fFoldChars = nullptr; + return foldedC; + } + // String foldings fall through here. + fFoldIndex = 0; + } + + U16_NEXT(fFoldChars, fFoldIndex, fFoldLength, foldedC); + if (fFoldIndex >= fFoldLength) { + fFoldChars = nullptr; + } + return foldedC; +} + + +UBool CaseFoldingUCharIterator::inExpansion() { + return fFoldChars != nullptr; +} + +int64_t CaseFoldingUCharIterator::getIndex() { + return fIndex; +} + + +U_NAMESPACE_END + +#endif + diff --git a/intl/icu/source/i18n/regeximp.h b/intl/icu/source/i18n/regeximp.h new file mode 100644 index 0000000000..446cd90747 --- /dev/null +++ b/intl/icu/source/i18n/regeximp.h @@ -0,0 +1,414 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// Copyright (C) 2002-2015 International Business Machines Corporation +// and others. All rights reserved. +// +// file: regeximp.h +// +// ICU Regular Expressions, +// Definitions of constant values used in the compiled form of +// a regular expression pattern. +// + +#ifndef _REGEXIMP_H +#define _REGEXIMP_H + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/uniset.h" +#include "unicode/utext.h" + +#include "cmemory.h" +#include "ucase.h" + +U_NAMESPACE_BEGIN + +// For debugging, define REGEX_DEBUG +// To define with configure, +// CPPFLAGS="-DREGEX_DEBUG" ./runConfigureICU --enable-debug --disable-release Linux + +#ifdef REGEX_DEBUG +// +// debugging options. Enable one or more of the three #defines immediately following +// + +//#define REGEX_SCAN_DEBUG +#define REGEX_DUMP_DEBUG +#define REGEX_RUN_DEBUG + +// End of #defines intended to be directly set. + +#include +#endif + +#ifdef REGEX_SCAN_DEBUG +#define REGEX_SCAN_DEBUG_PRINTF(a) printf a +#else +#define REGEX_SCAN_DEBUG_PRINTF(a) +#endif + + +// +// Opcode types In the compiled form of the regexp, these are the type, or opcodes, +// of the entries. +// +enum { + URX_RESERVED_OP = 0, // For multi-operand ops, most non-first words. + URX_RESERVED_OP_N = 255, // For multi-operand ops, negative operand values. + URX_BACKTRACK = 1, // Force a backtrack, as if a match test had failed. + URX_END = 2, + URX_ONECHAR = 3, // Value field is the 21 bit unicode char to match + URX_STRING = 4, // Value field is index of string start + URX_STRING_LEN = 5, // Value field is string length (code units) + URX_STATE_SAVE = 6, // Value field is pattern position to push + URX_NOP = 7, + URX_START_CAPTURE = 8, // Value field is capture group number. + URX_END_CAPTURE = 9, // Value field is capture group number + URX_STATIC_SETREF = 10, // Value field is index of set in array of sets. + URX_SETREF = 11, // Value field is index of set in array of sets. + URX_DOTANY = 12, + URX_JMP = 13, // Value field is destination position in + // the pattern. + URX_FAIL = 14, // Stop match operation, No match. + + URX_JMP_SAV = 15, // Operand: JMP destination location + URX_BACKSLASH_B = 16, // Value field: 0: \b 1: \B + URX_BACKSLASH_G = 17, + URX_JMP_SAV_X = 18, // Conditional JMP_SAV, + // Used in (x)+, breaks loop on zero length match. + // Operand: Jmp destination. + URX_BACKSLASH_X = 19, + URX_BACKSLASH_Z = 20, // \z Unconditional end of line. + + URX_DOTANY_ALL = 21, // ., in the . matches any mode. + URX_BACKSLASH_D = 22, // Value field: 0: \d 1: \D + URX_CARET = 23, // Value field: 1: multi-line mode. + URX_DOLLAR = 24, // Also for \Z + + URX_CTR_INIT = 25, // Counter Inits for {Interval} loops. + URX_CTR_INIT_NG = 26, // 2 kinds, normal and non-greedy. + // These are 4 word opcodes. See description. + // First Operand: Data loc of counter variable + // 2nd Operand: Pat loc of the URX_CTR_LOOPx + // at the end of the loop. + // 3rd Operand: Minimum count. + // 4th Operand: Max count, -1 for unbounded. + + URX_DOTANY_UNIX = 27, // '.' operator in UNIX_LINES mode, only \n marks end of line. + + URX_CTR_LOOP = 28, // Loop Ops for {interval} loops. + URX_CTR_LOOP_NG = 29, // Also in three flavors. + // Operand is loc of corresponding CTR_INIT. + + URX_CARET_M_UNIX = 30, // '^' operator, test for start of line in multi-line + // plus UNIX_LINES mode. + + URX_RELOC_OPRND = 31, // Operand value in multi-operand ops that refers + // back into compiled pattern code, and thus must + // be relocated when inserting/deleting ops in code. + + URX_STO_SP = 32, // Store the stack ptr. Operand is location within + // matcher data (not stack data) to store it. + URX_LD_SP = 33, // Load the stack pointer. Operand is location + // to load from. + URX_BACKREF = 34, // Back Reference. Parameter is the index of the + // capture group variables in the state stack frame. + URX_STO_INP_LOC = 35, // Store the input location. Operand is location + // within the matcher stack frame. + URX_JMPX = 36, // Conditional JMP. + // First Operand: JMP target location. + // Second Operand: Data location containing an + // input position. If current input position == + // saved input position, FAIL rather than taking + // the JMP + URX_LA_START = 37, // Starting a LookAround expression. + // Save InputPos, SP and active region in static data. + // Operand: Static data offset for the save + URX_LA_END = 38, // Ending a Lookaround expression. + // Restore InputPos and Stack to saved values. + // Operand: Static data offset for saved data. + URX_ONECHAR_I = 39, // Test for case-insensitive match of a literal character. + // Operand: the literal char. + URX_STRING_I = 40, // Case insensitive string compare. + // First Operand: Index of start of string in string literals + // Second Operand (next word in compiled code): + // the length of the string. + URX_BACKREF_I = 41, // Case insensitive back reference. + // Parameter is the index of the + // capture group variables in the state stack frame. + URX_DOLLAR_M = 42, // $ in multi-line mode. + URX_CARET_M = 43, // ^ in multi-line mode. + URX_LB_START = 44, // LookBehind Start. + // Parameter is data location + URX_LB_CONT = 45, // LookBehind Continue. + // Param 0: the data location + // Param 1: The minimum length of the look-behind match + // Param 2: The max length of the look-behind match + URX_LB_END = 46, // LookBehind End. + // Parameter is the data location. + // Check that match ended at the right spot, + // Restore original input string len. + URX_LBN_CONT = 47, // Negative LookBehind Continue + // Param 0: the data location + // Param 1: The minimum length of the look-behind match + // Param 2: The max length of the look-behind match + // Param 3: The pattern loc following the look-behind block. + URX_LBN_END = 48, // Negative LookBehind end + // Parameter is the data location. + // Check that the match ended at the right spot. + URX_STAT_SETREF_N = 49, // Reference to a prebuilt set (e.g. \w), negated + // Operand is index of set in array of sets. + URX_LOOP_SR_I = 50, // Init a [set]* loop. + // Operand is the sets index in array of user sets. + URX_LOOP_C = 51, // Continue a [set]* or OneChar* loop. + // Operand is a matcher static data location. + // Must always immediately follow LOOP_x_I instruction. + URX_LOOP_DOT_I = 52, // .*, initialization of the optimized loop. + // Operand value: + // bit 0: + // 0: Normal (. doesn't match new-line) mode. + // 1: . matches new-line mode. + // bit 1: controls what new-lines are recognized by this operation. + // 0: All Unicode New-lines + // 1: UNIX_LINES, \u000a only. + URX_BACKSLASH_BU = 53, // \b or \B in UREGEX_UWORD mode, using Unicode style + // word boundaries. + URX_DOLLAR_D = 54, // $ end of input test, in UNIX_LINES mode. + URX_DOLLAR_MD = 55, // $ end of input test, in MULTI_LINE and UNIX_LINES mode. + URX_BACKSLASH_H = 56, // Value field: 0: \h 1: \H + URX_BACKSLASH_R = 57, // Any line break sequence. + URX_BACKSLASH_V = 58 // Value field: 0: \v 1: \V + +}; + +// Keep this list of opcode names in sync with the above enum +// Used for debug printing only. +#define URX_OPCODE_NAMES \ + " ", \ + "BACKTRACK", \ + "END", \ + "ONECHAR", \ + "STRING", \ + "STRING_LEN", \ + "STATE_SAVE", \ + "NOP", \ + "START_CAPTURE", \ + "END_CAPTURE", \ + "URX_STATIC_SETREF", \ + "SETREF", \ + "DOTANY", \ + "JMP", \ + "FAIL", \ + "JMP_SAV", \ + "BACKSLASH_B", \ + "BACKSLASH_G", \ + "JMP_SAV_X", \ + "BACKSLASH_X", \ + "BACKSLASH_Z", \ + "DOTANY_ALL", \ + "BACKSLASH_D", \ + "CARET", \ + "DOLLAR", \ + "CTR_INIT", \ + "CTR_INIT_NG", \ + "DOTANY_UNIX", \ + "CTR_LOOP", \ + "CTR_LOOP_NG", \ + "URX_CARET_M_UNIX", \ + "RELOC_OPRND", \ + "STO_SP", \ + "LD_SP", \ + "BACKREF", \ + "STO_INP_LOC", \ + "JMPX", \ + "LA_START", \ + "LA_END", \ + "ONECHAR_I", \ + "STRING_I", \ + "BACKREF_I", \ + "DOLLAR_M", \ + "CARET_M", \ + "LB_START", \ + "LB_CONT", \ + "LB_END", \ + "LBN_CONT", \ + "LBN_END", \ + "STAT_SETREF_N", \ + "LOOP_SR_I", \ + "LOOP_C", \ + "LOOP_DOT_I", \ + "BACKSLASH_BU", \ + "DOLLAR_D", \ + "DOLLAR_MD", \ + "URX_BACKSLASH_H", \ + "URX_BACKSLASH_R", \ + "URX_BACKSLASH_V" + + +// +// Convenience macros for assembling and disassembling a compiled operation. +// +#define URX_TYPE(x) ((uint32_t)(x) >> 24) +#define URX_VAL(x) ((x) & 0xffffff) + + +// +// Access to Unicode Sets composite character properties +// The sets are accessed by the match engine for things like \w (word boundary) +// +enum { + URX_ISWORD_SET = 1, + URX_ISALNUM_SET = 2, + URX_ISALPHA_SET = 3, + URX_ISSPACE_SET = 4, + + URX_GC_NORMAL, // Sets for finding grapheme cluster boundaries. + URX_GC_EXTEND, + URX_GC_CONTROL, + URX_GC_L, + URX_GC_LV, + URX_GC_LVT, + URX_GC_V, + URX_GC_T, + + URX_LAST_SET, + + URX_NEG_SET = 0x800000 // Flag bit to reverse sense of set + // membership test. +}; + + +// +// Match Engine State Stack Frame Layout. +// +struct REStackFrame { + // Header + int64_t fInputIdx; // Position of next character in the input string + int64_t fPatIdx; // Position of next Op in the compiled pattern + // (int64_t for UVector64, values fit in an int32_t) + // Remainder + int64_t fExtra[1]; // Extra state, for capture group start/ends + // atomic parentheses, repeat counts, etc. + // Locations assigned at pattern compile time. + // Variable-length array. +}; +// number of UVector elements in the header +#define RESTACKFRAME_HDRCOUNT 2 + +// +// Start-Of-Match type. Used by find() to quickly scan to positions where a +// match might start before firing up the full match engine. +// +enum StartOfMatch { + START_NO_INFO, // No hint available. + START_CHAR, // Match starts with a literal code point. + START_SET, // Match starts with something matching a set. + START_START, // Match starts at start of buffer only (^ or \A) + START_LINE, // Match starts with ^ in multi-line mode. + START_STRING // Match starts with a literal string. +}; + +#define START_OF_MATCH_STR(v) ((v)==START_NO_INFO? "START_NO_INFO" : \ + (v)==START_CHAR? "START_CHAR" : \ + (v)==START_SET? "START_SET" : \ + (v)==START_START? "START_START" : \ + (v)==START_LINE? "START_LINE" : \ + (v)==START_STRING? "START_STRING" : \ + "ILLEGAL") + +// +// 8 bit set, to fast-path latin-1 set membership tests. +// +struct Regex8BitSet : public UMemory { + inline Regex8BitSet(); + inline void operator = (const Regex8BitSet &s); + inline void init(const UnicodeSet *src); + inline UBool contains(UChar32 c); + inline void add(UChar32 c); + int8_t d[32]; +}; + +inline Regex8BitSet::Regex8BitSet() { + uprv_memset(d, 0, sizeof(d)); +} + +inline UBool Regex8BitSet::contains(UChar32 c) { + // No bounds checking! This is deliberate. + return ((d[c>>3] & 1 <<(c&7)) != 0); +} + +inline void Regex8BitSet::add(UChar32 c) { + d[c>>3] |= 1 << (c&7); +} + +inline void Regex8BitSet::init(const UnicodeSet *s) { + if (s != nullptr) { + for (int32_t i=0; i<=255; i++) { + if (s->contains(i)) { + this->add(i); + } + } + } +} + +inline void Regex8BitSet::operator = (const Regex8BitSet &s) { + uprv_memcpy(d, s.d, sizeof(d)); +} + + +// Case folded UText Iterator helper class. +// Wraps a UText, provides a case-folded enumeration over its contents. +// Used in implementing case insensitive matching constructs. +// Implementation in rematch.cpp + +class CaseFoldingUTextIterator: public UMemory { + public: + CaseFoldingUTextIterator(UText &text); + ~CaseFoldingUTextIterator(); + + UChar32 next(); // Next case folded character + + UBool inExpansion(); // True if last char returned from next() and the + // next to be returned both originated from a string + // folding of the same code point from the original UText. + private: + UText &fUText; + const char16_t *fFoldChars; + int32_t fFoldLength; + int32_t fFoldIndex; + +}; + + +// Case folded char16_t * string iterator. +// Wraps a char16_t *, provides a case-folded enumeration over its contents. +// Used in implementing case insensitive matching constructs. +// Implementation in rematch.cpp + +class CaseFoldingUCharIterator: public UMemory { + public: + CaseFoldingUCharIterator(const char16_t *chars, int64_t start, int64_t limit); + ~CaseFoldingUCharIterator(); + + UChar32 next(); // Next case folded character + + UBool inExpansion(); // True if last char returned from next() and the + // next to be returned both originated from a string + // folding of the same code point from the original UText. + + int64_t getIndex(); // Return the current input buffer index. + + private: + const char16_t *fChars; + int64_t fIndex; + int64_t fLimit; + const char16_t *fFoldChars; + int32_t fFoldLength; + int32_t fFoldIndex; + +}; + +U_NAMESPACE_END +#endif + diff --git a/intl/icu/source/i18n/regexst.cpp b/intl/icu/source/i18n/regexst.cpp new file mode 100644 index 0000000000..9103230544 --- /dev/null +++ b/intl/icu/source/i18n/regexst.cpp @@ -0,0 +1,172 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// regexst.h +// +// Copyright (C) 2004-2015, International Business Machines Corporation and others. +// All Rights Reserved. +// +// This file contains class RegexStaticSets +// +// This class is internal to the regular expression implementation. +// For the public Regular Expression API, see the file "unicode/regex.h" +// +// RegexStaticSets groups together the common UnicodeSets that are needed +// for compiling or executing RegularExpressions. This grouping simplifies +// the thread safe lazy creation and sharing of these sets across +// all instances of regular expressions. +// +#include "unicode/utypes.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + +#include "unicode/unistr.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/regex.h" +#include "uprops.h" +#include "cmemory.h" +#include "cstring.h" +#include "uassert.h" +#include "ucln_in.h" +#include "umutex.h" + +#include "regexcst.h" // Contains state table for the regex pattern parser. + // generated by a Perl script. +#include "regexst.h" + +U_NAMESPACE_BEGIN + +// "Rule Char" Characters are those with special meaning, and therefore +// need to be escaped to appear as literals in a regexp. +constexpr char16_t const *gRuleSet_rule_chars = u"*?+[(){}^$|\\."; + +// +// The backslash escape characters that ICU's unescape() function will handle. +// +constexpr char16_t const *gUnescapeChars = u"acefnrtuUx"; + +// +// Unicode Set pattern for Regular Expression \w +// +constexpr char16_t const *gIsWordPattern = u"[\\p{Alphabetic}\\p{M}\\p{Nd}\\p{Pc}\\u200c\\u200d]"; + +// +// Unicode Set Definitions for Regular Expression \s +// +constexpr char16_t const *gIsSpacePattern = u"[\\p{WhiteSpace}]"; + +// +// UnicodeSets used in implementation of Grapheme Cluster detection, \X +// +constexpr char16_t const *gGC_ControlPattern = u"[[:Zl:][:Zp:][:Cc:][:Cf:]-[:Grapheme_Extend:]]"; +constexpr char16_t const *gGC_ExtendPattern = u"[\\p{Grapheme_Extend}]"; +constexpr char16_t const *gGC_LPattern = u"[\\p{Hangul_Syllable_Type=L}]"; +constexpr char16_t const *gGC_VPattern = u"[\\p{Hangul_Syllable_Type=V}]"; +constexpr char16_t const *gGC_TPattern = u"[\\p{Hangul_Syllable_Type=T}]"; +constexpr char16_t const *gGC_LVPattern = u"[\\p{Hangul_Syllable_Type=LV}]"; +constexpr char16_t const *gGC_LVTPattern = u"[\\p{Hangul_Syllable_Type=LVT}]"; + + +RegexStaticSets *RegexStaticSets::gStaticSets = nullptr; +UInitOnce gStaticSetsInitOnce {}; + + +RegexStaticSets::RegexStaticSets(UErrorCode *status) { + // Initialize the shared static sets to their correct values. + fUnescapeCharSet.addAll(UnicodeString(true, gUnescapeChars, -1)).freeze(); + fPropSets[URX_ISWORD_SET].applyPattern(UnicodeString(true, gIsWordPattern, -1), *status).freeze(); + fPropSets[URX_ISSPACE_SET].applyPattern(UnicodeString(true, gIsSpacePattern, -1), *status).freeze(); + fPropSets[URX_GC_EXTEND].applyPattern(UnicodeString(true, gGC_ExtendPattern, -1), *status).freeze(); + fPropSets[URX_GC_CONTROL].applyPattern(UnicodeString(true, gGC_ControlPattern, -1), *status).freeze(); + fPropSets[URX_GC_L].applyPattern(UnicodeString(true, gGC_LPattern, -1), *status).freeze(); + fPropSets[URX_GC_V].applyPattern(UnicodeString(true, gGC_VPattern, -1), *status).freeze(); + fPropSets[URX_GC_T].applyPattern(UnicodeString(true, gGC_TPattern, -1), *status).freeze(); + fPropSets[URX_GC_LV].applyPattern(UnicodeString(true, gGC_LVPattern, -1), *status).freeze(); + fPropSets[URX_GC_LVT].applyPattern(UnicodeString(true, gGC_LVTPattern, -1), *status).freeze(); + + + // + // "Normal" is the set of characters that don't need special handling + // when finding grapheme cluster boundaries. + // + fPropSets[URX_GC_NORMAL].complement(); + fPropSets[URX_GC_NORMAL].remove(0xac00, 0xd7a4); + fPropSets[URX_GC_NORMAL].removeAll(fPropSets[URX_GC_CONTROL]); + fPropSets[URX_GC_NORMAL].removeAll(fPropSets[URX_GC_L]); + fPropSets[URX_GC_NORMAL].removeAll(fPropSets[URX_GC_V]); + fPropSets[URX_GC_NORMAL].removeAll(fPropSets[URX_GC_T]); + fPropSets[URX_GC_NORMAL].freeze(); + + // Initialize the 8-bit fast bit sets from the parallel full + // UnicodeSets. + // + // TODO: 25 Oct 2019 are these fast 8-bit sets worth keeping? + // Measured 3.5% gain on (non) matching with the pattern "x(?:\\S+)+x" + // This runs in exponential time, making it easy to adjust the time for + // convenient measuring. + // + // This 8 bit optimization dates from the early days of ICU, + // with a less optimized UnicodeSet. At the time, the difference + // was substantial. + + for (int32_t i=0; ilastOffset + 1) { + c = UTEXT_NEXT32(context->text); + context->lastOffset++; + } else if (offset == context->lastOffset) { + c = UTEXT_PREVIOUS32(context->text); + UTEXT_NEXT32(context->text); + } else { + utext_moveIndex32(context->text, offset - context->lastOffset - 1); + c = UTEXT_NEXT32(context->text); + context->lastOffset = offset; + } + + // !!!: Doesn't handle characters outside BMP + if (U_IS_BMP(c)) { + return (char16_t)c; + } else { + return 0; + } +} + +U_CFUNC char16_t U_CALLCONV +uregex_ucstr_unescape_charAt(int32_t offset, void *context) { + return ((char16_t *)context)[offset]; +} + +U_NAMESPACE_END diff --git a/intl/icu/source/i18n/regextxt.h b/intl/icu/source/i18n/regextxt.h new file mode 100644 index 0000000000..0f64b8437e --- /dev/null +++ b/intl/icu/source/i18n/regextxt.h @@ -0,0 +1,50 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/******************************************************************** + * COPYRIGHT: + * Copyright (c) 2008-2010, International Business Machines Corporation and + * others. All Rights Reserved. + ********************************************************************/ +// +// file: regextxt.h +// +// This file contains utility code for supporting UText in the regular expression engine. +// +// This class is internal to the regular expression implementation. +// For the public Regular Expression API, see the file "unicode/regex.h" +// + +#ifndef _REGEXTXT_H +#define _REGEXTXT_H + +#include "unicode/utypes.h" +#include "unicode/utext.h" + +U_NAMESPACE_BEGIN + +#define UTEXT_USES_U16(ut) (NULL==((ut)->pFuncs->mapNativeIndexToUTF16)) + +#if 0 +#define REGEX_DISABLE_CHUNK_MODE 1 +#endif + +#ifdef REGEX_DISABLE_CHUNK_MODE +# define UTEXT_FULL_TEXT_IN_CHUNK(ut,len) (false) +#else +# define UTEXT_FULL_TEXT_IN_CHUNK(ut,len) ((0==((ut)->chunkNativeStart))&&((len)==((ut)->chunkNativeLimit))&&((len)==((ut)->nativeIndexingLimit))) +#endif + +struct URegexUTextUnescapeCharContext { + UText *text; + int32_t lastOffset; +}; +#define U_REGEX_UTEXT_UNESCAPE_CONTEXT(text) { (text), -1 } + +U_CFUNC UChar U_CALLCONV +uregex_utext_unescape_charAt(int32_t offset, void * /* struct URegexUTextUnescapeCharContext* */ context); +U_CFUNC UChar U_CALLCONV +uregex_ucstr_unescape_charAt(int32_t offset, void * /* UChar* */ context); + +U_NAMESPACE_END + +#endif diff --git a/intl/icu/source/i18n/region.cpp b/intl/icu/source/i18n/region.cpp new file mode 100644 index 0000000000..26440b6593 --- /dev/null +++ b/intl/icu/source/i18n/region.cpp @@ -0,0 +1,772 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2014-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +* +* +* File REGION.CPP +* +* Modification History:* +* Date Name Description +* 01/15/13 Emmons Original Port from ICU4J +******************************************************************************** +*/ + +/** + * \file + * \brief C++ API: Region classes (territory containment) + */ + +#include "unicode/region.h" +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/ures.h" +#include "ucln_in.h" +#include "cstring.h" +#include "mutex.h" +#include "uhash.h" +#include "umutex.h" +#include "uresimp.h" +#include "region_impl.h" +#include "util.h" + +#if !UCONFIG_NO_FORMATTING + + +U_CDECL_BEGIN + +/** + * Cleanup callback func + */ +static UBool U_CALLCONV region_cleanup() +{ + icu::Region::cleanupRegionData(); + + return true; +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +static UInitOnce gRegionDataInitOnce {}; +static UVector* availableRegions[URGN_LIMIT]; + +static UHashtable *regionAliases = nullptr; +static UHashtable *regionIDMap = nullptr; +static UHashtable *numericCodeMap = nullptr; +static UVector *allRegions = nullptr; + +static const char16_t UNKNOWN_REGION_ID [] = { 0x5A, 0x5A, 0 }; /* "ZZ" */ +static const char16_t OUTLYING_OCEANIA_REGION_ID [] = { 0x51, 0x4F, 0 }; /* "QO" */ +static const char16_t WORLD_ID [] = { 0x30, 0x30, 0x31, 0 }; /* "001" */ +static const char16_t RANGE_MARKER = 0x7E; /* '~' */ + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RegionNameEnumeration) + +/* + * Initializes the region data from the ICU resource bundles. The region data + * contains the basic relationships such as which regions are known, what the numeric + * codes are, any known aliases, and the territory containment data. + * + * If the region data has already loaded, then this method simply returns without doing + * anything meaningful. + */ +void U_CALLCONV Region::loadRegionData(UErrorCode &status) { + + // Construct service objs first + LocalUHashtablePointer newRegionIDMap(uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, nullptr, &status)); + LocalUHashtablePointer newNumericCodeMap(uhash_open(uhash_hashLong,uhash_compareLong,nullptr,&status)); + LocalUHashtablePointer newRegionAliases(uhash_open(uhash_hashUnicodeString,uhash_compareUnicodeString,nullptr,&status)); + + LocalPointer continents(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + LocalPointer groupings(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + LocalPointer lpAllRegions(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + allRegions = lpAllRegions.orphan(); + + LocalUResourceBundlePointer metadata(ures_openDirect(nullptr,"metadata",&status)); + LocalUResourceBundlePointer metadataAlias(ures_getByKey(metadata.getAlias(),"alias",nullptr,&status)); + LocalUResourceBundlePointer territoryAlias(ures_getByKey(metadataAlias.getAlias(),"territory",nullptr,&status)); + + LocalUResourceBundlePointer supplementalData(ures_openDirect(nullptr,"supplementalData",&status)); + LocalUResourceBundlePointer codeMappings(ures_getByKey(supplementalData.getAlias(),"codeMappings",nullptr,&status)); + + LocalUResourceBundlePointer idValidity(ures_getByKey(supplementalData.getAlias(),"idValidity",nullptr,&status)); + LocalUResourceBundlePointer regionList(ures_getByKey(idValidity.getAlias(),"region",nullptr,&status)); + LocalUResourceBundlePointer regionRegular(ures_getByKey(regionList.getAlias(),"regular",nullptr,&status)); + LocalUResourceBundlePointer regionMacro(ures_getByKey(regionList.getAlias(),"macroregion",nullptr,&status)); + LocalUResourceBundlePointer regionUnknown(ures_getByKey(regionList.getAlias(),"unknown",nullptr,&status)); + + LocalUResourceBundlePointer territoryContainment(ures_getByKey(supplementalData.getAlias(),"territoryContainment",nullptr,&status)); + LocalUResourceBundlePointer worldContainment(ures_getByKey(territoryContainment.getAlias(),"001",nullptr,&status)); + LocalUResourceBundlePointer groupingContainment(ures_getByKey(territoryContainment.getAlias(),"grouping",nullptr,&status)); + + ucln_i18n_registerCleanup(UCLN_I18N_REGION, region_cleanup); + if (U_FAILURE(status)) { + return; + } + + // now, initialize + uhash_setValueDeleter(newRegionIDMap.getAlias(), uprv_deleteUObject); // regionIDMap owns objs + uhash_setKeyDeleter(newRegionAliases.getAlias(), uprv_deleteUObject); // regionAliases owns the string keys + + + while (U_SUCCESS(status) && ures_hasNext(regionRegular.getAlias())) { + UnicodeString regionName = ures_getNextUnicodeString(regionRegular.getAlias(),nullptr,&status); + int32_t rangeMarkerLocation = regionName.indexOf(RANGE_MARKER); + char16_t buf[6]; + regionName.extract(buf,6,status); + if ( rangeMarkerLocation > 0 ) { + char16_t endRange = regionName.charAt(rangeMarkerLocation+1); + buf[rangeMarkerLocation] = 0; + while (U_SUCCESS(status) && buf[rangeMarkerLocation-1] <= endRange) { + LocalPointer newRegion(new UnicodeString(buf), status); + allRegions->adoptElement(newRegion.orphan(), status); + buf[rangeMarkerLocation-1]++; + } + } else { + LocalPointer newRegion(new UnicodeString(regionName), status); + allRegions->adoptElement(newRegion.orphan(), status); + } + } + + while (U_SUCCESS(status) && ures_hasNext(regionMacro.getAlias())) { + UnicodeString regionName = ures_getNextUnicodeString(regionMacro.getAlias(),nullptr,&status); + int32_t rangeMarkerLocation = regionName.indexOf(RANGE_MARKER); + char16_t buf[6]; + regionName.extract(buf,6,status); + if ( rangeMarkerLocation > 0 ) { + char16_t endRange = regionName.charAt(rangeMarkerLocation+1); + buf[rangeMarkerLocation] = 0; + while ( buf[rangeMarkerLocation-1] <= endRange && U_SUCCESS(status)) { + LocalPointer newRegion(new UnicodeString(buf), status); + allRegions->adoptElement(newRegion.orphan(),status); + buf[rangeMarkerLocation-1]++; + } + } else { + LocalPointer newRegion(new UnicodeString(regionName), status); + allRegions->adoptElement(newRegion.orphan(),status); + } + } + + while (U_SUCCESS(status) && ures_hasNext(regionUnknown.getAlias())) { + LocalPointer regionName ( + new UnicodeString(ures_getNextUnicodeString(regionUnknown.getAlias(), nullptr, &status), status)); + allRegions->adoptElement(regionName.orphan(),status); + } + + while (U_SUCCESS(status) && ures_hasNext(worldContainment.getAlias())) { + UnicodeString *continentName = new UnicodeString(ures_getNextUnicodeString(worldContainment.getAlias(),nullptr,&status)); + continents->adoptElement(continentName,status); + } + if (U_FAILURE(status)) { + return; + } + + for ( int32_t i = 0 ; i < allRegions->size() ; i++ ) { + LocalPointer r(new Region(), status); + if ( U_FAILURE(status) ) { + return; + } + UnicodeString *regionName = (UnicodeString *)allRegions->elementAt(i); + r->idStr = *regionName; + + r->idStr.extract(0,r->idStr.length(),r->id,sizeof(r->id),US_INV); + r->fType = URGN_TERRITORY; // Only temporary - figure out the real type later once the aliases are known. + + int32_t pos = 0; + int32_t result = ICU_Utility::parseAsciiInteger(r->idStr, pos); + if (pos > 0) { + r->code = result; // Convert string to number + uhash_iput(newNumericCodeMap.getAlias(),r->code,(void *)(r.getAlias()),&status); + r->fType = URGN_SUBCONTINENT; + } else { + r->code = -1; + } + void* idStrAlias = (void*)&(r->idStr); // about to orphan 'r'. Save this off. + uhash_put(newRegionIDMap.getAlias(),idStrAlias,(void *)(r.orphan()),&status); // regionIDMap takes ownership + } + + UResourceBundle *groupingBundle = nullptr; + while (U_SUCCESS(status) && ures_hasNext(groupingContainment.getAlias())) { + groupingBundle = ures_getNextResource(groupingContainment.getAlias(), groupingBundle, &status); + if (U_FAILURE(status)) { + break; + } + UnicodeString *groupingName = new UnicodeString(ures_getKey(groupingBundle), -1, US_INV); + LocalPointer lpGroupingName(groupingName, status); + groupings->adoptElement(lpGroupingName.orphan(), status); + if (U_FAILURE(status)) { + break; + } + Region *grouping = (Region *) uhash_get(newRegionIDMap.getAlias(), groupingName); + if (grouping != nullptr) { + for (int32_t i = 0; i < ures_getSize(groupingBundle) && U_SUCCESS(status); i++) { + UnicodeString child = ures_getUnicodeStringByIndex(groupingBundle, i, &status); + if (U_SUCCESS(status)) { + if (grouping->containedRegions == nullptr) { + LocalPointer lpContainedRegions( + new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + grouping->containedRegions = lpContainedRegions.orphan(); + if (U_FAILURE(status)) { + break; + } + } + LocalPointer lpChildCopy(new UnicodeString(child), status); + grouping->containedRegions->adoptElement(lpChildCopy.orphan(), status); + } + } + } + } + ures_close(groupingBundle); + + // Process the territory aliases + while (U_SUCCESS(status) && ures_hasNext(territoryAlias.getAlias())) { + LocalUResourceBundlePointer res(ures_getNextResource(territoryAlias.getAlias(),nullptr,&status)); + const char *aliasFrom = ures_getKey(res.getAlias()); + LocalPointer aliasFromStr(new UnicodeString(aliasFrom, -1, US_INV), status); + UnicodeString aliasTo = ures_getUnicodeStringByKey(res.getAlias(),"replacement",&status); + res.adoptInstead(nullptr); + + const Region *aliasToRegion = (Region *) uhash_get(newRegionIDMap.getAlias(),&aliasTo); + Region *aliasFromRegion = (Region *)uhash_get(newRegionIDMap.getAlias(),aliasFromStr.getAlias()); + + if ( aliasToRegion != nullptr && aliasFromRegion == nullptr ) { // This is just an alias from some string to a region + uhash_put(newRegionAliases.getAlias(),(void *)aliasFromStr.orphan(), (void *)aliasToRegion,&status); + } else { + if ( aliasFromRegion == nullptr ) { // Deprecated region code not in the primary codes list - so need to create a deprecated region for it. + LocalPointer newRgn(new Region, status); + if ( U_SUCCESS(status) ) { + aliasFromRegion = newRgn.orphan(); + } else { + return; // error out + } + aliasFromRegion->idStr.setTo(*aliasFromStr); + aliasFromRegion->idStr.extract(0,aliasFromRegion->idStr.length(),aliasFromRegion->id,sizeof(aliasFromRegion->id),US_INV); + uhash_put(newRegionIDMap.getAlias(),(void *)&(aliasFromRegion->idStr),(void *)aliasFromRegion,&status); + int32_t pos = 0; + int32_t result = ICU_Utility::parseAsciiInteger(aliasFromRegion->idStr, pos); + if ( pos > 0 ) { + aliasFromRegion->code = result; // Convert string to number + uhash_iput(newNumericCodeMap.getAlias(),aliasFromRegion->code,(void *)aliasFromRegion,&status); + } else { + aliasFromRegion->code = -1; + } + aliasFromRegion->fType = URGN_DEPRECATED; + } else { + aliasFromRegion->fType = URGN_DEPRECATED; + } + + { + LocalPointer newPreferredValues(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + aliasFromRegion->preferredValues = newPreferredValues.orphan(); + } + if( U_FAILURE(status)) { + return; + } + UnicodeString currentRegion; + //currentRegion.remove(); TODO: was already 0 length? + for (int32_t i = 0 ; i < aliasTo.length() && U_SUCCESS(status); i++ ) { + if ( aliasTo.charAt(i) != 0x0020 ) { + currentRegion.append(aliasTo.charAt(i)); + } + if ( aliasTo.charAt(i) == 0x0020 || i+1 == aliasTo.length() ) { + Region *target = (Region *)uhash_get(newRegionIDMap.getAlias(),(void *)¤tRegion); + if (target) { + LocalPointer preferredValue(new UnicodeString(target->idStr), status); + aliasFromRegion->preferredValues->adoptElement(preferredValue.orphan(),status); // may add null if err + } + currentRegion.remove(); + } + } + } + } + + // Process the code mappings - This will allow us to assign numeric codes to most of the territories. + while (U_SUCCESS(status) && ures_hasNext(codeMappings.getAlias())) { + UResourceBundle *mapping = ures_getNextResource(codeMappings.getAlias(),nullptr,&status); + if (U_SUCCESS(status) && ures_getType(mapping) == URES_ARRAY && ures_getSize(mapping) == 3) { + UnicodeString codeMappingID = ures_getUnicodeStringByIndex(mapping,0,&status); + UnicodeString codeMappingNumber = ures_getUnicodeStringByIndex(mapping,1,&status); + UnicodeString codeMapping3Letter = ures_getUnicodeStringByIndex(mapping,2,&status); + + Region *r = (Region *)uhash_get(newRegionIDMap.getAlias(),(void *)&codeMappingID); + if ( r ) { + int32_t pos = 0; + int32_t result = ICU_Utility::parseAsciiInteger(codeMappingNumber, pos); + if ( pos > 0 ) { + r->code = result; // Convert string to number + uhash_iput(newNumericCodeMap.getAlias(),r->code,(void *)r,&status); + } + LocalPointer code3(new UnicodeString(codeMapping3Letter), status); + uhash_put(newRegionAliases.getAlias(),(void *)code3.orphan(), (void *)r,&status); + } + } + ures_close(mapping); + } + + // Now fill in the special cases for WORLD, UNKNOWN, CONTINENTS, and GROUPINGS + Region *r; + UnicodeString WORLD_ID_STRING(WORLD_ID); + r = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)&WORLD_ID_STRING); + if ( r ) { + r->fType = URGN_WORLD; + } + + UnicodeString UNKNOWN_REGION_ID_STRING(UNKNOWN_REGION_ID); + r = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)&UNKNOWN_REGION_ID_STRING); + if ( r ) { + r->fType = URGN_UNKNOWN; + } + + for ( int32_t i = 0 ; i < continents->size() ; i++ ) { + r = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)continents->elementAt(i)); + if ( r ) { + r->fType = URGN_CONTINENT; + } + } + + for ( int32_t i = 0 ; i < groupings->size() ; i++ ) { + r = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)groupings->elementAt(i)); + if ( r ) { + r->fType = URGN_GROUPING; + } + } + + // Special case: The region code "QO" (Outlying Oceania) is a subcontinent code added by CLDR + // even though it looks like a territory code. Need to handle it here. + + UnicodeString OUTLYING_OCEANIA_REGION_ID_STRING(OUTLYING_OCEANIA_REGION_ID); + r = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)&OUTLYING_OCEANIA_REGION_ID_STRING); + if ( r ) { + r->fType = URGN_SUBCONTINENT; + } + + // Load territory containment info from the supplemental data. + while ( ures_hasNext(territoryContainment.getAlias()) ) { + LocalUResourceBundlePointer mapping(ures_getNextResource(territoryContainment.getAlias(),nullptr,&status)); + if( U_FAILURE(status) ) { + return; // error out + } + const char *parent = ures_getKey(mapping.getAlias()); + if (uprv_strcmp(parent, "containedGroupings") == 0 || uprv_strcmp(parent, "deprecated") == 0) { + continue; // handle new pseudo-parent types added in ICU data per cldrbug 7808; for now just skip. + // #11232 is to do something useful with these. + } + UnicodeString parentStr = UnicodeString(parent, -1 , US_INV); + Region *parentRegion = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)&parentStr); + + for ( int j = 0 ; j < ures_getSize(mapping.getAlias()); j++ ) { + UnicodeString child = ures_getUnicodeStringByIndex(mapping.getAlias(),j,&status); + Region *childRegion = (Region *) uhash_get(newRegionIDMap.getAlias(),(void *)&child); + if ( parentRegion != nullptr && childRegion != nullptr ) { + + // Add the child region to the set of regions contained by the parent + if (parentRegion->containedRegions == nullptr) { + LocalPointer lpContainedRegions( + new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + parentRegion->containedRegions = lpContainedRegions.orphan(); + if (U_FAILURE(status)) { + return; + } + } + + LocalPointer childStr(new UnicodeString(), status); + if (U_FAILURE(status)) { + return; // error out + } + childStr->fastCopyFrom(childRegion->idStr); + parentRegion->containedRegions->adoptElement(childStr.orphan(),status); + if (U_FAILURE(status)) { + return; + } + + // Set the parent region to be the containing region of the child. + // Regions of type GROUPING can't be set as the parent, since another region + // such as a SUBCONTINENT, CONTINENT, or WORLD must always be the parent. + if ( parentRegion->fType != URGN_GROUPING) { + childRegion->containingRegion = parentRegion; + } + } + } + } + + // Create the availableRegions lists + int32_t pos = UHASH_FIRST; + while ( const UHashElement* element = uhash_nextElement(newRegionIDMap.getAlias(),&pos)) { + Region *ar = (Region *)element->value.pointer; + if ( availableRegions[ar->fType] == nullptr ) { + LocalPointer newAr(new UVector(uprv_deleteUObject, uhash_compareUnicodeString, status), status); + availableRegions[ar->fType] = newAr.orphan(); + } + LocalPointer arString(new UnicodeString(ar->idStr), status); + if( U_FAILURE(status) ) { + return; // error out + } + availableRegions[ar->fType]->adoptElement(arString.orphan(), status); + } + + // copy hashtables + numericCodeMap = newNumericCodeMap.orphan(); + regionIDMap = newRegionIDMap.orphan(); + regionAliases = newRegionAliases.orphan(); +} + +void Region::cleanupRegionData() { + for (int32_t i = 0 ; i < URGN_LIMIT ; i++ ) { + if ( availableRegions[i] ) { + delete availableRegions[i]; + availableRegions[i] = nullptr; + } + } + + if (regionAliases) { + uhash_close(regionAliases); + } + + if (numericCodeMap) { + uhash_close(numericCodeMap); + } + + if (regionIDMap) { + uhash_close(regionIDMap); + } + if (allRegions) { + delete allRegions; + allRegions = nullptr; + } + + regionAliases = numericCodeMap = regionIDMap = nullptr; + + gRegionDataInitOnce.reset(); +} + +Region::Region () + : code(-1), + fType(URGN_UNKNOWN), + containingRegion(nullptr), + containedRegions(nullptr), + preferredValues(nullptr) { + id[0] = 0; +} + +Region::~Region () { + if (containedRegions) { + delete containedRegions; + } + if (preferredValues) { + delete preferredValues; + } +} + +/** + * Returns true if the two regions are equal. + * Per PMC, just use pointer compare, since we have at most one instance of each Region. + */ +bool +Region::operator==(const Region &that) const { + return (idStr == that.idStr); +} + +/** + * Returns true if the two regions are NOT equal; that is, if operator ==() returns false. + * Per PMC, just use pointer compare, since we have at most one instance of each Region. + */ +bool +Region::operator!=(const Region &that) const { + return (idStr != that.idStr); +} + +/** + * Returns a pointer to a Region using the given region code. The region code can be either 2-letter ISO code, + * 3-letter ISO code, UNM.49 numeric code, or other valid Unicode Region Code as defined by the LDML specification. + * The identifier will be canonicalized internally using the supplemental metadata as defined in the CLDR. + * If the region code is nullptr or not recognized, the appropriate error code will be set ( U_ILLEGAL_ARGUMENT_ERROR ) + */ +const Region* U_EXPORT2 +Region::getInstance(const char *region_code, UErrorCode &status) { + + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); + if (U_FAILURE(status)) { + return nullptr; + } + + if ( !region_code ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + UnicodeString regionCodeString = UnicodeString(region_code, -1, US_INV); + Region *r = (Region *)uhash_get(regionIDMap,(void *)®ionCodeString); + + if ( !r ) { + r = (Region *)uhash_get(regionAliases,(void *)®ionCodeString); + } + + if ( !r ) { // Unknown region code + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + if ( r->fType == URGN_DEPRECATED && r->preferredValues->size() == 1) { + StringEnumeration *pv = r->getPreferredValues(status); + pv->reset(status); + const UnicodeString *ustr = pv->snext(status); + r = (Region *)uhash_get(regionIDMap,(void *)ustr); + delete pv; + } + + return r; + +} + +/** + * Returns a pointer to a Region using the given numeric region code. If the numeric region code is not recognized, + * the appropriate error code will be set ( U_ILLEGAL_ARGUMENT_ERROR ). + */ +const Region* U_EXPORT2 +Region::getInstance (int32_t code, UErrorCode &status) { + + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); + if (U_FAILURE(status)) { + return nullptr; + } + + Region *r = (Region *)uhash_iget(numericCodeMap,code); + + if ( !r ) { // Just in case there's an alias that's numeric, try to find it. + UnicodeString id; + ICU_Utility::appendNumber(id, code, 10, 1); + r = (Region *)uhash_get(regionAliases,&id); + } + + if( U_FAILURE(status) ) { + return nullptr; + } + + if ( !r ) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + + if ( r->fType == URGN_DEPRECATED && r->preferredValues->size() == 1) { + StringEnumeration *pv = r->getPreferredValues(status); + pv->reset(status); + const UnicodeString *ustr = pv->snext(status); + r = (Region *)uhash_get(regionIDMap,(void *)ustr); + delete pv; + } + + return r; +} + + +/** + * Returns an enumeration over the IDs of all known regions that match the given type. + */ +StringEnumeration* U_EXPORT2 +Region::getAvailable(URegionType type, UErrorCode &status) { + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); // returns immediately if U_FAILURE(status) + if (U_FAILURE(status)) { + return nullptr; + } + return new RegionNameEnumeration(availableRegions[type],status); +} + +/** + * Returns a pointer to the region that contains this region. Returns nullptr if this region is code "001" (World) + * or "ZZ" (Unknown region). For example, calling this method with region "IT" (Italy) returns the + * region "039" (Southern Europe). + */ +const Region* +Region::getContainingRegion() const { + UErrorCode status = U_ZERO_ERROR; + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); + return containingRegion; +} + +/** + * Return a pointer to the region that geographically contains this region and matches the given type, + * moving multiple steps up the containment chain if necessary. Returns nullptr if no containing region can be found + * that matches the given type. Note: The URegionTypes = "URGN_GROUPING", "URGN_DEPRECATED", or "URGN_UNKNOWN" + * are not appropriate for use in this API. nullptr will be returned in this case. For example, calling this method + * with region "IT" (Italy) for type "URGN_CONTINENT" returns the region "150" ( Europe ). + */ +const Region* +Region::getContainingRegion(URegionType type) const { + UErrorCode status = U_ZERO_ERROR; + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); + if ( containingRegion == nullptr ) { + return nullptr; + } + + return ( containingRegion->fType == type)? containingRegion: containingRegion->getContainingRegion(type); +} + +/** + * Return an enumeration over the IDs of all the regions that are immediate children of this region in the + * region hierarchy. These returned regions could be either macro regions, territories, or a mixture of the two, + * depending on the containment data as defined in CLDR. This API may return nullptr if this region doesn't have + * any sub-regions. For example, calling this method with region "150" (Europe) returns an enumeration containing + * the various sub regions of Europe - "039" (Southern Europe) - "151" (Eastern Europe) - "154" (Northern Europe) + * and "155" (Western Europe). + */ +StringEnumeration* +Region::getContainedRegions(UErrorCode &status) const { + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); // returns immediately if U_FAILURE(status) + if (U_FAILURE(status)) { + return nullptr; + } + return new RegionNameEnumeration(containedRegions,status); +} + +/** + * Returns an enumeration over the IDs of all the regions that are children of this region anywhere in the region + * hierarchy and match the given type. This API may return an empty enumeration if this region doesn't have any + * sub-regions that match the given type. For example, calling this method with region "150" (Europe) and type + * "URGN_TERRITORY" returns a set containing all the territories in Europe ( "FR" (France) - "IT" (Italy) - "DE" (Germany) etc. ) + */ +StringEnumeration* +Region::getContainedRegions( URegionType type, UErrorCode &status ) const { + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); // returns immediately if U_FAILURE(status) + + UVector result(nullptr, uhash_compareChars, status); + LocalPointer cr(getContainedRegions(status), status); + if (U_FAILURE(status)) { + return nullptr; + } + + const char *regionId; + while((regionId = cr->next(nullptr, status)) != nullptr && U_SUCCESS(status)) { + const Region *r = Region::getInstance(regionId, status); + if ( r->getType() == type) { + result.addElement(const_cast(&r->idStr), status); + } else { + LocalPointer children(r->getContainedRegions(type, status)); + const char *id2; + while(U_SUCCESS(status) && ((id2 = children->next(nullptr, status)) != nullptr)) { + const Region *r2 = Region::getInstance(id2,status); + result.addElement(const_cast(&r2->idStr), status); + } + } + } + LocalPointer resultEnumeration( + new RegionNameEnumeration(&result, status), status); + return U_SUCCESS(status) ? resultEnumeration.orphan() : nullptr; +} + +/** + * Returns true if this region contains the supplied other region anywhere in the region hierarchy. + */ +UBool +Region::contains(const Region &other) const { + UErrorCode status = U_ZERO_ERROR; + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); + + if (!containedRegions) { + return false; + } + if (containedRegions->contains((void *)&other.idStr)) { + return true; + } else { + for ( int32_t i = 0 ; i < containedRegions->size() ; i++ ) { + UnicodeString *crStr = (UnicodeString *)containedRegions->elementAt(i); + Region *cr = (Region *) uhash_get(regionIDMap,(void *)crStr); + if ( cr && cr->contains(other) ) { + return true; + } + } + } + + return false; +} + +/** + * For deprecated regions, return an enumeration over the IDs of the regions that are the preferred replacement + * regions for this region. Returns nullptr for a non-deprecated region. For example, calling this method with region + * "SU" (Soviet Union) would return a list of the regions containing "RU" (Russia), "AM" (Armenia), "AZ" (Azerbaijan), etc... + */ +StringEnumeration* +Region::getPreferredValues(UErrorCode &status) const { + umtx_initOnce(gRegionDataInitOnce, &loadRegionData, status); // returns immediately if U_FAILURE(status) + if (U_FAILURE(status) || fType != URGN_DEPRECATED) { + return nullptr; + } + return new RegionNameEnumeration(preferredValues,status); +} + + +/** + * Return this region's canonical region code. + */ +const char* +Region::getRegionCode() const { + return id; +} + +int32_t +Region::getNumericCode() const { + return code; +} + +/** + * Returns the region type of this region. + */ +URegionType +Region::getType() const { + return fType; +} + +RegionNameEnumeration::RegionNameEnumeration(UVector *nameList, UErrorCode& status) : + pos(0), fRegionNames(nullptr) { + // TODO: https://unicode-org.atlassian.net/browse/ICU-21829 + // Is all of the copying going on here really necessary? + if (nameList && U_SUCCESS(status)) { + LocalPointer regionNames( + new UVector(uprv_deleteUObject, uhash_compareUnicodeString, nameList->size(), status), status); + for ( int32_t i = 0 ; U_SUCCESS(status) && i < nameList->size() ; i++ ) { + UnicodeString* this_region_name = (UnicodeString *)nameList->elementAt(i); + LocalPointer new_region_name(new UnicodeString(*this_region_name), status); + regionNames->adoptElement(new_region_name.orphan(), status); + } + if (U_SUCCESS(status)) { + fRegionNames = regionNames.orphan(); + } + } +} + +const UnicodeString* +RegionNameEnumeration::snext(UErrorCode& status) { + if (U_FAILURE(status) || (fRegionNames==nullptr)) { + return nullptr; + } + const UnicodeString* nextStr = (const UnicodeString *)fRegionNames->elementAt(pos); + if (nextStr!=nullptr) { + pos++; + } + return nextStr; +} + +void +RegionNameEnumeration::reset(UErrorCode& /*status*/) { + pos=0; +} + +int32_t +RegionNameEnumeration::count(UErrorCode& /*status*/) const { + return (fRegionNames==nullptr) ? 0 : fRegionNames->size(); +} + +RegionNameEnumeration::~RegionNameEnumeration() { + delete fRegionNames; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +//eof diff --git a/intl/icu/source/i18n/region_impl.h b/intl/icu/source/i18n/region_impl.h new file mode 100644 index 0000000000..edabc6d0fe --- /dev/null +++ b/intl/icu/source/i18n/region_impl.h @@ -0,0 +1,49 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2013, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +* +* File REGION_IMPL.H +* +******************************************************************************* +*/ + +#ifndef __REGION_IMPL_H__ +#define __REGION_IMPL_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "uvector.h" +#include "unicode/strenum.h" + +U_NAMESPACE_BEGIN + + +class RegionNameEnumeration : public StringEnumeration { +public: + /** + * Construct an string enumeration over the supplied name list. + * Makes a copy of the supplied input name list; does not retain a reference to the original. + */ + RegionNameEnumeration(UVector *nameList, UErrorCode& status); + virtual ~RegionNameEnumeration(); + static UClassID U_EXPORT2 getStaticClassID(); + virtual UClassID getDynamicClassID() const override; + virtual const UnicodeString* snext(UErrorCode& status) override; + virtual void reset(UErrorCode& status) override; + virtual int32_t count(UErrorCode& status) const override; +private: + int32_t pos; + UVector *fRegionNames; +}; + +U_NAMESPACE_END + +#endif + +#endif diff --git a/intl/icu/source/i18n/reldatefmt.cpp b/intl/icu/source/i18n/reldatefmt.cpp new file mode 100644 index 0000000000..24d22a4b4b --- /dev/null +++ b/intl/icu/source/i18n/reldatefmt.cpp @@ -0,0 +1,1424 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* Copyright (C) 2014-2016, International Business Machines Corporation and +* others. All Rights Reserved. +****************************************************************************** +* +* File reldatefmt.cpp +****************************************************************************** +*/ + +#include "unicode/reldatefmt.h" + +#if !UCONFIG_NO_FORMATTING && !UCONFIG_NO_BREAK_ITERATION + +#include +#include +#include "unicode/calendar.h" +#include "unicode/datefmt.h" +#include "unicode/dtfmtsym.h" +#include "unicode/ucasemap.h" +#include "unicode/ureldatefmt.h" +#include "unicode/udisplaycontext.h" +#include "unicode/unum.h" +#include "unicode/localpointer.h" +#include "unicode/plurrule.h" +#include "unicode/simpleformatter.h" +#include "unicode/decimfmt.h" +#include "unicode/numfmt.h" +#include "unicode/brkiter.h" +#include "unicode/simpleformatter.h" +#include "uresimp.h" +#include "unicode/ures.h" +#include "cstring.h" +#include "ucln_in.h" +#include "mutex.h" +#include "charstr.h" +#include "uassert.h" +#include "quantityformatter.h" +#include "resource.h" +#include "sharedbreakiterator.h" +#include "sharedpluralrules.h" +#include "sharednumberformat.h" +#include "standardplural.h" +#include "unifiedcache.h" +#include "util.h" +#include "formatted_string_builder.h" +#include "number_utypes.h" +#include "number_modifiers.h" +#include "formattedval_impl.h" +#include "number_utils.h" + +// Copied from uscript_props.cpp + +U_NAMESPACE_BEGIN + +// RelativeDateTimeFormatter specific data for a single locale +class RelativeDateTimeCacheData: public SharedObject { +public: + RelativeDateTimeCacheData() : combinedDateAndTime(nullptr) { + // Initialize the cache arrays + for (int32_t style = 0; style < UDAT_STYLE_COUNT; ++style) { + for (int32_t relUnit = 0; relUnit < UDAT_REL_UNIT_COUNT; ++relUnit) { + for (int32_t pl = 0; pl < StandardPlural::COUNT; ++pl) { + relativeUnitsFormatters[style][relUnit][0][pl] = nullptr; + relativeUnitsFormatters[style][relUnit][1][pl] = nullptr; + } + } + } + for (int32_t i = 0; i < UDAT_STYLE_COUNT; ++i) { + fallBackCache[i] = -1; + } + } + virtual ~RelativeDateTimeCacheData(); + + // no numbers: e.g Next Tuesday; Yesterday; etc. + UnicodeString absoluteUnits[UDAT_STYLE_COUNT][UDAT_ABSOLUTE_UNIT_COUNT][UDAT_DIRECTION_COUNT]; + + // SimpleFormatter pointers for relative unit format, + // e.g., Next Tuesday; Yesterday; etc. For third index, 0 + // means past, e.g., 5 days ago; 1 means future, e.g., in 5 days. + SimpleFormatter *relativeUnitsFormatters[UDAT_STYLE_COUNT] + [UDAT_REL_UNIT_COUNT][2][StandardPlural::COUNT]; + + const UnicodeString& getAbsoluteUnitString(int32_t fStyle, + UDateAbsoluteUnit unit, + UDateDirection direction) const; + const SimpleFormatter* getRelativeUnitFormatter(int32_t fStyle, + UDateRelativeUnit unit, + int32_t pastFutureIndex, + int32_t pluralUnit) const; + const SimpleFormatter* getRelativeDateTimeUnitFormatter(int32_t fStyle, + URelativeDateTimeUnit unit, + int32_t pastFutureIndex, + int32_t pluralUnit) const; + + const UnicodeString emptyString; + + // Mapping from source to target styles for alias fallback. + int32_t fallBackCache[UDAT_STYLE_COUNT]; + + void adoptCombinedDateAndTime(SimpleFormatter *fmtToAdopt) { + delete combinedDateAndTime; + combinedDateAndTime = fmtToAdopt; + } + const SimpleFormatter *getCombinedDateAndTime() const { + return combinedDateAndTime; + } + +private: + SimpleFormatter *combinedDateAndTime; + RelativeDateTimeCacheData(const RelativeDateTimeCacheData &other); + RelativeDateTimeCacheData& operator=( + const RelativeDateTimeCacheData &other); +}; + +RelativeDateTimeCacheData::~RelativeDateTimeCacheData() { + // clear out the cache arrays + for (int32_t style = 0; style < UDAT_STYLE_COUNT; ++style) { + for (int32_t relUnit = 0; relUnit < UDAT_REL_UNIT_COUNT; ++relUnit) { + for (int32_t pl = 0; pl < StandardPlural::COUNT; ++pl) { + delete relativeUnitsFormatters[style][relUnit][0][pl]; + delete relativeUnitsFormatters[style][relUnit][1][pl]; + } + } + } + delete combinedDateAndTime; +} + + +// Use fallback cache for absolute units. +const UnicodeString& RelativeDateTimeCacheData::getAbsoluteUnitString( + int32_t fStyle, UDateAbsoluteUnit unit, UDateDirection direction) const { + int32_t style = fStyle; + do { + if (!absoluteUnits[style][unit][direction].isEmpty()) { + return absoluteUnits[style][unit][direction]; + } + style = fallBackCache[style]; + } while (style != -1); + return emptyString; +} + + const SimpleFormatter* RelativeDateTimeCacheData::getRelativeUnitFormatter( + int32_t fStyle, + UDateRelativeUnit unit, + int32_t pastFutureIndex, + int32_t pluralUnit) const { + URelativeDateTimeUnit rdtunit = UDAT_REL_UNIT_COUNT; + switch (unit) { + case UDAT_RELATIVE_YEARS: rdtunit = UDAT_REL_UNIT_YEAR; break; + case UDAT_RELATIVE_MONTHS: rdtunit = UDAT_REL_UNIT_MONTH; break; + case UDAT_RELATIVE_WEEKS: rdtunit = UDAT_REL_UNIT_WEEK; break; + case UDAT_RELATIVE_DAYS: rdtunit = UDAT_REL_UNIT_DAY; break; + case UDAT_RELATIVE_HOURS: rdtunit = UDAT_REL_UNIT_HOUR; break; + case UDAT_RELATIVE_MINUTES: rdtunit = UDAT_REL_UNIT_MINUTE; break; + case UDAT_RELATIVE_SECONDS: rdtunit = UDAT_REL_UNIT_SECOND; break; + default: // a unit that the above method does not handle + return nullptr; + } + + return getRelativeDateTimeUnitFormatter(fStyle, rdtunit, pastFutureIndex, pluralUnit); + } + + // Use fallback cache for SimpleFormatter relativeUnits. + const SimpleFormatter* RelativeDateTimeCacheData::getRelativeDateTimeUnitFormatter( + int32_t fStyle, + URelativeDateTimeUnit unit, + int32_t pastFutureIndex, + int32_t pluralUnit) const { + while (true) { + int32_t style = fStyle; + do { + if (relativeUnitsFormatters[style][unit][pastFutureIndex][pluralUnit] != nullptr) { + return relativeUnitsFormatters[style][unit][pastFutureIndex][pluralUnit]; + } + style = fallBackCache[style]; + } while (style != -1); + + if (pluralUnit == StandardPlural::OTHER) { + break; + } + pluralUnit = StandardPlural::OTHER; + } + return nullptr; // No formatter found. + } + +static UBool getStringByIndex( + const UResourceBundle *resource, + int32_t idx, + UnicodeString &result, + UErrorCode &status) { + int32_t len = 0; + const char16_t *resStr = ures_getStringByIndex( + resource, idx, &len, &status); + if (U_FAILURE(status)) { + return false; + } + result.setTo(true, resStr, len); + return true; +} + +namespace { + +/** + * Sink for enumerating all of the measurement unit display names. + * + * More specific bundles (en_GB) are enumerated before their parents (en_001, en, root): + * Only store a value if it is still missing, that is, it has not been overridden. + */ +struct RelDateTimeFmtDataSink : public ResourceSink { + + /** + * Sink for patterns for relative dates and times. For example, + * fields/relative/... + */ + + // Generic unit enum for storing Unit info. + typedef enum RelAbsUnit { + INVALID_UNIT = -1, + SECOND, + MINUTE, + HOUR, + DAY, + WEEK, + MONTH, + QUARTER, + YEAR, + SUNDAY, + MONDAY, + TUESDAY, + WEDNESDAY, + THURSDAY, + FRIDAY, + SATURDAY + } RelAbsUnit; + + static int32_t relUnitFromGeneric(RelAbsUnit genUnit) { + // Converts the generic units to UDAT_RELATIVE version. + switch (genUnit) { + case SECOND: + return UDAT_REL_UNIT_SECOND; + case MINUTE: + return UDAT_REL_UNIT_MINUTE; + case HOUR: + return UDAT_REL_UNIT_HOUR; + case DAY: + return UDAT_REL_UNIT_DAY; + case WEEK: + return UDAT_REL_UNIT_WEEK; + case MONTH: + return UDAT_REL_UNIT_MONTH; + case QUARTER: + return UDAT_REL_UNIT_QUARTER; + case YEAR: + return UDAT_REL_UNIT_YEAR; + case SUNDAY: + return UDAT_REL_UNIT_SUNDAY; + case MONDAY: + return UDAT_REL_UNIT_MONDAY; + case TUESDAY: + return UDAT_REL_UNIT_TUESDAY; + case WEDNESDAY: + return UDAT_REL_UNIT_WEDNESDAY; + case THURSDAY: + return UDAT_REL_UNIT_THURSDAY; + case FRIDAY: + return UDAT_REL_UNIT_FRIDAY; + case SATURDAY: + return UDAT_REL_UNIT_SATURDAY; + default: + return -1; + } + } + + static int32_t absUnitFromGeneric(RelAbsUnit genUnit) { + // Converts the generic units to UDAT_RELATIVE version. + switch (genUnit) { + case DAY: + return UDAT_ABSOLUTE_DAY; + case WEEK: + return UDAT_ABSOLUTE_WEEK; + case MONTH: + return UDAT_ABSOLUTE_MONTH; + case QUARTER: + return UDAT_ABSOLUTE_QUARTER; + case YEAR: + return UDAT_ABSOLUTE_YEAR; + case SUNDAY: + return UDAT_ABSOLUTE_SUNDAY; + case MONDAY: + return UDAT_ABSOLUTE_MONDAY; + case TUESDAY: + return UDAT_ABSOLUTE_TUESDAY; + case WEDNESDAY: + return UDAT_ABSOLUTE_WEDNESDAY; + case THURSDAY: + return UDAT_ABSOLUTE_THURSDAY; + case FRIDAY: + return UDAT_ABSOLUTE_FRIDAY; + case SATURDAY: + return UDAT_ABSOLUTE_SATURDAY; + case HOUR: + return UDAT_ABSOLUTE_HOUR; + case MINUTE: + return UDAT_ABSOLUTE_MINUTE; + default: + return -1; + } + } + + static int32_t keyToDirection(const char* key) { + if (uprv_strcmp(key, "-2") == 0) { + return UDAT_DIRECTION_LAST_2; + } + if (uprv_strcmp(key, "-1") == 0) { + return UDAT_DIRECTION_LAST; + } + if (uprv_strcmp(key, "0") == 0) { + return UDAT_DIRECTION_THIS; + } + if (uprv_strcmp(key, "1") == 0) { + return UDAT_DIRECTION_NEXT; + } + if (uprv_strcmp(key, "2") == 0) { + return UDAT_DIRECTION_NEXT_2; + } + return -1; + } + + // Values kept between levels of parsing the CLDR data. + int32_t pastFutureIndex; // 0 == past or 1 == future + UDateRelativeDateTimeFormatterStyle style; // {LONG, SHORT, NARROW} + RelAbsUnit genericUnit; + + RelativeDateTimeCacheData &outputData; + + // Constructor + RelDateTimeFmtDataSink(RelativeDateTimeCacheData& cacheData) + : outputData(cacheData) { + // Clear cacheData.fallBackCache + cacheData.fallBackCache[UDAT_STYLE_LONG] = -1; + cacheData.fallBackCache[UDAT_STYLE_SHORT] = -1; + cacheData.fallBackCache[UDAT_STYLE_NARROW] = -1; + } + + ~RelDateTimeFmtDataSink(); + + // Utility functions + static UDateRelativeDateTimeFormatterStyle styleFromString(const char *s) { + int32_t len = static_cast(uprv_strlen(s)); + if (len >= 7 && uprv_strcmp(s + len - 7, "-narrow") == 0) { + return UDAT_STYLE_NARROW; + } + if (len >= 6 && uprv_strcmp(s + len - 6, "-short") == 0) { + return UDAT_STYLE_SHORT; + } + return UDAT_STYLE_LONG; + } + + static int32_t styleSuffixLength(UDateRelativeDateTimeFormatterStyle style) { + switch (style) { + case UDAT_STYLE_NARROW: + return 7; + case UDAT_STYLE_SHORT: + return 6; + default: + return 0; + } + } + + // Utility functions + static UDateRelativeDateTimeFormatterStyle styleFromAliasUnicodeString(UnicodeString s) { + static const char16_t narrow[7] = {0x002D, 0x006E, 0x0061, 0x0072, 0x0072, 0x006F, 0x0077}; + static const char16_t sshort[6] = {0x002D, 0x0073, 0x0068, 0x006F, 0x0072, 0x0074,}; + if (s.endsWith(narrow, 7)) { + return UDAT_STYLE_NARROW; + } + if (s.endsWith(sshort, 6)) { + return UDAT_STYLE_SHORT; + } + return UDAT_STYLE_LONG; + } + + static RelAbsUnit unitOrNegativeFromString(const char* keyword, int32_t length) { + // Quick check from string to enum. + switch (length) { + case 3: + if (uprv_strncmp(keyword, "day", length) == 0) { + return DAY; + } else if (uprv_strncmp(keyword, "sun", length) == 0) { + return SUNDAY; + } else if (uprv_strncmp(keyword, "mon", length) == 0) { + return MONDAY; + } else if (uprv_strncmp(keyword, "tue", length) == 0) { + return TUESDAY; + } else if (uprv_strncmp(keyword, "wed", length) == 0) { + return WEDNESDAY; + } else if (uprv_strncmp(keyword, "thu", length) == 0) { + return THURSDAY; + } else if (uprv_strncmp(keyword, "fri", length) == 0) { + return FRIDAY; + } else if (uprv_strncmp(keyword, "sat", length) == 0) { + return SATURDAY; + } + break; + case 4: + if (uprv_strncmp(keyword, "hour", length) == 0) { + return HOUR; + } else if (uprv_strncmp(keyword, "week", length) == 0) { + return WEEK; + } else if (uprv_strncmp(keyword, "year", length) == 0) { + return YEAR; + } + break; + case 5: + if (uprv_strncmp(keyword, "month", length) == 0) { + return MONTH; + } + break; + case 6: + if (uprv_strncmp(keyword, "minute", length) == 0) { + return MINUTE; + } else if (uprv_strncmp(keyword, "second", length) == 0) { + return SECOND; + } + break; + case 7: + if (uprv_strncmp(keyword, "quarter", length) == 0) { + return QUARTER; // TODO: Check @provisional + } + break; + default: + break; + } + return INVALID_UNIT; + } + + void handlePlainDirection(ResourceValue &value, UErrorCode &errorCode) { + // Handle Display Name for PLAIN direction for some units. + if (U_FAILURE(errorCode)) { return; } + + int32_t absUnit = absUnitFromGeneric(genericUnit); + if (absUnit < 0) { + return; // Not interesting. + } + + // Store displayname if not set. + if (outputData.absoluteUnits[style] + [absUnit][UDAT_DIRECTION_PLAIN].isEmpty()) { + outputData.absoluteUnits[style] + [absUnit][UDAT_DIRECTION_PLAIN].fastCopyFrom(value.getUnicodeString(errorCode)); + return; + } + } + + void consumeTableRelative(const char *key, ResourceValue &value, UErrorCode &errorCode) { + ResourceTable unitTypesTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; unitTypesTable.getKeyAndValue(i, key, value); ++i) { + if (value.getType() == URES_STRING) { + int32_t direction = keyToDirection(key); + if (direction < 0) { + continue; + } + + int32_t relUnitIndex = relUnitFromGeneric(genericUnit); + if (relUnitIndex == UDAT_REL_UNIT_SECOND && uprv_strcmp(key, "0") == 0 && + outputData.absoluteUnits[style][UDAT_ABSOLUTE_NOW][UDAT_DIRECTION_PLAIN].isEmpty()) { + // Handle "NOW" + outputData.absoluteUnits[style][UDAT_ABSOLUTE_NOW] + [UDAT_DIRECTION_PLAIN].fastCopyFrom(value.getUnicodeString(errorCode)); + } + + int32_t absUnitIndex = absUnitFromGeneric(genericUnit); + if (absUnitIndex < 0) { + continue; + } + // Only reset if slot is empty. + if (outputData.absoluteUnits[style][absUnitIndex][direction].isEmpty()) { + outputData.absoluteUnits[style][absUnitIndex] + [direction].fastCopyFrom(value.getUnicodeString(errorCode)); + } + } + } + } + + void consumeTimeDetail(int32_t relUnitIndex, + const char *key, ResourceValue &value, UErrorCode &errorCode) { + ResourceTable unitTypesTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; unitTypesTable.getKeyAndValue(i, key, value); ++i) { + if (value.getType() == URES_STRING) { + int32_t pluralIndex = StandardPlural::indexOrNegativeFromString(key); + if (pluralIndex >= 0) { + SimpleFormatter **patterns = + outputData.relativeUnitsFormatters[style][relUnitIndex] + [pastFutureIndex]; + // Only set if not already established. + if (patterns[pluralIndex] == nullptr) { + patterns[pluralIndex] = new SimpleFormatter( + value.getUnicodeString(errorCode), 0, 1, errorCode); + if (patterns[pluralIndex] == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } + } + } + } + } + } + + void consumeTableRelativeTime(const char *key, ResourceValue &value, UErrorCode &errorCode) { + ResourceTable relativeTimeTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + int32_t relUnitIndex = relUnitFromGeneric(genericUnit); + if (relUnitIndex < 0) { + return; + } + for (int32_t i = 0; relativeTimeTable.getKeyAndValue(i, key, value); ++i) { + if (uprv_strcmp(key, "past") == 0) { + pastFutureIndex = 0; + } else if (uprv_strcmp(key, "future") == 0) { + pastFutureIndex = 1; + } else { + // Unknown key. + continue; + } + consumeTimeDetail(relUnitIndex, key, value, errorCode); + } + } + + void consumeAlias(const char *key, const ResourceValue &value, UErrorCode &errorCode) { + + UDateRelativeDateTimeFormatterStyle sourceStyle = styleFromString(key); + const UnicodeString valueStr = value.getAliasUnicodeString(errorCode); + if (U_FAILURE(errorCode)) { return; } + + UDateRelativeDateTimeFormatterStyle targetStyle = + styleFromAliasUnicodeString(valueStr); + + if (sourceStyle == targetStyle) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + if (outputData.fallBackCache[sourceStyle] != -1 && + outputData.fallBackCache[sourceStyle] != targetStyle) { + errorCode = U_INVALID_FORMAT_ERROR; + return; + } + outputData.fallBackCache[sourceStyle] = targetStyle; + } + + void consumeTimeUnit(const char *key, ResourceValue &value, UErrorCode &errorCode) { + ResourceTable unitTypesTable = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + + for (int32_t i = 0; unitTypesTable.getKeyAndValue(i, key, value); ++i) { + // Handle display name. + if (uprv_strcmp(key, "dn") == 0 && value.getType() == URES_STRING) { + handlePlainDirection(value, errorCode); + } + if (value.getType() == URES_TABLE) { + if (uprv_strcmp(key, "relative") == 0) { + consumeTableRelative(key, value, errorCode); + } else if (uprv_strcmp(key, "relativeTime") == 0) { + consumeTableRelativeTime(key, value, errorCode); + } + } + } + } + + virtual void put(const char *key, ResourceValue &value, + UBool /*noFallback*/, UErrorCode &errorCode) override { + // Main entry point to sink + ResourceTable table = value.getTable(errorCode); + if (U_FAILURE(errorCode)) { return; } + for (int32_t i = 0; table.getKeyAndValue(i, key, value); ++i) { + if (value.getType() == URES_ALIAS) { + consumeAlias(key, value, errorCode); + } else { + style = styleFromString(key); + int32_t unitSize = static_cast(uprv_strlen(key)) - styleSuffixLength(style); + genericUnit = unitOrNegativeFromString(key, unitSize); + if (style >= 0 && genericUnit != INVALID_UNIT) { + consumeTimeUnit(key, value, errorCode); + } + } + } + } + +}; + +// Virtual destructors must be defined out of line. +RelDateTimeFmtDataSink::~RelDateTimeFmtDataSink() {} +} // namespace + +static const DateFormatSymbols::DtWidthType styleToDateFormatSymbolWidth[UDAT_STYLE_COUNT] = { + DateFormatSymbols::WIDE, DateFormatSymbols::SHORT, DateFormatSymbols::NARROW +}; + +// Get days of weeks from the DateFormatSymbols class. +static void loadWeekdayNames(UnicodeString absoluteUnits[UDAT_STYLE_COUNT] + [UDAT_ABSOLUTE_UNIT_COUNT][UDAT_DIRECTION_COUNT], + const char* localeId, + UErrorCode& status) { + if (U_FAILURE(status)) { + return; + } + Locale locale(localeId); + DateFormatSymbols dfSym(locale, status); + if (U_FAILURE(status)) { + return; + } + for (int32_t style = 0; style < UDAT_STYLE_COUNT; ++style) { + DateFormatSymbols::DtWidthType dtfmtWidth = styleToDateFormatSymbolWidth[style]; + int32_t count; + const UnicodeString* weekdayNames = + dfSym.getWeekdays(count, DateFormatSymbols::STANDALONE, dtfmtWidth); + for (int32_t dayIndex = UDAT_ABSOLUTE_SUNDAY; + dayIndex <= UDAT_ABSOLUTE_SATURDAY; ++ dayIndex) { + int32_t dateSymbolIndex = (dayIndex - UDAT_ABSOLUTE_SUNDAY) + UCAL_SUNDAY; + absoluteUnits[style][dayIndex][UDAT_DIRECTION_PLAIN].fastCopyFrom( + weekdayNames[dateSymbolIndex]); + } + } +} + +static UBool loadUnitData( + const UResourceBundle *resource, + RelativeDateTimeCacheData &cacheData, + const char* localeId, + UErrorCode &status) { + + RelDateTimeFmtDataSink sink(cacheData); + + ures_getAllItemsWithFallback(resource, "fields", sink, status); + if (U_FAILURE(status)) { + return false; + } + + // Get the weekday names from DateFormatSymbols. + loadWeekdayNames(cacheData.absoluteUnits, localeId, status); + return U_SUCCESS(status); +} + +static const int32_t cTypeBufMax = 32; + +static UBool getDateTimePattern( + Locale locale, + const UResourceBundle *resource, + UnicodeString &result, + UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + char cType[cTypeBufMax + 1]; + Calendar::getCalendarTypeFromLocale(locale, cType, cTypeBufMax, status); + cType[cTypeBufMax] = 0; + if (U_FAILURE(status) || cType[0] == 0) { + status = U_ZERO_ERROR; + uprv_strcpy(cType, "gregorian"); + } + + LocalUResourceBundlePointer topLevel; + int32_t dateTimeFormatOffset = DateFormat::kMedium; + CharString pathBuffer; + // Currently, for compatibility with pre-CLDR-42 data, we default to the "atTime" + // combining patterns. Depending on guidance in CLDR 42 spec and on DisplayOptions, + // we may change this. + pathBuffer.append("calendar/", status) + .append(cType, status) + .append("/DateTimePatterns%atTime", status); + topLevel.adoptInstead( + ures_getByKeyWithFallback( + resource, pathBuffer.data(), nullptr, &status)); + if (U_FAILURE(status) || ures_getSize(topLevel.getAlias()) < 4) { + // Fall back to standard combining patterns + status = U_ZERO_ERROR; + dateTimeFormatOffset = DateFormat::kDateTime; + pathBuffer.clear(); + pathBuffer.append("calendar/", status) + .append(cType, status) + .append("/DateTimePatterns", status); + topLevel.adoptInstead( + ures_getByKeyWithFallback( + resource, pathBuffer.data(), nullptr, &status)); + } + if (U_FAILURE(status)) { + return false; + } + if (dateTimeFormatOffset == DateFormat::kDateTime && ures_getSize(topLevel.getAlias()) <= DateFormat::kDateTime) { + // Oops, size is too small to access the index that we want, fallback + // to a hard-coded value. + result = UNICODE_STRING_SIMPLE("{1} {0}"); + return true; + } + return getStringByIndex(topLevel.getAlias(), dateTimeFormatOffset, result, status); +} + +template<> +const RelativeDateTimeCacheData *LocaleCacheKey::createObject(const void * /*unused*/, UErrorCode &status) const { + const char *localeId = fLoc.getName(); + LocalUResourceBundlePointer topLevel(ures_open(nullptr, localeId, &status)); + if (U_FAILURE(status)) { + return nullptr; + } + LocalPointer result( + new RelativeDateTimeCacheData()); + if (result.isNull()) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if (!loadUnitData( + topLevel.getAlias(), + *result, + localeId, + status)) { + return nullptr; + } + UnicodeString dateTimePattern; + if (!getDateTimePattern(fLoc, topLevel.getAlias(), dateTimePattern, status)) { + return nullptr; + } + result->adoptCombinedDateAndTime( + new SimpleFormatter(dateTimePattern, 2, 2, status)); + if (U_FAILURE(status)) { + return nullptr; + } + result->addRef(); + return result.orphan(); +} + + + +static constexpr FormattedStringBuilder::Field kRDTNumericField + = {UFIELD_CATEGORY_RELATIVE_DATETIME, UDAT_REL_NUMERIC_FIELD}; + +static constexpr FormattedStringBuilder::Field kRDTLiteralField + = {UFIELD_CATEGORY_RELATIVE_DATETIME, UDAT_REL_LITERAL_FIELD}; + +class FormattedRelativeDateTimeData : public FormattedValueStringBuilderImpl { +public: + FormattedRelativeDateTimeData() : FormattedValueStringBuilderImpl(kRDTNumericField) {} + virtual ~FormattedRelativeDateTimeData(); +}; + +FormattedRelativeDateTimeData::~FormattedRelativeDateTimeData() = default; + + +UPRV_FORMATTED_VALUE_SUBCLASS_AUTO_IMPL(FormattedRelativeDateTime) + + +RelativeDateTimeFormatter::RelativeDateTimeFormatter(UErrorCode& status) : + fCache(nullptr), + fNumberFormat(nullptr), + fPluralRules(nullptr), + fStyle(UDAT_STYLE_LONG), + fContext(UDISPCTX_CAPITALIZATION_NONE), + fOptBreakIterator(nullptr) { + init(nullptr, nullptr, status); +} + +RelativeDateTimeFormatter::RelativeDateTimeFormatter( + const Locale& locale, UErrorCode& status) : + fCache(nullptr), + fNumberFormat(nullptr), + fPluralRules(nullptr), + fStyle(UDAT_STYLE_LONG), + fContext(UDISPCTX_CAPITALIZATION_NONE), + fOptBreakIterator(nullptr), + fLocale(locale) { + init(nullptr, nullptr, status); +} + +RelativeDateTimeFormatter::RelativeDateTimeFormatter( + const Locale& locale, NumberFormat *nfToAdopt, UErrorCode& status) : + fCache(nullptr), + fNumberFormat(nullptr), + fPluralRules(nullptr), + fStyle(UDAT_STYLE_LONG), + fContext(UDISPCTX_CAPITALIZATION_NONE), + fOptBreakIterator(nullptr), + fLocale(locale) { + init(nfToAdopt, nullptr, status); +} + +RelativeDateTimeFormatter::RelativeDateTimeFormatter( + const Locale& locale, + NumberFormat *nfToAdopt, + UDateRelativeDateTimeFormatterStyle styl, + UDisplayContext capitalizationContext, + UErrorCode& status) : + fCache(nullptr), + fNumberFormat(nullptr), + fPluralRules(nullptr), + fStyle(styl), + fContext(capitalizationContext), + fOptBreakIterator(nullptr), + fLocale(locale) { + if (U_FAILURE(status)) { + return; + } + if ((capitalizationContext >> 8) != UDISPCTX_TYPE_CAPITALIZATION) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if (capitalizationContext == UDISPCTX_CAPITALIZATION_FOR_BEGINNING_OF_SENTENCE) { + BreakIterator *bi = BreakIterator::createSentenceInstance(locale, status); + if (U_FAILURE(status)) { + return; + } + init(nfToAdopt, bi, status); + } else { + init(nfToAdopt, nullptr, status); + } +} + +RelativeDateTimeFormatter::RelativeDateTimeFormatter( + const RelativeDateTimeFormatter& other) + : UObject(other), + fCache(other.fCache), + fNumberFormat(other.fNumberFormat), + fPluralRules(other.fPluralRules), + fStyle(other.fStyle), + fContext(other.fContext), + fOptBreakIterator(other.fOptBreakIterator), + fLocale(other.fLocale) { + fCache->addRef(); + fNumberFormat->addRef(); + fPluralRules->addRef(); + if (fOptBreakIterator != nullptr) { + fOptBreakIterator->addRef(); + } +} + +RelativeDateTimeFormatter& RelativeDateTimeFormatter::operator=( + const RelativeDateTimeFormatter& other) { + if (this != &other) { + SharedObject::copyPtr(other.fCache, fCache); + SharedObject::copyPtr(other.fNumberFormat, fNumberFormat); + SharedObject::copyPtr(other.fPluralRules, fPluralRules); + SharedObject::copyPtr(other.fOptBreakIterator, fOptBreakIterator); + fStyle = other.fStyle; + fContext = other.fContext; + fLocale = other.fLocale; + } + return *this; +} + +RelativeDateTimeFormatter::~RelativeDateTimeFormatter() { + if (fCache != nullptr) { + fCache->removeRef(); + } + if (fNumberFormat != nullptr) { + fNumberFormat->removeRef(); + } + if (fPluralRules != nullptr) { + fPluralRules->removeRef(); + } + if (fOptBreakIterator != nullptr) { + fOptBreakIterator->removeRef(); + } +} + +const NumberFormat& RelativeDateTimeFormatter::getNumberFormat() const { + return **fNumberFormat; +} + +UDisplayContext RelativeDateTimeFormatter::getCapitalizationContext() const { + return fContext; +} + +UDateRelativeDateTimeFormatterStyle RelativeDateTimeFormatter::getFormatStyle() const { + return fStyle; +} + + +// To reduce boilerplate code, we use a helper function that forwards variadic +// arguments to the formatImpl function. + +template +UnicodeString& RelativeDateTimeFormatter::doFormat( + F callback, + UnicodeString& appendTo, + UErrorCode& status, + Args... args) const { + FormattedRelativeDateTimeData output; + (this->*callback)(std::forward(args)..., output, status); + if (U_FAILURE(status)) { + return appendTo; + } + UnicodeString result = output.getStringRef().toUnicodeString(); + return appendTo.append(adjustForContext(result)); +} + +template +FormattedRelativeDateTime RelativeDateTimeFormatter::doFormatToValue( + F callback, + UErrorCode& status, + Args... args) const { + if (!checkNoAdjustForContext(status)) { + return FormattedRelativeDateTime(status); + } + LocalPointer output( + new FormattedRelativeDateTimeData(), status); + if (U_FAILURE(status)) { + return FormattedRelativeDateTime(status); + } + (this->*callback)(std::forward(args)..., *output, status); + output->getStringRef().writeTerminator(status); + return FormattedRelativeDateTime(output.orphan()); +} + +UnicodeString& RelativeDateTimeFormatter::format( + double quantity, + UDateDirection direction, + UDateRelativeUnit unit, + UnicodeString& appendTo, + UErrorCode& status) const { + return doFormat( + &RelativeDateTimeFormatter::formatImpl, + appendTo, + status, + quantity, + direction, + unit); +} + +FormattedRelativeDateTime RelativeDateTimeFormatter::formatToValue( + double quantity, + UDateDirection direction, + UDateRelativeUnit unit, + UErrorCode& status) const { + return doFormatToValue( + &RelativeDateTimeFormatter::formatImpl, + status, + quantity, + direction, + unit); +} + +void RelativeDateTimeFormatter::formatImpl( + double quantity, + UDateDirection direction, + UDateRelativeUnit unit, + FormattedRelativeDateTimeData& output, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + if (direction != UDAT_DIRECTION_LAST && direction != UDAT_DIRECTION_NEXT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + int32_t bFuture = direction == UDAT_DIRECTION_NEXT ? 1 : 0; + + StandardPlural::Form pluralForm; + QuantityFormatter::formatAndSelect( + quantity, + **fNumberFormat, + **fPluralRules, + output.getStringRef(), + pluralForm, + status); + if (U_FAILURE(status)) { + return; + } + + const SimpleFormatter* formatter = + fCache->getRelativeUnitFormatter(fStyle, unit, bFuture, pluralForm); + if (formatter == nullptr) { + // TODO: WARN - look at quantity formatter's action with an error. + status = U_INVALID_FORMAT_ERROR; + return; + } + + number::impl::SimpleModifier modifier(*formatter, kRDTLiteralField, false); + modifier.formatAsPrefixSuffix( + output.getStringRef(), 0, output.getStringRef().length(), status); +} + +UnicodeString& RelativeDateTimeFormatter::formatNumeric( + double offset, + URelativeDateTimeUnit unit, + UnicodeString& appendTo, + UErrorCode& status) const { + return doFormat( + &RelativeDateTimeFormatter::formatNumericImpl, + appendTo, + status, + offset, + unit); +} + +FormattedRelativeDateTime RelativeDateTimeFormatter::formatNumericToValue( + double offset, + URelativeDateTimeUnit unit, + UErrorCode& status) const { + return doFormatToValue( + &RelativeDateTimeFormatter::formatNumericImpl, + status, + offset, + unit); +} + +void RelativeDateTimeFormatter::formatNumericImpl( + double offset, + URelativeDateTimeUnit unit, + FormattedRelativeDateTimeData& output, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + UDateDirection direction = UDAT_DIRECTION_NEXT; + if (std::signbit(offset)) { // needed to handle -0.0 + direction = UDAT_DIRECTION_LAST; + offset = -offset; + } + if (direction != UDAT_DIRECTION_LAST && direction != UDAT_DIRECTION_NEXT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + int32_t bFuture = direction == UDAT_DIRECTION_NEXT ? 1 : 0; + + StandardPlural::Form pluralForm; + QuantityFormatter::formatAndSelect( + offset, + **fNumberFormat, + **fPluralRules, + output.getStringRef(), + pluralForm, + status); + if (U_FAILURE(status)) { + return; + } + + const SimpleFormatter* formatter = + fCache->getRelativeDateTimeUnitFormatter(fStyle, unit, bFuture, pluralForm); + if (formatter == nullptr) { + // TODO: WARN - look at quantity formatter's action with an error. + status = U_INVALID_FORMAT_ERROR; + return; + } + + number::impl::SimpleModifier modifier(*formatter, kRDTLiteralField, false); + modifier.formatAsPrefixSuffix( + output.getStringRef(), 0, output.getStringRef().length(), status); +} + +UnicodeString& RelativeDateTimeFormatter::format( + UDateDirection direction, + UDateAbsoluteUnit unit, + UnicodeString& appendTo, + UErrorCode& status) const { + return doFormat( + &RelativeDateTimeFormatter::formatAbsoluteImpl, + appendTo, + status, + direction, + unit); +} + +FormattedRelativeDateTime RelativeDateTimeFormatter::formatToValue( + UDateDirection direction, + UDateAbsoluteUnit unit, + UErrorCode& status) const { + return doFormatToValue( + &RelativeDateTimeFormatter::formatAbsoluteImpl, + status, + direction, + unit); +} + +void RelativeDateTimeFormatter::formatAbsoluteImpl( + UDateDirection direction, + UDateAbsoluteUnit unit, + FormattedRelativeDateTimeData& output, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + if (unit == UDAT_ABSOLUTE_NOW && direction != UDAT_DIRECTION_PLAIN) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + // Get string using fallback. + output.getStringRef().append( + fCache->getAbsoluteUnitString(fStyle, unit, direction), + kRDTLiteralField, + status); +} + +UnicodeString& RelativeDateTimeFormatter::format( + double offset, + URelativeDateTimeUnit unit, + UnicodeString& appendTo, + UErrorCode& status) const { + return doFormat( + &RelativeDateTimeFormatter::formatRelativeImpl, + appendTo, + status, + offset, + unit); +} + +FormattedRelativeDateTime RelativeDateTimeFormatter::formatToValue( + double offset, + URelativeDateTimeUnit unit, + UErrorCode& status) const { + return doFormatToValue( + &RelativeDateTimeFormatter::formatRelativeImpl, + status, + offset, + unit); +} + +void RelativeDateTimeFormatter::formatRelativeImpl( + double offset, + URelativeDateTimeUnit unit, + FormattedRelativeDateTimeData& output, + UErrorCode& status) const { + if (U_FAILURE(status)) { + return; + } + // TODO: + // The full implementation of this depends on CLDR data that is not yet available, + // see: http://unicode.org/cldr/trac/ticket/9165 Add more relative field data. + // In the meantime do a quick bring-up by calling the old format method; this + // leaves some holes (even for data that is currently available, such as quarter). + // When the new CLDR data is available, update the data storage accordingly, + // rewrite this to use it directly, and rewrite the old format method to call this + // new one; that is covered by https://unicode-org.atlassian.net/browse/ICU-12171. + UDateDirection direction = UDAT_DIRECTION_COUNT; + if (offset > -2.1 && offset < 2.1) { + // Allow a 1% epsilon, so offsets in -1.01..-0.99 map to LAST + double offsetx100 = offset * 100.0; + int32_t intoffset = (offsetx100 < 0)? (int32_t)(offsetx100-0.5) : (int32_t)(offsetx100+0.5); + switch (intoffset) { + case -200/*-2*/: direction = UDAT_DIRECTION_LAST_2; break; + case -100/*-1*/: direction = UDAT_DIRECTION_LAST; break; + case 0/* 0*/: direction = UDAT_DIRECTION_THIS; break; + case 100/* 1*/: direction = UDAT_DIRECTION_NEXT; break; + case 200/* 2*/: direction = UDAT_DIRECTION_NEXT_2; break; + default: break; + } + } + UDateAbsoluteUnit absunit = UDAT_ABSOLUTE_UNIT_COUNT; + switch (unit) { + case UDAT_REL_UNIT_YEAR: absunit = UDAT_ABSOLUTE_YEAR; break; + case UDAT_REL_UNIT_QUARTER: absunit = UDAT_ABSOLUTE_QUARTER; break; + case UDAT_REL_UNIT_MONTH: absunit = UDAT_ABSOLUTE_MONTH; break; + case UDAT_REL_UNIT_WEEK: absunit = UDAT_ABSOLUTE_WEEK; break; + case UDAT_REL_UNIT_DAY: absunit = UDAT_ABSOLUTE_DAY; break; + case UDAT_REL_UNIT_SECOND: + if (direction == UDAT_DIRECTION_THIS) { + absunit = UDAT_ABSOLUTE_NOW; + direction = UDAT_DIRECTION_PLAIN; + } + break; + case UDAT_REL_UNIT_SUNDAY: absunit = UDAT_ABSOLUTE_SUNDAY; break; + case UDAT_REL_UNIT_MONDAY: absunit = UDAT_ABSOLUTE_MONDAY; break; + case UDAT_REL_UNIT_TUESDAY: absunit = UDAT_ABSOLUTE_TUESDAY; break; + case UDAT_REL_UNIT_WEDNESDAY: absunit = UDAT_ABSOLUTE_WEDNESDAY; break; + case UDAT_REL_UNIT_THURSDAY: absunit = UDAT_ABSOLUTE_THURSDAY; break; + case UDAT_REL_UNIT_FRIDAY: absunit = UDAT_ABSOLUTE_FRIDAY; break; + case UDAT_REL_UNIT_SATURDAY: absunit = UDAT_ABSOLUTE_SATURDAY; break; + case UDAT_REL_UNIT_HOUR: absunit = UDAT_ABSOLUTE_HOUR; break; + case UDAT_REL_UNIT_MINUTE: absunit = UDAT_ABSOLUTE_MINUTE; break; + default: break; + } + if (direction != UDAT_DIRECTION_COUNT && absunit != UDAT_ABSOLUTE_UNIT_COUNT) { + formatAbsoluteImpl(direction, absunit, output, status); + if (output.getStringRef().length() != 0) { + return; + } + } + // otherwise fallback to formatNumeric + formatNumericImpl(offset, unit, output, status); +} + +UnicodeString& RelativeDateTimeFormatter::combineDateAndTime( + const UnicodeString& relativeDateString, const UnicodeString& timeString, + UnicodeString& appendTo, UErrorCode& status) const { + return fCache->getCombinedDateAndTime()->format( + timeString, relativeDateString, appendTo, status); +} + +UnicodeString& RelativeDateTimeFormatter::adjustForContext(UnicodeString &str) const { + if (fOptBreakIterator == nullptr + || str.length() == 0 || !u_islower(str.char32At(0))) { + return str; + } + + // Must guarantee that one thread at a time accesses the shared break + // iterator. + static UMutex gBrkIterMutex; + Mutex lock(&gBrkIterMutex); + str.toTitle( + fOptBreakIterator->get(), + fLocale, + U_TITLECASE_NO_LOWERCASE | U_TITLECASE_NO_BREAK_ADJUSTMENT); + return str; +} + +UBool RelativeDateTimeFormatter::checkNoAdjustForContext(UErrorCode& status) const { + // This is unsupported because it's hard to keep fields in sync with title + // casing. The code could be written and tested if there is demand. + if (fOptBreakIterator != nullptr) { + status = U_UNSUPPORTED_ERROR; + return false; + } + return true; +} + +void RelativeDateTimeFormatter::init( + NumberFormat *nfToAdopt, + BreakIterator *biToAdopt, + UErrorCode &status) { + LocalPointer nf(nfToAdopt); + LocalPointer bi(biToAdopt); + UnifiedCache::getByLocale(fLocale, fCache, status); + if (U_FAILURE(status)) { + return; + } + const SharedPluralRules *pr = PluralRules::createSharedInstance( + fLocale, UPLURAL_TYPE_CARDINAL, status); + if (U_FAILURE(status)) { + return; + } + SharedObject::copyPtr(pr, fPluralRules); + pr->removeRef(); + if (nf.isNull()) { + const SharedNumberFormat *shared = NumberFormat::createSharedInstance( + fLocale, UNUM_DECIMAL, status); + if (U_FAILURE(status)) { + return; + } + SharedObject::copyPtr(shared, fNumberFormat); + shared->removeRef(); + } else { + SharedNumberFormat *shared = new SharedNumberFormat(nf.getAlias()); + if (shared == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + nf.orphan(); + SharedObject::copyPtr(shared, fNumberFormat); + } + if (bi.isNull()) { + SharedObject::clearPtr(fOptBreakIterator); + } else { + SharedBreakIterator *shared = new SharedBreakIterator(bi.getAlias()); + if (shared == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + bi.orphan(); + SharedObject::copyPtr(shared, fOptBreakIterator); + } +} + +U_NAMESPACE_END + +// Plain C API + +U_NAMESPACE_USE + + +// Magic number: "FRDT" (FormattedRelativeDateTime) in ASCII +UPRV_FORMATTED_VALUE_CAPI_AUTO_IMPL( + FormattedRelativeDateTime, + UFormattedRelativeDateTime, + UFormattedRelativeDateTimeImpl, + UFormattedRelativeDateTimeApiHelper, + ureldatefmt, + 0x46524454) + + +U_CAPI URelativeDateTimeFormatter* U_EXPORT2 +ureldatefmt_open( const char* locale, + UNumberFormat* nfToAdopt, + UDateRelativeDateTimeFormatterStyle width, + UDisplayContext capitalizationContext, + UErrorCode* status ) +{ + if (U_FAILURE(*status)) { + return nullptr; + } + LocalPointer formatter(new RelativeDateTimeFormatter(Locale(locale), + (NumberFormat*)nfToAdopt, width, + capitalizationContext, *status), *status); + if (U_FAILURE(*status)) { + return nullptr; + } + return (URelativeDateTimeFormatter*)formatter.orphan(); +} + +U_CAPI void U_EXPORT2 +ureldatefmt_close(URelativeDateTimeFormatter *reldatefmt) +{ + delete (RelativeDateTimeFormatter*)reldatefmt; +} + +U_CAPI int32_t U_EXPORT2 +ureldatefmt_formatNumeric( const URelativeDateTimeFormatter* reldatefmt, + double offset, + URelativeDateTimeUnit unit, + char16_t* result, + int32_t resultCapacity, + UErrorCode* status) +{ + if (U_FAILURE(*status)) { + return 0; + } + if (result == nullptr ? resultCapacity != 0 : resultCapacity < 0) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UnicodeString res; + if (result != nullptr) { + // nullptr destination for pure preflighting: empty dummy string + // otherwise, alias the destination buffer (copied from udat_format) + res.setTo(result, 0, resultCapacity); + } + ((RelativeDateTimeFormatter*)reldatefmt)->formatNumeric(offset, unit, res, *status); + if (U_FAILURE(*status)) { + return 0; + } + return res.extract(result, resultCapacity, *status); +} + +U_CAPI void U_EXPORT2 +ureldatefmt_formatNumericToResult( + const URelativeDateTimeFormatter* reldatefmt, + double offset, + URelativeDateTimeUnit unit, + UFormattedRelativeDateTime* result, + UErrorCode* status) { + if (U_FAILURE(*status)) { + return; + } + auto* fmt = reinterpret_cast(reldatefmt); + auto* resultImpl = UFormattedRelativeDateTimeApiHelper::validate(result, *status); + resultImpl->fImpl = fmt->formatNumericToValue(offset, unit, *status); +} + +U_CAPI int32_t U_EXPORT2 +ureldatefmt_format( const URelativeDateTimeFormatter* reldatefmt, + double offset, + URelativeDateTimeUnit unit, + char16_t* result, + int32_t resultCapacity, + UErrorCode* status) +{ + if (U_FAILURE(*status)) { + return 0; + } + if (result == nullptr ? resultCapacity != 0 : resultCapacity < 0) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UnicodeString res; + if (result != nullptr) { + // nullptr destination for pure preflighting: empty dummy string + // otherwise, alias the destination buffer (copied from udat_format) + res.setTo(result, 0, resultCapacity); + } + ((RelativeDateTimeFormatter*)reldatefmt)->format(offset, unit, res, *status); + if (U_FAILURE(*status)) { + return 0; + } + return res.extract(result, resultCapacity, *status); +} + +U_CAPI void U_EXPORT2 +ureldatefmt_formatToResult( + const URelativeDateTimeFormatter* reldatefmt, + double offset, + URelativeDateTimeUnit unit, + UFormattedRelativeDateTime* result, + UErrorCode* status) { + if (U_FAILURE(*status)) { + return; + } + auto* fmt = reinterpret_cast(reldatefmt); + auto* resultImpl = UFormattedRelativeDateTimeApiHelper::validate(result, *status); + resultImpl->fImpl = fmt->formatToValue(offset, unit, *status); +} + +U_CAPI int32_t U_EXPORT2 +ureldatefmt_combineDateAndTime( const URelativeDateTimeFormatter* reldatefmt, + const char16_t * relativeDateString, + int32_t relativeDateStringLen, + const char16_t * timeString, + int32_t timeStringLen, + char16_t* result, + int32_t resultCapacity, + UErrorCode* status ) +{ + if (U_FAILURE(*status)) { + return 0; + } + if (result == nullptr ? resultCapacity != 0 : resultCapacity < 0 || + (relativeDateString == nullptr ? relativeDateStringLen != 0 : relativeDateStringLen < -1) || + (timeString == nullptr ? timeStringLen != 0 : timeStringLen < -1)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UnicodeString relDateStr((UBool)(relativeDateStringLen == -1), relativeDateString, relativeDateStringLen); + UnicodeString timeStr((UBool)(timeStringLen == -1), timeString, timeStringLen); + UnicodeString res(result, 0, resultCapacity); + ((RelativeDateTimeFormatter*)reldatefmt)->combineDateAndTime(relDateStr, timeStr, res, *status); + if (U_FAILURE(*status)) { + return 0; + } + return res.extract(result, resultCapacity, *status); +} + +#endif /* !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/reldtfmt.cpp b/intl/icu/source/i18n/reldtfmt.cpp new file mode 100644 index 0000000000..0c836a0b79 --- /dev/null +++ b/intl/icu/source/i18n/reldtfmt.cpp @@ -0,0 +1,595 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and +* others. All Rights Reserved. +******************************************************************************* +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include + +#include "unicode/datefmt.h" +#include "unicode/reldatefmt.h" +#include "unicode/simpleformatter.h" +#include "unicode/smpdtfmt.h" +#include "unicode/udisplaycontext.h" +#include "unicode/uchar.h" +#include "unicode/brkiter.h" +#include "unicode/ucasemap.h" +#include "reldtfmt.h" +#include "cmemory.h" +#include "uresimp.h" + +U_NAMESPACE_BEGIN + + +/** + * An array of URelativeString structs is used to store the resource data loaded out of the bundle. + */ +struct URelativeString { + int32_t offset; /** offset of this item, such as, the relative date **/ + int32_t len; /** length of the string **/ + const char16_t* string; /** string, or nullptr if not set **/ +}; + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RelativeDateFormat) + +RelativeDateFormat::RelativeDateFormat(const RelativeDateFormat& other) : + DateFormat(other), fDateTimeFormatter(nullptr), fDatePattern(other.fDatePattern), + fTimePattern(other.fTimePattern), fCombinedFormat(nullptr), + fDateStyle(other.fDateStyle), fLocale(other.fLocale), + fDatesLen(other.fDatesLen), fDates(nullptr), + fCombinedHasDateAtStart(other.fCombinedHasDateAtStart), + fCapitalizationInfoSet(other.fCapitalizationInfoSet), + fCapitalizationOfRelativeUnitsForUIListMenu(other.fCapitalizationOfRelativeUnitsForUIListMenu), + fCapitalizationOfRelativeUnitsForStandAlone(other.fCapitalizationOfRelativeUnitsForStandAlone), + fCapitalizationBrkIter(nullptr) +{ + if(other.fDateTimeFormatter != nullptr) { + fDateTimeFormatter = other.fDateTimeFormatter->clone(); + } + if(other.fCombinedFormat != nullptr) { + fCombinedFormat = new SimpleFormatter(*other.fCombinedFormat); + } + if (fDatesLen > 0) { + fDates = (URelativeString*) uprv_malloc(sizeof(fDates[0])*(size_t)fDatesLen); + uprv_memcpy(fDates, other.fDates, sizeof(fDates[0])*(size_t)fDatesLen); + } +#if !UCONFIG_NO_BREAK_ITERATION + if (other.fCapitalizationBrkIter != nullptr) { + fCapitalizationBrkIter = (other.fCapitalizationBrkIter)->clone(); + } +#endif +} + +RelativeDateFormat::RelativeDateFormat( UDateFormatStyle timeStyle, UDateFormatStyle dateStyle, + const Locale& locale, UErrorCode& status) : + DateFormat(), fDateTimeFormatter(nullptr), fDatePattern(), fTimePattern(), fCombinedFormat(nullptr), + fDateStyle(dateStyle), fLocale(locale), fDatesLen(0), fDates(nullptr), + fCombinedHasDateAtStart(false), fCapitalizationInfoSet(false), + fCapitalizationOfRelativeUnitsForUIListMenu(false), fCapitalizationOfRelativeUnitsForStandAlone(false), + fCapitalizationBrkIter(nullptr) +{ + if(U_FAILURE(status) ) { + return; + } + + if (timeStyle < UDAT_NONE || timeStyle > UDAT_SHORT) { + // don't support other time styles (e.g. relative styles), for now + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + UDateFormatStyle baseDateStyle = (dateStyle > UDAT_SHORT)? (UDateFormatStyle)(dateStyle & ~UDAT_RELATIVE): dateStyle; + DateFormat * df; + // Get fDateTimeFormatter from either date or time style (does not matter, we will override the pattern). + // We do need to get separate patterns for the date & time styles. + if (baseDateStyle != UDAT_NONE) { + df = createDateInstance((EStyle)baseDateStyle, locale); + fDateTimeFormatter=dynamic_cast(df); + if (fDateTimeFormatter == nullptr) { + status = U_UNSUPPORTED_ERROR; + return; + } + fDateTimeFormatter->toPattern(fDatePattern); + if (timeStyle != UDAT_NONE) { + df = createTimeInstance((EStyle)timeStyle, locale); + SimpleDateFormat *sdf = dynamic_cast(df); + if (sdf != nullptr) { + sdf->toPattern(fTimePattern); + delete sdf; + } + } + } else { + // does not matter whether timeStyle is UDAT_NONE, we need something for fDateTimeFormatter + df = createTimeInstance((EStyle)timeStyle, locale); + fDateTimeFormatter=dynamic_cast(df); + if (fDateTimeFormatter == nullptr) { + status = U_UNSUPPORTED_ERROR; + delete df; + return; + } + fDateTimeFormatter->toPattern(fTimePattern); + } + + // Initialize the parent fCalendar, so that parse() works correctly. + initializeCalendar(nullptr, locale, status); + loadDates(status); +} + +RelativeDateFormat::~RelativeDateFormat() { + delete fDateTimeFormatter; + delete fCombinedFormat; + uprv_free(fDates); +#if !UCONFIG_NO_BREAK_ITERATION + delete fCapitalizationBrkIter; +#endif +} + + +RelativeDateFormat* RelativeDateFormat::clone() const { + return new RelativeDateFormat(*this); +} + +bool RelativeDateFormat::operator==(const Format& other) const { + if(DateFormat::operator==(other)) { + // The DateFormat::operator== check for fCapitalizationContext equality above + // is sufficient to check equality of all derived context-related data. + // DateFormat::operator== guarantees following cast is safe + RelativeDateFormat* that = (RelativeDateFormat*)&other; + return (fDateStyle==that->fDateStyle && + fDatePattern==that->fDatePattern && + fTimePattern==that->fTimePattern && + fLocale==that->fLocale ); + } + return false; +} + +static const char16_t APOSTROPHE = (char16_t)0x0027; + +UnicodeString& RelativeDateFormat::format( Calendar& cal, + UnicodeString& appendTo, + FieldPosition& pos) const { + + UErrorCode status = U_ZERO_ERROR; + UnicodeString relativeDayString; + UDisplayContext capitalizationContext = getContext(UDISPCTX_TYPE_CAPITALIZATION, status); + + // calculate the difference, in days, between 'cal' and now. + int dayDiff = dayDifference(cal, status); + + // look up string + int32_t len = 0; + const char16_t *theString = getStringForDay(dayDiff, len, status); + if(U_SUCCESS(status) && (theString!=nullptr)) { + // found a relative string + relativeDayString.setTo(theString, len); + } + + if ( relativeDayString.length() > 0 && !fDatePattern.isEmpty() && + (fTimePattern.isEmpty() || fCombinedFormat == nullptr || fCombinedHasDateAtStart)) { +#if !UCONFIG_NO_BREAK_ITERATION + // capitalize relativeDayString according to context for relative, set formatter no context + if ( u_islower(relativeDayString.char32At(0)) && fCapitalizationBrkIter!= nullptr && + ( capitalizationContext==UDISPCTX_CAPITALIZATION_FOR_BEGINNING_OF_SENTENCE || + (capitalizationContext==UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU && fCapitalizationOfRelativeUnitsForUIListMenu) || + (capitalizationContext==UDISPCTX_CAPITALIZATION_FOR_STANDALONE && fCapitalizationOfRelativeUnitsForStandAlone) ) ) { + // titlecase first word of relativeDayString + relativeDayString.toTitle(fCapitalizationBrkIter, fLocale, U_TITLECASE_NO_LOWERCASE | U_TITLECASE_NO_BREAK_ADJUSTMENT); + } +#endif + fDateTimeFormatter->setContext(UDISPCTX_CAPITALIZATION_NONE, status); + } else { + // set our context for the formatter + fDateTimeFormatter->setContext(capitalizationContext, status); + } + + if (fDatePattern.isEmpty()) { + fDateTimeFormatter->applyPattern(fTimePattern); + fDateTimeFormatter->format(cal,appendTo,pos); + } else if (fTimePattern.isEmpty() || fCombinedFormat == nullptr) { + if (relativeDayString.length() > 0) { + appendTo.append(relativeDayString); + } else { + fDateTimeFormatter->applyPattern(fDatePattern); + fDateTimeFormatter->format(cal,appendTo,pos); + } + } else { + UnicodeString datePattern; + if (relativeDayString.length() > 0) { + // Need to quote the relativeDayString to make it a legal date pattern + relativeDayString.findAndReplace(UNICODE_STRING("'", 1), UNICODE_STRING("''", 2)); // double any existing APOSTROPHE + relativeDayString.insert(0, APOSTROPHE); // add APOSTROPHE at beginning... + relativeDayString.append(APOSTROPHE); // and at end + datePattern.setTo(relativeDayString); + } else { + datePattern.setTo(fDatePattern); + } + UnicodeString combinedPattern; + fCombinedFormat->format(fTimePattern, datePattern, combinedPattern, status); + fDateTimeFormatter->applyPattern(combinedPattern); + fDateTimeFormatter->format(cal,appendTo,pos); + } + + return appendTo; +} + + + +UnicodeString& +RelativeDateFormat::format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const +{ + // this is just here to get around the hiding problem + // (the previous format() override would hide the version of + // format() on DateFormat that this function correspond to, so we + // have to redefine it here) + return DateFormat::format(obj, appendTo, pos, status); +} + + +void RelativeDateFormat::parse( const UnicodeString& text, + Calendar& cal, + ParsePosition& pos) const { + + int32_t startIndex = pos.getIndex(); + if (fDatePattern.isEmpty()) { + // no date pattern, try parsing as time + fDateTimeFormatter->applyPattern(fTimePattern); + fDateTimeFormatter->parse(text,cal,pos); + } else if (fTimePattern.isEmpty() || fCombinedFormat == nullptr) { + // no time pattern or way to combine, try parsing as date + // first check whether text matches a relativeDayString + UBool matchedRelative = false; + for (int n=0; n < fDatesLen && !matchedRelative; n++) { + if (fDates[n].string != nullptr && + text.compare(startIndex, fDates[n].len, fDates[n].string) == 0) { + // it matched, handle the relative day string + UErrorCode status = U_ZERO_ERROR; + matchedRelative = true; + + // Set the calendar to now+offset + cal.setTime(Calendar::getNow(),status); + cal.add(UCAL_DATE,fDates[n].offset, status); + + if(U_FAILURE(status)) { + // failure in setting calendar field, set offset to beginning of rel day string + pos.setErrorIndex(startIndex); + } else { + pos.setIndex(startIndex + fDates[n].len); + } + } + } + if (!matchedRelative) { + // just parse as normal date + fDateTimeFormatter->applyPattern(fDatePattern); + fDateTimeFormatter->parse(text,cal,pos); + } + } else { + // Here we replace any relativeDayString in text with the equivalent date + // formatted per fDatePattern, then parse text normally using the combined pattern. + UnicodeString modifiedText(text); + FieldPosition fPos; + int32_t dateStart = 0, origDateLen = 0, modDateLen = 0; + UErrorCode status = U_ZERO_ERROR; + for (int n=0; n < fDatesLen; n++) { + int32_t relativeStringOffset; + if (fDates[n].string != nullptr && + (relativeStringOffset = modifiedText.indexOf(fDates[n].string, fDates[n].len, startIndex)) >= startIndex) { + // it matched, replace the relative date with a real one for parsing + UnicodeString dateString; + Calendar * tempCal = cal.clone(); + + // Set the calendar to now+offset + tempCal->setTime(Calendar::getNow(),status); + tempCal->add(UCAL_DATE,fDates[n].offset, status); + if(U_FAILURE(status)) { + pos.setErrorIndex(startIndex); + delete tempCal; + return; + } + + fDateTimeFormatter->applyPattern(fDatePattern); + fDateTimeFormatter->format(*tempCal, dateString, fPos); + dateStart = relativeStringOffset; + origDateLen = fDates[n].len; + modDateLen = dateString.length(); + modifiedText.replace(dateStart, origDateLen, dateString); + delete tempCal; + break; + } + } + UnicodeString combinedPattern; + fCombinedFormat->format(fTimePattern, fDatePattern, combinedPattern, status); + fDateTimeFormatter->applyPattern(combinedPattern); + fDateTimeFormatter->parse(modifiedText,cal,pos); + + // Adjust offsets + UBool noError = (pos.getErrorIndex() < 0); + int32_t offset = (noError)? pos.getIndex(): pos.getErrorIndex(); + if (offset >= dateStart + modDateLen) { + // offset at or after the end of the replaced text, + // correct by the difference between original and replacement + offset -= (modDateLen - origDateLen); + } else if (offset >= dateStart) { + // offset in the replaced text, set it to the beginning of that text + // (i.e. the beginning of the relative day string) + offset = dateStart; + } + if (noError) { + pos.setIndex(offset); + } else { + pos.setErrorIndex(offset); + } + } +} + +UDate +RelativeDateFormat::parse( const UnicodeString& text, + ParsePosition& pos) const { + // redefined here because the other parse() function hides this function's + // counterpart on DateFormat + return DateFormat::parse(text, pos); +} + +UDate +RelativeDateFormat::parse(const UnicodeString& text, UErrorCode& status) const +{ + // redefined here because the other parse() function hides this function's + // counterpart on DateFormat + return DateFormat::parse(text, status); +} + + +const char16_t *RelativeDateFormat::getStringForDay(int32_t day, int32_t &len, UErrorCode &status) const { + if(U_FAILURE(status)) { + return nullptr; + } + + // Is it inside the resource bundle's range? + int n = day + UDAT_DIRECTION_THIS; + if (n >= 0 && n < fDatesLen) { + if (fDates[n].offset == day && fDates[n].string != nullptr) { + len = fDates[n].len; + return fDates[n].string; + } + } + return nullptr; // not found. +} + +UnicodeString& +RelativeDateFormat::toPattern(UnicodeString& result, UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + result.remove(); + if (fDatePattern.isEmpty()) { + result.setTo(fTimePattern); + } else if (fTimePattern.isEmpty() || fCombinedFormat == nullptr) { + result.setTo(fDatePattern); + } else { + fCombinedFormat->format(fTimePattern, fDatePattern, result, status); + } + } + return result; +} + +UnicodeString& +RelativeDateFormat::toPatternDate(UnicodeString& result, UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + result.remove(); + result.setTo(fDatePattern); + } + return result; +} + +UnicodeString& +RelativeDateFormat::toPatternTime(UnicodeString& result, UErrorCode& status) const +{ + if (!U_FAILURE(status)) { + result.remove(); + result.setTo(fTimePattern); + } + return result; +} + +void +RelativeDateFormat::applyPatterns(const UnicodeString& datePattern, const UnicodeString& timePattern, UErrorCode &status) +{ + if (!U_FAILURE(status)) { + fDatePattern.setTo(datePattern); + fTimePattern.setTo(timePattern); + } +} + +const DateFormatSymbols* +RelativeDateFormat::getDateFormatSymbols() const +{ + return fDateTimeFormatter->getDateFormatSymbols(); +} + +// override the DateFormat implementation in order to +// lazily initialize relevant items +void +RelativeDateFormat::setContext(UDisplayContext value, UErrorCode& status) +{ + DateFormat::setContext(value, status); + if (U_SUCCESS(status)) { + if (!fCapitalizationInfoSet && + (value==UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU || value==UDISPCTX_CAPITALIZATION_FOR_STANDALONE)) { + initCapitalizationContextInfo(fLocale); + fCapitalizationInfoSet = true; + } +#if !UCONFIG_NO_BREAK_ITERATION + if ( fCapitalizationBrkIter == nullptr && (value==UDISPCTX_CAPITALIZATION_FOR_BEGINNING_OF_SENTENCE || + (value==UDISPCTX_CAPITALIZATION_FOR_UI_LIST_OR_MENU && fCapitalizationOfRelativeUnitsForUIListMenu) || + (value==UDISPCTX_CAPITALIZATION_FOR_STANDALONE && fCapitalizationOfRelativeUnitsForStandAlone)) ) { + status = U_ZERO_ERROR; + fCapitalizationBrkIter = BreakIterator::createSentenceInstance(fLocale, status); + if (U_FAILURE(status)) { + delete fCapitalizationBrkIter; + fCapitalizationBrkIter = nullptr; + } + } +#endif + } +} + +void +RelativeDateFormat::initCapitalizationContextInfo(const Locale& thelocale) +{ +#if !UCONFIG_NO_BREAK_ITERATION + const char * localeID = (thelocale != nullptr)? thelocale.getBaseName(): nullptr; + UErrorCode status = U_ZERO_ERROR; + LocalUResourceBundlePointer rb(ures_open(nullptr, localeID, &status)); + ures_getByKeyWithFallback(rb.getAlias(), + "contextTransforms/relative", + rb.getAlias(), &status); + if (U_SUCCESS(status) && rb != nullptr) { + int32_t len = 0; + const int32_t * intVector = ures_getIntVector(rb.getAlias(), + &len, &status); + if (U_SUCCESS(status) && intVector != nullptr && len >= 2) { + fCapitalizationOfRelativeUnitsForUIListMenu = static_cast(intVector[0]); + fCapitalizationOfRelativeUnitsForStandAlone = static_cast(intVector[1]); + } + } +#endif +} + +namespace { + +/** + * Sink for getting data from fields/day/relative data. + * For loading relative day names, e.g., "yesterday", "today". + */ + +struct RelDateFmtDataSink : public ResourceSink { + URelativeString *fDatesPtr; + int32_t fDatesLen; + + RelDateFmtDataSink(URelativeString* fDates, int32_t len) : fDatesPtr(fDates), fDatesLen(len) { + for (int32_t i = 0; i < fDatesLen; ++i) { + fDatesPtr[i].offset = 0; + fDatesPtr[i].string = nullptr; + fDatesPtr[i].len = -1; + } + } + + virtual ~RelDateFmtDataSink(); + + virtual void put(const char *key, ResourceValue &value, + UBool /*noFallback*/, UErrorCode &errorCode) override { + ResourceTable relDayTable = value.getTable(errorCode); + int32_t n = 0; + int32_t len = 0; + for (int32_t i = 0; relDayTable.getKeyAndValue(i, key, value); ++i) { + // Find the relative offset. + int32_t offset = atoi(key); + + // Put in the proper spot, but don't override existing data. + n = offset + UDAT_DIRECTION_THIS; // Converts to index in UDAT_R + if (n < fDatesLen && fDatesPtr[n].string == nullptr) { + // Not found and n is an empty slot. + fDatesPtr[n].offset = offset; + fDatesPtr[n].string = value.getString(len, errorCode); + fDatesPtr[n].len = len; + } + } + } +}; + + +// Virtual destructors must be defined out of line. +RelDateFmtDataSink::~RelDateFmtDataSink() {} + +} // Namespace + + +static const char16_t patItem1[] = {0x7B,0x31,0x7D}; // "{1}" +static const int32_t patItem1Len = 3; + +void RelativeDateFormat::loadDates(UErrorCode &status) { + UResourceBundle *rb = ures_open(nullptr, fLocale.getBaseName(), &status); + LocalUResourceBundlePointer dateTimePatterns( + ures_getByKeyWithFallback(rb, + "calendar/gregorian/DateTimePatterns", + (UResourceBundle*)nullptr, &status)); + if(U_SUCCESS(status)) { + int32_t patternsSize = ures_getSize(dateTimePatterns.getAlias()); + if (patternsSize > kDateTime) { + int32_t resStrLen = 0; + int32_t glueIndex = kDateTime; + if (patternsSize >= (kDateTimeOffset + kShort + 1)) { + int32_t offsetIncrement = (fDateStyle & ~kRelative); // Remove relative bit. + if (offsetIncrement >= (int32_t)kFull && + offsetIncrement <= (int32_t)kShortRelative) { + glueIndex = kDateTimeOffset + offsetIncrement; + } + } + + const char16_t *resStr = ures_getStringByIndex(dateTimePatterns.getAlias(), glueIndex, &resStrLen, &status); + if (U_SUCCESS(status) && resStrLen >= patItem1Len && u_strncmp(resStr,patItem1,patItem1Len)==0) { + fCombinedHasDateAtStart = true; + } + fCombinedFormat = new SimpleFormatter(UnicodeString(true, resStr, resStrLen), 2, 2, status); + } + } + + // Data loading for relative names, e.g., "yesterday", "today", "tomorrow". + fDatesLen = UDAT_DIRECTION_COUNT; // Maximum defined by data. + fDates = (URelativeString*) uprv_malloc(sizeof(fDates[0])*fDatesLen); + + RelDateFmtDataSink sink(fDates, fDatesLen); + ures_getAllItemsWithFallback(rb, "fields/day/relative", sink, status); + + ures_close(rb); + + if(U_FAILURE(status)) { + fDatesLen=0; + return; + } +} + +//---------------------------------------------------------------------- + +// this should to be in DateFormat, instead it was copied from SimpleDateFormat. + +Calendar* +RelativeDateFormat::initializeCalendar(TimeZone* adoptZone, const Locale& locale, UErrorCode& status) +{ + if(!U_FAILURE(status)) { + fCalendar = Calendar::createInstance(adoptZone?adoptZone:TimeZone::createDefault(), locale, status); + } + if (U_SUCCESS(status) && fCalendar == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } + return fCalendar; +} + +int32_t RelativeDateFormat::dayDifference(Calendar &cal, UErrorCode &status) { + if(U_FAILURE(status)) { + return 0; + } + // TODO: Cache the nowCal to avoid heap allocs? Would be difficult, don't know the calendar type + Calendar *nowCal = cal.clone(); + nowCal->setTime(Calendar::getNow(), status); + + // For the day difference, we are interested in the difference in the (modified) julian day number + // which is midnight to midnight. Using fieldDifference() is NOT correct here, because + // 6pm Jan 4th to 10am Jan 5th should be considered "tomorrow". + int32_t dayDiff = cal.get(UCAL_JULIAN_DAY, status) - nowCal->get(UCAL_JULIAN_DAY, status); + + delete nowCal; + return dayDiff; +} + +U_NAMESPACE_END + +#endif /* !UCONFIG_NO_FORMATTING */ diff --git a/intl/icu/source/i18n/reldtfmt.h b/intl/icu/source/i18n/reldtfmt.h new file mode 100644 index 0000000000..3af43d49ca --- /dev/null +++ b/intl/icu/source/i18n/reldtfmt.h @@ -0,0 +1,338 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 2007-2016, International Business Machines Corporation and * +* others. All Rights Reserved. * +******************************************************************************* +*/ + +#ifndef RELDTFMT_H +#define RELDTFMT_H + +#include "unicode/utypes.h" + +/** + * \file + * \brief C++ API: Format and parse relative dates and times. + */ + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/datefmt.h" +#include "unicode/smpdtfmt.h" +#include "unicode/brkiter.h" + +U_NAMESPACE_BEGIN + +// forward declarations +class DateFormatSymbols; +class SimpleFormatter; + +// internal structure used for caching strings +struct URelativeString; + +/** + * This class is normally accessed using the kRelative or k...Relative values of EStyle as + * parameters to DateFormat::createDateInstance. + * + * Example: + * DateFormat *fullrelative = DateFormat::createDateInstance(DateFormat::kFullRelative, loc); + * + * @internal ICU 3.8 + */ + +class RelativeDateFormat : public DateFormat { +public: + RelativeDateFormat( UDateFormatStyle timeStyle, UDateFormatStyle dateStyle, const Locale& locale, UErrorCode& status); + + // overrides + /** + * Copy constructor. + * @internal ICU 3.8 + */ + RelativeDateFormat(const RelativeDateFormat&); + + /** + * Assignment operator. + * @internal ICU 3.8 + */ + RelativeDateFormat& operator=(const RelativeDateFormat&); + + /** + * Destructor. + * @internal ICU 3.8 + */ + virtual ~RelativeDateFormat(); + + /** + * Clone this Format object polymorphically. The caller owns the result and + * should delete it when done. + * @return A copy of the object. + * @internal ICU 3.8 + */ + virtual RelativeDateFormat* clone() const override; + + /** + * Return true if the given Format objects are semantically equal. Objects + * of different subclasses are considered unequal. + * @param other the object to be compared with. + * @return true if the given Format objects are semantically equal. + * @internal ICU 3.8 + */ + virtual bool operator==(const Format& other) const override; + + + using DateFormat::format; + + /** + * Format a date or time, which is the standard millis since 24:00 GMT, Jan + * 1, 1970. Overrides DateFormat pure virtual method. + *

    + * Example: using the US locale: "yyyy.MM.dd e 'at' HH:mm:ss zzz" ->> + * 1996.07.10 AD at 15:08:56 PDT + * + * @param cal Calendar set to the date and time to be formatted + * into a date/time string. + * @param appendTo Output parameter to receive result. + * Result is appended to existing contents. + * @param pos The formatting position. On input: an alignment field, + * if desired. On output: the offsets of the alignment field. + * @return Reference to 'appendTo' parameter. + * @internal ICU 3.8 + */ + virtual UnicodeString& format( Calendar& cal, + UnicodeString& appendTo, + FieldPosition& pos) const override; + + /** + * Format an object to produce a string. This method handles Formattable + * objects with a UDate type. If a the Formattable object type is not a Date, + * then it returns a failing UErrorCode. + * + * @param obj The object to format. Must be a Date. + * @param appendTo Output parameter to receive result. + * Result is appended to existing contents. + * @param pos On input: an alignment field, if desired. + * On output: the offsets of the alignment field. + * @param status Output param filled with success/failure status. + * @return Reference to 'appendTo' parameter. + * @internal ICU 3.8 + */ + virtual UnicodeString& format(const Formattable& obj, + UnicodeString& appendTo, + FieldPosition& pos, + UErrorCode& status) const override; + + + /** + * Parse a date/time string beginning at the given parse position. For + * example, a time text "07/10/96 4:5 PM, PDT" will be parsed into a Date + * that is equivalent to Date(837039928046). + *

    + * By default, parsing is lenient: If the input is not in the form used by + * this object's format method but can still be parsed as a date, then the + * parse succeeds. Clients may insist on strict adherence to the format by + * calling setLenient(false). + * + * @param text The date/time string to be parsed + * @param cal a Calendar set to the date and time to be formatted + * into a date/time string. + * @param pos On input, the position at which to start parsing; on + * output, the position at which parsing terminated, or the + * start position if the parse failed. + * @return A valid UDate if the input could be parsed. + * @internal ICU 3.8 + */ + virtual void parse( const UnicodeString& text, + Calendar& cal, + ParsePosition& pos) const override; + + /** + * Parse a date/time string starting at the given parse position. For + * example, a time text "07/10/96 4:5 PM, PDT" will be parsed into a Date + * that is equivalent to Date(837039928046). + *

    + * By default, parsing is lenient: If the input is not in the form used by + * this object's format method but can still be parsed as a date, then the + * parse succeeds. Clients may insist on strict adherence to the format by + * calling setLenient(false). + * + * @see DateFormat::setLenient(boolean) + * + * @param text The date/time string to be parsed + * @param pos On input, the position at which to start parsing; on + * output, the position at which parsing terminated, or the + * start position if the parse failed. + * @return A valid UDate if the input could be parsed. + * @internal ICU 3.8 + */ + UDate parse( const UnicodeString& text, + ParsePosition& pos) const; + + + /** + * Parse a date/time string. For example, a time text "07/10/96 4:5 PM, PDT" + * will be parsed into a UDate that is equivalent to Date(837039928046). + * Parsing begins at the beginning of the string and proceeds as far as + * possible. Assuming no parse errors were encountered, this function + * doesn't return any information about how much of the string was consumed + * by the parsing. If you need that information, use the version of + * parse() that takes a ParsePosition. + * + * @param text The date/time string to be parsed + * @param status Filled in with U_ZERO_ERROR if the parse was successful, and with + * an error value if there was a parse error. + * @return A valid UDate if the input could be parsed. + * @internal ICU 3.8 + */ + virtual UDate parse( const UnicodeString& text, + UErrorCode& status) const override; + + /** + * Return a single pattern string generated by combining the patterns for the + * date and time formatters associated with this object. + * @param result Output param to receive the pattern. + * @return A reference to 'result'. + * @internal ICU 4.2 technology preview + */ + virtual UnicodeString& toPattern(UnicodeString& result, UErrorCode& status) const; + + /** + * Get the date pattern for the the date formatter associated with this object. + * @param result Output param to receive the date pattern. + * @return A reference to 'result'. + * @internal ICU 4.2 technology preview + */ + virtual UnicodeString& toPatternDate(UnicodeString& result, UErrorCode& status) const; + + /** + * Get the time pattern for the the time formatter associated with this object. + * @param result Output param to receive the time pattern. + * @return A reference to 'result'. + * @internal ICU 4.2 technology preview + */ + virtual UnicodeString& toPatternTime(UnicodeString& result, UErrorCode& status) const; + + /** + * Apply the given unlocalized date & time pattern strings to this relative date format. + * (i.e., after this call, this formatter will format dates and times according to + * the new patterns) + * + * @param datePattern The date pattern to be applied. + * @param timePattern The time pattern to be applied. + * @internal ICU 4.2 technology preview + */ + virtual void applyPatterns(const UnicodeString& datePattern, const UnicodeString& timePattern, UErrorCode &status); + + /** + * Gets the date/time formatting symbols (this is an object carrying + * the various strings and other symbols used in formatting: e.g., month + * names and abbreviations, time zone names, AM/PM strings, etc.) + * @return a copy of the date-time formatting data associated + * with this date-time formatter. + * @internal ICU 4.8 + */ + virtual const DateFormatSymbols* getDateFormatSymbols() const; + + /** + * Set a particular UDisplayContext value in the formatter, such as + * UDISPCTX_CAPITALIZATION_FOR_STANDALONE. Note: For getContext, see + * DateFormat. + * @param value The UDisplayContext value to set. + * @param status Input/output status. If at entry this indicates a failure + * status, the function will do nothing; otherwise this will be + * updated with any new status from the function. + * @internal ICU 53 + */ + virtual void setContext(UDisplayContext value, UErrorCode& status) override; + +private: + SimpleDateFormat *fDateTimeFormatter; + UnicodeString fDatePattern; + UnicodeString fTimePattern; + SimpleFormatter *fCombinedFormat; // the {0} {1} format. + + UDateFormatStyle fDateStyle; + Locale fLocale; + + int32_t fDatesLen; // Length of array + URelativeString *fDates; // array of strings + + UBool fCombinedHasDateAtStart; + UBool fCapitalizationInfoSet; + UBool fCapitalizationOfRelativeUnitsForUIListMenu; + UBool fCapitalizationOfRelativeUnitsForStandAlone; +#if !UCONFIG_NO_BREAK_ITERATION + BreakIterator* fCapitalizationBrkIter; +#else + UObject* fCapitalizationBrkIter; +#endif + + /** + * Get the string at a specific offset. + * @param day day offset ( -1, 0, 1, etc.. ) + * @param len on output, length of string. + * @return the string, or nullptr if none at that location. + */ + const char16_t *getStringForDay(int32_t day, int32_t &len, UErrorCode &status) const; + + /** + * Load the Date string array + */ + void loadDates(UErrorCode &status); + + /** + * Set fCapitalizationOfRelativeUnitsForUIListMenu, fCapitalizationOfRelativeUnitsForStandAlone + */ + void initCapitalizationContextInfo(const Locale& thelocale); + + /** + * @return the number of days in "until-now" + */ + static int32_t dayDifference(Calendar &until, UErrorCode &status); + + /** + * initializes fCalendar from parameters. Returns fCalendar as a convenience. + * @param adoptZone Zone to be adopted, or nullptr for TimeZone::createDefault(). + * @param locale Locale of the calendar + * @param status Error code + * @return the newly constructed fCalendar + * @internal ICU 3.8 + */ + Calendar* initializeCalendar(TimeZone* adoptZone, const Locale& locale, UErrorCode& status); + +public: + /** + * Return the class ID for this class. This is useful only for comparing to + * a return value from getDynamicClassID(). For example: + * 

    +     * .   Base* polymorphic_pointer = createPolymorphicObject();
+     * .   if (polymorphic_pointer->getDynamicClassID() ==
+     * .       erived::getStaticClassID()) ...
+     *
    + * @return The class ID for all objects of this class. + * @internal ICU 3.8 + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + + /** + * Returns a unique class ID POLYMORPHICALLY. Pure virtual override. This + * method is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and clone() + * methods call this method. + * + * @return The class ID for this object. All objects of a + * given class have the same class ID. Objects of + * other classes have different class IDs. + * @internal ICU 3.8 + */ + virtual UClassID getDynamicClassID() const override; +}; + + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_FORMATTING */ + +#endif // RELDTFMT_H diff --git a/intl/icu/source/i18n/rematch.cpp b/intl/icu/source/i18n/rematch.cpp new file mode 100644 index 0000000000..7a39afbf7b --- /dev/null +++ b/intl/icu/source/i18n/rematch.cpp @@ -0,0 +1,5733 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +************************************************************************** +* Copyright (C) 2002-2016 International Business Machines Corporation +* and others. All rights reserved. +************************************************************************** +*/ +// +// file: rematch.cpp +// +// Contains the implementation of class RegexMatcher, +// which is one of the main API classes for the ICU regular expression package. +// + +#include "unicode/utypes.h" +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + +#include "unicode/regex.h" +#include "unicode/uniset.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/rbbi.h" +#include "unicode/utf.h" +#include "unicode/utf16.h" +#include "uassert.h" +#include "cmemory.h" +#include "cstr.h" +#include "uvector.h" +#include "uvectr32.h" +#include "uvectr64.h" +#include "regeximp.h" +#include "regexst.h" +#include "regextxt.h" +#include "ucase.h" + +// #include // Needed for heapcheck testing + + +U_NAMESPACE_BEGIN + +// Default limit for the size of the back track stack, to avoid system +// failures causedby heap exhaustion. Units are in 32 bit words, not bytes. +// This value puts ICU's limits higher than most other regexp implementations, +// which use recursion rather than the heap, and take more storage per +// backtrack point. +// +static const int32_t DEFAULT_BACKTRACK_STACK_CAPACITY = 8000000; + +// Time limit counter constant. +// Time limits for expression evaluation are in terms of quanta of work by +// the engine, each of which is 10,000 state saves. +// This constant determines that state saves per tick number. +static const int32_t TIMER_INITIAL_VALUE = 10000; + + +// Test for any of the Unicode line terminating characters. +static inline UBool isLineTerminator(UChar32 c) { + if (c & ~(0x0a | 0x0b | 0x0c | 0x0d | 0x85 | 0x2028 | 0x2029)) { + return false; + } + return (c<=0x0d && c>=0x0a) || c==0x85 || c==0x2028 || c==0x2029; +} + +//----------------------------------------------------------------------------- +// +// Constructor and Destructor +// +//----------------------------------------------------------------------------- +RegexMatcher::RegexMatcher(const RegexPattern *pat) { + fDeferredStatus = U_ZERO_ERROR; + init(fDeferredStatus); + if (U_FAILURE(fDeferredStatus)) { + return; + } + if (pat==nullptr) { + fDeferredStatus = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + fPattern = pat; + init2(RegexStaticSets::gStaticSets->fEmptyText, fDeferredStatus); +} + + + +RegexMatcher::RegexMatcher(const UnicodeString ®exp, const UnicodeString &input, + uint32_t flags, UErrorCode &status) { + init(status); + if (U_FAILURE(status)) { + return; + } + UParseError pe; + fPatternOwned = RegexPattern::compile(regexp, flags, pe, status); + fPattern = fPatternOwned; + + UText inputText = UTEXT_INITIALIZER; + utext_openConstUnicodeString(&inputText, &input, &status); + init2(&inputText, status); + utext_close(&inputText); + + fInputUniStrMaybeMutable = true; +} + + +RegexMatcher::RegexMatcher(UText *regexp, UText *input, + uint32_t flags, UErrorCode &status) { + init(status); + if (U_FAILURE(status)) { + return; + } + UParseError pe; + fPatternOwned = RegexPattern::compile(regexp, flags, pe, status); + if (U_FAILURE(status)) { + return; + } + + fPattern = fPatternOwned; + init2(input, status); +} + + +RegexMatcher::RegexMatcher(const UnicodeString ®exp, + uint32_t flags, UErrorCode &status) { + init(status); + if (U_FAILURE(status)) { + return; + } + UParseError pe; + fPatternOwned = RegexPattern::compile(regexp, flags, pe, status); + if (U_FAILURE(status)) { + return; + } + fPattern = fPatternOwned; + init2(RegexStaticSets::gStaticSets->fEmptyText, status); +} + +RegexMatcher::RegexMatcher(UText *regexp, + uint32_t flags, UErrorCode &status) { + init(status); + if (U_FAILURE(status)) { + return; + } + UParseError pe; + fPatternOwned = RegexPattern::compile(regexp, flags, pe, status); + if (U_FAILURE(status)) { + return; + } + + fPattern = fPatternOwned; + init2(RegexStaticSets::gStaticSets->fEmptyText, status); +} + + + + +RegexMatcher::~RegexMatcher() { + delete fStack; + if (fData != fSmallData) { + uprv_free(fData); + fData = nullptr; + } + if (fPatternOwned) { + delete fPatternOwned; + fPatternOwned = nullptr; + fPattern = nullptr; + } + + if (fInput) { + delete fInput; + } + if (fInputText) { + utext_close(fInputText); + } + if (fAltInputText) { + utext_close(fAltInputText); + } + + #if UCONFIG_NO_BREAK_ITERATION==0 + delete fWordBreakItr; + delete fGCBreakItr; + #endif +} + +// +// init() common initialization for use by all constructors. +// Initialize all fields, get the object into a consistent state. +// This must be done even when the initial status shows an error, +// so that the object is initialized sufficiently well for the destructor +// to run safely. +// +void RegexMatcher::init(UErrorCode &status) { + fPattern = nullptr; + fPatternOwned = nullptr; + fFrameSize = 0; + fRegionStart = 0; + fRegionLimit = 0; + fAnchorStart = 0; + fAnchorLimit = 0; + fLookStart = 0; + fLookLimit = 0; + fActiveStart = 0; + fActiveLimit = 0; + fTransparentBounds = false; + fAnchoringBounds = true; + fMatch = false; + fMatchStart = 0; + fMatchEnd = 0; + fLastMatchEnd = -1; + fAppendPosition = 0; + fHitEnd = false; + fRequireEnd = false; + fStack = nullptr; + fFrame = nullptr; + fTimeLimit = 0; + fTime = 0; + fTickCounter = 0; + fStackLimit = DEFAULT_BACKTRACK_STACK_CAPACITY; + fCallbackFn = nullptr; + fCallbackContext = nullptr; + fFindProgressCallbackFn = nullptr; + fFindProgressCallbackContext = nullptr; + fTraceDebug = false; + fDeferredStatus = status; + fData = fSmallData; + fWordBreakItr = nullptr; + fGCBreakItr = nullptr; + + fStack = nullptr; + fInputText = nullptr; + fAltInputText = nullptr; + fInput = nullptr; + fInputLength = 0; + fInputUniStrMaybeMutable = false; +} + +// +// init2() Common initialization for use by RegexMatcher constructors, part 2. +// This handles the common setup to be done after the Pattern is available. +// +void RegexMatcher::init2(UText *input, UErrorCode &status) { + if (U_FAILURE(status)) { + fDeferredStatus = status; + return; + } + + if (fPattern->fDataSize > UPRV_LENGTHOF(fSmallData)) { + fData = (int64_t *)uprv_malloc(fPattern->fDataSize * sizeof(int64_t)); + if (fData == nullptr) { + status = fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + } + + fStack = new UVector64(status); + if (fStack == nullptr) { + status = fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + + reset(input); + setStackLimit(DEFAULT_BACKTRACK_STACK_CAPACITY, status); + if (U_FAILURE(status)) { + fDeferredStatus = status; + return; + } +} + + +static const char16_t BACKSLASH = 0x5c; +static const char16_t DOLLARSIGN = 0x24; +static const char16_t LEFTBRACKET = 0x7b; +static const char16_t RIGHTBRACKET = 0x7d; + +//-------------------------------------------------------------------------------- +// +// appendReplacement +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::appendReplacement(UnicodeString &dest, + const UnicodeString &replacement, + UErrorCode &status) { + UText replacementText = UTEXT_INITIALIZER; + + utext_openConstUnicodeString(&replacementText, &replacement, &status); + if (U_SUCCESS(status)) { + UText resultText = UTEXT_INITIALIZER; + utext_openUnicodeString(&resultText, &dest, &status); + + if (U_SUCCESS(status)) { + appendReplacement(&resultText, &replacementText, status); + utext_close(&resultText); + } + utext_close(&replacementText); + } + + return *this; +} + +// +// appendReplacement, UText mode +// +RegexMatcher &RegexMatcher::appendReplacement(UText *dest, + UText *replacement, + UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return *this; + } + if (fMatch == false) { + status = U_REGEX_INVALID_STATE; + return *this; + } + + // Copy input string from the end of previous match to start of current match + int64_t destLen = utext_nativeLength(dest); + if (fMatchStart > fAppendPosition) { + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + destLen += utext_replace(dest, destLen, destLen, fInputText->chunkContents+fAppendPosition, + (int32_t)(fMatchStart-fAppendPosition), &status); + } else { + int32_t len16; + if (UTEXT_USES_U16(fInputText)) { + len16 = (int32_t)(fMatchStart-fAppendPosition); + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + len16 = utext_extract(fInputText, fAppendPosition, fMatchStart, nullptr, 0, &lengthStatus); + } + char16_t *inputChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(len16+1)); + if (inputChars == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + utext_extract(fInputText, fAppendPosition, fMatchStart, inputChars, len16+1, &status); + destLen += utext_replace(dest, destLen, destLen, inputChars, len16, &status); + uprv_free(inputChars); + } + } + fAppendPosition = fMatchEnd; + + + // scan the replacement text, looking for substitutions ($n) and \escapes. + // TODO: optimize this loop by efficiently scanning for '$' or '\', + // move entire ranges not containing substitutions. + UTEXT_SETNATIVEINDEX(replacement, 0); + for (UChar32 c = UTEXT_NEXT32(replacement); U_SUCCESS(status) && c != U_SENTINEL; c = UTEXT_NEXT32(replacement)) { + if (c == BACKSLASH) { + // Backslash Escape. Copy the following char out without further checks. + // Note: Surrogate pairs don't need any special handling + // The second half wont be a '$' or a '\', and + // will move to the dest normally on the next + // loop iteration. + c = UTEXT_CURRENT32(replacement); + if (c == U_SENTINEL) { + break; + } + + if (c==0x55/*U*/ || c==0x75/*u*/) { + // We have a \udddd or \Udddddddd escape sequence. + int32_t offset = 0; + struct URegexUTextUnescapeCharContext context = U_REGEX_UTEXT_UNESCAPE_CONTEXT(replacement); + UChar32 escapedChar = u_unescapeAt(uregex_utext_unescape_charAt, &offset, INT32_MAX, &context); + if (escapedChar != (UChar32)0xFFFFFFFF) { + if (U_IS_BMP(escapedChar)) { + char16_t c16 = (char16_t)escapedChar; + destLen += utext_replace(dest, destLen, destLen, &c16, 1, &status); + } else { + char16_t surrogate[2]; + surrogate[0] = U16_LEAD(escapedChar); + surrogate[1] = U16_TRAIL(escapedChar); + if (U_SUCCESS(status)) { + destLen += utext_replace(dest, destLen, destLen, surrogate, 2, &status); + } + } + // TODO: Report errors for mal-formed \u escapes? + // As this is, the original sequence is output, which may be OK. + if (context.lastOffset == offset) { + (void)UTEXT_PREVIOUS32(replacement); + } else if (context.lastOffset != offset-1) { + utext_moveIndex32(replacement, offset - context.lastOffset - 1); + } + } + } else { + (void)UTEXT_NEXT32(replacement); + // Plain backslash escape. Just put out the escaped character. + if (U_IS_BMP(c)) { + char16_t c16 = (char16_t)c; + destLen += utext_replace(dest, destLen, destLen, &c16, 1, &status); + } else { + char16_t surrogate[2]; + surrogate[0] = U16_LEAD(c); + surrogate[1] = U16_TRAIL(c); + if (U_SUCCESS(status)) { + destLen += utext_replace(dest, destLen, destLen, surrogate, 2, &status); + } + } + } + } else if (c != DOLLARSIGN) { + // Normal char, not a $. Copy it out without further checks. + if (U_IS_BMP(c)) { + char16_t c16 = (char16_t)c; + destLen += utext_replace(dest, destLen, destLen, &c16, 1, &status); + } else { + char16_t surrogate[2]; + surrogate[0] = U16_LEAD(c); + surrogate[1] = U16_TRAIL(c); + if (U_SUCCESS(status)) { + destLen += utext_replace(dest, destLen, destLen, surrogate, 2, &status); + } + } + } else { + // We've got a $. Pick up a capture group name or number if one follows. + // Consume digits so long as the resulting group number <= the number of + // number of capture groups in the pattern. + + int32_t groupNum = 0; + int32_t numDigits = 0; + UChar32 nextChar = utext_current32(replacement); + if (nextChar == LEFTBRACKET) { + // Scan for a Named Capture Group, ${name}. + UnicodeString groupName; + utext_next32(replacement); + while(U_SUCCESS(status) && nextChar != RIGHTBRACKET) { + nextChar = utext_next32(replacement); + if (nextChar == U_SENTINEL) { + status = U_REGEX_INVALID_CAPTURE_GROUP_NAME; + } else if ((nextChar >= 0x41 && nextChar <= 0x5a) || // A..Z + (nextChar >= 0x61 && nextChar <= 0x7a) || // a..z + (nextChar >= 0x31 && nextChar <= 0x39)) { // 0..9 + groupName.append(nextChar); + } else if (nextChar == RIGHTBRACKET) { + groupNum = fPattern->fNamedCaptureMap ? uhash_geti(fPattern->fNamedCaptureMap, &groupName) : 0; + if (groupNum == 0) { + status = U_REGEX_INVALID_CAPTURE_GROUP_NAME; + } + } else { + // Character was something other than a name char or a closing '}' + status = U_REGEX_INVALID_CAPTURE_GROUP_NAME; + } + } + + } else if (u_isdigit(nextChar)) { + // $n Scan for a capture group number + int32_t numCaptureGroups = fPattern->fGroupMap->size(); + for (;;) { + nextChar = UTEXT_CURRENT32(replacement); + if (nextChar == U_SENTINEL) { + break; + } + if (u_isdigit(nextChar) == false) { + break; + } + int32_t nextDigitVal = u_charDigitValue(nextChar); + if (groupNum*10 + nextDigitVal > numCaptureGroups) { + // Don't consume the next digit if it makes the capture group number too big. + if (numDigits == 0) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + } + break; + } + (void)UTEXT_NEXT32(replacement); + groupNum=groupNum*10 + nextDigitVal; + ++numDigits; + } + } else { + // $ not followed by capture group name or number. + status = U_REGEX_INVALID_CAPTURE_GROUP_NAME; + } + + if (U_SUCCESS(status)) { + destLen += appendGroup(groupNum, dest, status); + } + } // End of $ capture group handling + } // End of per-character loop through the replacement string. + + return *this; +} + + + +//-------------------------------------------------------------------------------- +// +// appendTail Intended to be used in conjunction with appendReplacement() +// To the destination string, append everything following +// the last match position from the input string. +// +// Note: Match ranges do not affect appendTail or appendReplacement +// +//-------------------------------------------------------------------------------- +UnicodeString &RegexMatcher::appendTail(UnicodeString &dest) { + UErrorCode status = U_ZERO_ERROR; + UText resultText = UTEXT_INITIALIZER; + utext_openUnicodeString(&resultText, &dest, &status); + + if (U_SUCCESS(status)) { + appendTail(&resultText, status); + utext_close(&resultText); + } + + return dest; +} + +// +// appendTail, UText mode +// +UText *RegexMatcher::appendTail(UText *dest, UErrorCode &status) { + if (U_FAILURE(status)) { + return dest; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return dest; + } + + if (fInputLength > fAppendPosition) { + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + int64_t destLen = utext_nativeLength(dest); + utext_replace(dest, destLen, destLen, fInputText->chunkContents+fAppendPosition, + (int32_t)(fInputLength-fAppendPosition), &status); + } else { + int32_t len16; + if (UTEXT_USES_U16(fInputText)) { + len16 = (int32_t)(fInputLength-fAppendPosition); + } else { + len16 = utext_extract(fInputText, fAppendPosition, fInputLength, nullptr, 0, &status); + status = U_ZERO_ERROR; // buffer overflow + } + + char16_t *inputChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(len16)); + if (inputChars == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + } else { + utext_extract(fInputText, fAppendPosition, fInputLength, inputChars, len16, &status); // unterminated + int64_t destLen = utext_nativeLength(dest); + utext_replace(dest, destLen, destLen, inputChars, len16, &status); + uprv_free(inputChars); + } + } + } + return dest; +} + + + +//-------------------------------------------------------------------------------- +// +// end +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::end(UErrorCode &err) const { + return end(0, err); +} + +int64_t RegexMatcher::end64(UErrorCode &err) const { + return end64(0, err); +} + +int64_t RegexMatcher::end64(int32_t group, UErrorCode &err) const { + if (U_FAILURE(err)) { + return -1; + } + if (fMatch == false) { + err = U_REGEX_INVALID_STATE; + return -1; + } + if (group < 0 || group > fPattern->fGroupMap->size()) { + err = U_INDEX_OUTOFBOUNDS_ERROR; + return -1; + } + int64_t e = -1; + if (group == 0) { + e = fMatchEnd; + } else { + // Get the position within the stack frame of the variables for + // this capture group. + int32_t groupOffset = fPattern->fGroupMap->elementAti(group-1); + U_ASSERT(groupOffset < fPattern->fFrameSize); + U_ASSERT(groupOffset >= 0); + e = fFrame->fExtra[groupOffset + 1]; + } + + return e; +} + +int32_t RegexMatcher::end(int32_t group, UErrorCode &err) const { + return (int32_t)end64(group, err); +} + +//-------------------------------------------------------------------------------- +// +// findProgressInterrupt This function is called once for each advance in the target +// string from the find() function, and calls the user progress callback +// function if there is one installed. +// +// Return: true if the find operation is to be terminated. +// false if the find operation is to continue running. +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::findProgressInterrupt(int64_t pos, UErrorCode &status) { + if (fFindProgressCallbackFn && !(*fFindProgressCallbackFn)(fFindProgressCallbackContext, pos)) { + status = U_REGEX_STOPPED_BY_CALLER; + return true; + } + return false; +} + +//-------------------------------------------------------------------------------- +// +// find() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::find() { + if (U_FAILURE(fDeferredStatus)) { + return false; + } + UErrorCode status = U_ZERO_ERROR; + UBool result = find(status); + return result; +} + +//-------------------------------------------------------------------------------- +// +// find() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::find(UErrorCode &status) { + // Start at the position of the last match end. (Will be zero if the + // matcher has been reset.) + // + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + return findUsingChunk(status); + } + + int64_t startPos = fMatchEnd; + if (startPos==0) { + startPos = fActiveStart; + } + + if (fMatch) { + // Save the position of any previous successful match. + fLastMatchEnd = fMatchEnd; + + if (fMatchStart == fMatchEnd) { + // Previous match had zero length. Move start position up one position + // to avoid sending find() into a loop on zero-length matches. + if (startPos >= fActiveLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + (void)UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + } + } else { + if (fLastMatchEnd >= 0) { + // A previous find() failed to match. Don't try again. + // (without this test, a pattern with a zero-length match + // could match again at the end of an input string.) + fHitEnd = true; + return false; + } + } + + + // Compute the position in the input string beyond which a match can not begin, because + // the minimum length match would extend past the end of the input. + // Note: some patterns that cannot match anything will have fMinMatchLength==Max Int. + // Be aware of possible overflows if making changes here. + int64_t testStartLimit; + if (UTEXT_USES_U16(fInputText)) { + testStartLimit = fActiveLimit - fPattern->fMinMatchLen; + if (startPos > testStartLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + } else { + // We don't know exactly how long the minimum match length is in native characters. + // Treat anything > 0 as 1. + testStartLimit = fActiveLimit - (fPattern->fMinMatchLen > 0 ? 1 : 0); + } + + UChar32 c; + U_ASSERT(startPos >= 0); + + switch (fPattern->fStartType) { + case START_NO_INFO: + // No optimization was found. + // Try a match at each input position. + for (;;) { + MatchAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + if (startPos >= testStartLimit) { + fHitEnd = true; + return false; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + (void)UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testStartLimit the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + UPRV_UNREACHABLE_EXIT; + + case START_START: + // Matches are only possible at the start of the input string + // (pattern begins with ^ or \A) + if (startPos > fActiveStart) { + fMatch = false; + return false; + } + MatchAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + return fMatch; + + + case START_SET: + { + // Match may start on any char from a pre-computed set. + U_ASSERT(fPattern->fMinMatchLen > 0); + UTEXT_SETNATIVEINDEX(fInputText, startPos); + for (;;) { + int64_t pos = startPos; + c = UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + // c will be -1 (U_SENTINEL) at end of text, in which case we + // skip this next block (so we don't have a negative array index) + // and handle end of text in the following block. + if (c >= 0 && ((c<256 && fPattern->fInitialChars8->contains(c)) || + (c>=256 && fPattern->fInitialChars->contains(c)))) { + MatchAt(pos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + UTEXT_SETNATIVEINDEX(fInputText, pos); + } + if (startPos > testStartLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + if (findProgressInterrupt(startPos, status)) + return false; + } + } + UPRV_UNREACHABLE_EXIT; + + case START_STRING: + case START_CHAR: + { + // Match starts on exactly one char. + U_ASSERT(fPattern->fMinMatchLen > 0); + UChar32 theChar = fPattern->fInitialChar; + UTEXT_SETNATIVEINDEX(fInputText, startPos); + for (;;) { + int64_t pos = startPos; + c = UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + if (c == theChar) { + MatchAt(pos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + } + if (startPos > testStartLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + if (findProgressInterrupt(startPos, status)) + return false; + } + } + UPRV_UNREACHABLE_EXIT; + + case START_LINE: + { + UChar32 ch; + if (startPos == fAnchorStart) { + MatchAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + ch = UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + } else { + UTEXT_SETNATIVEINDEX(fInputText, startPos); + ch = UTEXT_PREVIOUS32(fInputText); + UTEXT_SETNATIVEINDEX(fInputText, startPos); + } + + if (fPattern->fFlags & UREGEX_UNIX_LINES) { + for (;;) { + if (ch == 0x0a) { + MatchAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + } + if (startPos >= testStartLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + ch = UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testStartLimit the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + } else { + for (;;) { + if (isLineTerminator(ch)) { + if (ch == 0x0d && startPos < fActiveLimit && UTEXT_CURRENT32(fInputText) == 0x0a) { + (void)UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + } + MatchAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + UTEXT_SETNATIVEINDEX(fInputText, startPos); + } + if (startPos >= testStartLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + ch = UTEXT_NEXT32(fInputText); + startPos = UTEXT_GETNATIVEINDEX(fInputText); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testStartLimit the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + } + } + + default: + UPRV_UNREACHABLE_ASSERT; + // Unknown value in fPattern->fStartType, should be from StartOfMatch enum. But + // we have reports of this in production code, don't use UPRV_UNREACHABLE_EXIT. + // See ICU-21669. + status = U_INTERNAL_PROGRAM_ERROR; + return false; + } + + UPRV_UNREACHABLE_EXIT; +} + + + +UBool RegexMatcher::find(int64_t start, UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + this->reset(); // Note: Reset() is specified by Java Matcher documentation. + // This will reset the region to be the full input length. + if (start < 0) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + + int64_t nativeStart = start; + if (nativeStart < fActiveStart || nativeStart > fActiveLimit) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + fMatchEnd = nativeStart; + return find(status); +} + + +//-------------------------------------------------------------------------------- +// +// findUsingChunk() -- like find(), but with the advance knowledge that the +// entire string is available in the UText's chunk buffer. +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::findUsingChunk(UErrorCode &status) { + // Start at the position of the last match end. (Will be zero if the + // matcher has been reset. + // + + int32_t startPos = (int32_t)fMatchEnd; + if (startPos==0) { + startPos = (int32_t)fActiveStart; + } + + const char16_t *inputBuf = fInputText->chunkContents; + + if (fMatch) { + // Save the position of any previous successful match. + fLastMatchEnd = fMatchEnd; + + if (fMatchStart == fMatchEnd) { + // Previous match had zero length. Move start position up one position + // to avoid sending find() into a loop on zero-length matches. + if (startPos >= fActiveLimit) { + fMatch = false; + fHitEnd = true; + return false; + } + U16_FWD_1(inputBuf, startPos, fInputLength); + } + } else { + if (fLastMatchEnd >= 0) { + // A previous find() failed to match. Don't try again. + // (without this test, a pattern with a zero-length match + // could match again at the end of an input string.) + fHitEnd = true; + return false; + } + } + + + // Compute the position in the input string beyond which a match can not begin, because + // the minimum length match would extend past the end of the input. + // Note: some patterns that cannot match anything will have fMinMatchLength==Max Int. + // Be aware of possible overflows if making changes here. + // Note: a match can begin at inputBuf + testLen; it is an inclusive limit. + int32_t testLen = (int32_t)(fActiveLimit - fPattern->fMinMatchLen); + if (startPos > testLen) { + fMatch = false; + fHitEnd = true; + return false; + } + + UChar32 c; + U_ASSERT(startPos >= 0); + + switch (fPattern->fStartType) { + case START_NO_INFO: + // No optimization was found. + // Try a match at each input position. + for (;;) { + MatchChunkAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + if (startPos >= testLen) { + fHitEnd = true; + return false; + } + U16_FWD_1(inputBuf, startPos, fActiveLimit); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testLen the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + UPRV_UNREACHABLE_EXIT; + + case START_START: + // Matches are only possible at the start of the input string + // (pattern begins with ^ or \A) + if (startPos > fActiveStart) { + fMatch = false; + return false; + } + MatchChunkAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + return fMatch; + + + case START_SET: + { + // Match may start on any char from a pre-computed set. + U_ASSERT(fPattern->fMinMatchLen > 0); + for (;;) { + int32_t pos = startPos; + U16_NEXT(inputBuf, startPos, fActiveLimit, c); // like c = inputBuf[startPos++]; + if ((c<256 && fPattern->fInitialChars8->contains(c)) || + (c>=256 && fPattern->fInitialChars->contains(c))) { + MatchChunkAt(pos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + } + if (startPos > testLen) { + fMatch = false; + fHitEnd = true; + return false; + } + if (findProgressInterrupt(startPos, status)) + return false; + } + } + UPRV_UNREACHABLE_EXIT; + + case START_STRING: + case START_CHAR: + { + // Match starts on exactly one char. + U_ASSERT(fPattern->fMinMatchLen > 0); + UChar32 theChar = fPattern->fInitialChar; + for (;;) { + int32_t pos = startPos; + U16_NEXT(inputBuf, startPos, fActiveLimit, c); // like c = inputBuf[startPos++]; + if (c == theChar) { + MatchChunkAt(pos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + } + if (startPos > testLen) { + fMatch = false; + fHitEnd = true; + return false; + } + if (findProgressInterrupt(startPos, status)) + return false; + } + } + UPRV_UNREACHABLE_EXIT; + + case START_LINE: + { + UChar32 ch; + if (startPos == fAnchorStart) { + MatchChunkAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + U16_FWD_1(inputBuf, startPos, fActiveLimit); + } + + if (fPattern->fFlags & UREGEX_UNIX_LINES) { + for (;;) { + ch = inputBuf[startPos-1]; + if (ch == 0x0a) { + MatchChunkAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + } + if (startPos >= testLen) { + fMatch = false; + fHitEnd = true; + return false; + } + U16_FWD_1(inputBuf, startPos, fActiveLimit); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testLen the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + } else { + for (;;) { + ch = inputBuf[startPos-1]; + if (isLineTerminator(ch)) { + if (ch == 0x0d && startPos < fActiveLimit && inputBuf[startPos] == 0x0a) { + startPos++; + } + MatchChunkAt(startPos, false, status); + if (U_FAILURE(status)) { + return false; + } + if (fMatch) { + return true; + } + } + if (startPos >= testLen) { + fMatch = false; + fHitEnd = true; + return false; + } + U16_FWD_1(inputBuf, startPos, fActiveLimit); + // Note that it's perfectly OK for a pattern to have a zero-length + // match at the end of a string, so we must make sure that the loop + // runs with startPos == testLen the last time through. + if (findProgressInterrupt(startPos, status)) + return false; + } + } + } + + default: + UPRV_UNREACHABLE_ASSERT; + // Unknown value in fPattern->fStartType, should be from StartOfMatch enum. But + // we have reports of this in production code, don't use UPRV_UNREACHABLE_EXIT. + // See ICU-21669. + status = U_INTERNAL_PROGRAM_ERROR; + return false; + } + + UPRV_UNREACHABLE_EXIT; +} + + + +//-------------------------------------------------------------------------------- +// +// group() +// +//-------------------------------------------------------------------------------- +UnicodeString RegexMatcher::group(UErrorCode &status) const { + return group(0, status); +} + +// Return immutable shallow clone +UText *RegexMatcher::group(UText *dest, int64_t &group_len, UErrorCode &status) const { + return group(0, dest, group_len, status); +} + +// Return immutable shallow clone +UText *RegexMatcher::group(int32_t groupNum, UText *dest, int64_t &group_len, UErrorCode &status) const { + group_len = 0; + if (U_FAILURE(status)) { + return dest; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + } else if (fMatch == false) { + status = U_REGEX_INVALID_STATE; + } else if (groupNum < 0 || groupNum > fPattern->fGroupMap->size()) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + } + + if (U_FAILURE(status)) { + return dest; + } + + int64_t s, e; + if (groupNum == 0) { + s = fMatchStart; + e = fMatchEnd; + } else { + int32_t groupOffset = fPattern->fGroupMap->elementAti(groupNum-1); + U_ASSERT(groupOffset < fPattern->fFrameSize); + U_ASSERT(groupOffset >= 0); + s = fFrame->fExtra[groupOffset]; + e = fFrame->fExtra[groupOffset+1]; + } + + if (s < 0) { + // A capture group wasn't part of the match + return utext_clone(dest, fInputText, false, true, &status); + } + U_ASSERT(s <= e); + group_len = e - s; + + dest = utext_clone(dest, fInputText, false, true, &status); + if (dest) + UTEXT_SETNATIVEINDEX(dest, s); + return dest; +} + +UnicodeString RegexMatcher::group(int32_t groupNum, UErrorCode &status) const { + UnicodeString result; + int64_t groupStart = start64(groupNum, status); + int64_t groupEnd = end64(groupNum, status); + if (U_FAILURE(status) || groupStart == -1 || groupStart == groupEnd) { + return result; + } + + // Get the group length using a utext_extract preflight. + // UText is actually pretty efficient at this when underlying encoding is UTF-16. + int32_t length = utext_extract(fInputText, groupStart, groupEnd, nullptr, 0, &status); + if (status != U_BUFFER_OVERFLOW_ERROR) { + return result; + } + + status = U_ZERO_ERROR; + char16_t *buf = result.getBuffer(length); + if (buf == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + } else { + int32_t extractLength = utext_extract(fInputText, groupStart, groupEnd, buf, length, &status); + result.releaseBuffer(extractLength); + U_ASSERT(length == extractLength); + } + return result; +} + + +//-------------------------------------------------------------------------------- +// +// appendGroup() -- currently internal only, appends a group to a UText rather +// than replacing its contents +// +//-------------------------------------------------------------------------------- + +int64_t RegexMatcher::appendGroup(int32_t groupNum, UText *dest, UErrorCode &status) const { + if (U_FAILURE(status)) { + return 0; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return 0; + } + int64_t destLen = utext_nativeLength(dest); + + if (fMatch == false) { + status = U_REGEX_INVALID_STATE; + return utext_replace(dest, destLen, destLen, nullptr, 0, &status); + } + if (groupNum < 0 || groupNum > fPattern->fGroupMap->size()) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return utext_replace(dest, destLen, destLen, nullptr, 0, &status); + } + + int64_t s, e; + if (groupNum == 0) { + s = fMatchStart; + e = fMatchEnd; + } else { + int32_t groupOffset = fPattern->fGroupMap->elementAti(groupNum-1); + U_ASSERT(groupOffset < fPattern->fFrameSize); + U_ASSERT(groupOffset >= 0); + s = fFrame->fExtra[groupOffset]; + e = fFrame->fExtra[groupOffset+1]; + } + + if (s < 0) { + // A capture group wasn't part of the match + return utext_replace(dest, destLen, destLen, nullptr, 0, &status); + } + U_ASSERT(s <= e); + + int64_t deltaLen; + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + U_ASSERT(e <= fInputLength); + deltaLen = utext_replace(dest, destLen, destLen, fInputText->chunkContents+s, (int32_t)(e-s), &status); + } else { + int32_t len16; + if (UTEXT_USES_U16(fInputText)) { + len16 = (int32_t)(e-s); + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + len16 = utext_extract(fInputText, s, e, nullptr, 0, &lengthStatus); + } + char16_t *groupChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(len16+1)); + if (groupChars == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + utext_extract(fInputText, s, e, groupChars, len16+1, &status); + + deltaLen = utext_replace(dest, destLen, destLen, groupChars, len16, &status); + uprv_free(groupChars); + } + return deltaLen; +} + + + +//-------------------------------------------------------------------------------- +// +// groupCount() +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::groupCount() const { + return fPattern->fGroupMap->size(); +} + +//-------------------------------------------------------------------------------- +// +// hasAnchoringBounds() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::hasAnchoringBounds() const { + return fAnchoringBounds; +} + + +//-------------------------------------------------------------------------------- +// +// hasTransparentBounds() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::hasTransparentBounds() const { + return fTransparentBounds; +} + + + +//-------------------------------------------------------------------------------- +// +// hitEnd() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::hitEnd() const { + return fHitEnd; +} + + +//-------------------------------------------------------------------------------- +// +// input() +// +//-------------------------------------------------------------------------------- +const UnicodeString &RegexMatcher::input() const { + if (!fInput) { + UErrorCode status = U_ZERO_ERROR; + int32_t len16; + if (UTEXT_USES_U16(fInputText)) { + len16 = (int32_t)fInputLength; + } else { + len16 = utext_extract(fInputText, 0, fInputLength, nullptr, 0, &status); + status = U_ZERO_ERROR; // overflow, length status + } + UnicodeString *result = new UnicodeString(len16, 0, 0); + + char16_t *inputChars = result->getBuffer(len16); + utext_extract(fInputText, 0, fInputLength, inputChars, len16, &status); // unterminated warning + result->releaseBuffer(len16); + + (*(const UnicodeString **)&fInput) = result; // pointer assignment, rather than operator= + } + + return *fInput; +} + +//-------------------------------------------------------------------------------- +// +// inputText() +// +//-------------------------------------------------------------------------------- +UText *RegexMatcher::inputText() const { + return fInputText; +} + + +//-------------------------------------------------------------------------------- +// +// getInput() -- like inputText(), but makes a clone or copies into another UText +// +//-------------------------------------------------------------------------------- +UText *RegexMatcher::getInput (UText *dest, UErrorCode &status) const { + if (U_FAILURE(status)) { + return dest; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return dest; + } + + if (dest) { + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + utext_replace(dest, 0, utext_nativeLength(dest), fInputText->chunkContents, (int32_t)fInputLength, &status); + } else { + int32_t input16Len; + if (UTEXT_USES_U16(fInputText)) { + input16Len = (int32_t)fInputLength; + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + input16Len = utext_extract(fInputText, 0, fInputLength, nullptr, 0, &lengthStatus); // buffer overflow error + } + char16_t *inputChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(input16Len)); + if (inputChars == nullptr) { + return dest; + } + + status = U_ZERO_ERROR; + utext_extract(fInputText, 0, fInputLength, inputChars, input16Len, &status); // not terminated warning + status = U_ZERO_ERROR; + utext_replace(dest, 0, utext_nativeLength(dest), inputChars, input16Len, &status); + + uprv_free(inputChars); + } + return dest; + } else { + return utext_clone(nullptr, fInputText, false, true, &status); + } +} + + +static UBool compat_SyncMutableUTextContents(UText *ut); +static UBool compat_SyncMutableUTextContents(UText *ut) { + UBool retVal = false; + + // In the following test, we're really only interested in whether the UText should switch + // between heap and stack allocation. If length hasn't changed, we won't, so the chunkContents + // will still point to the correct data. + if (utext_nativeLength(ut) != ut->nativeIndexingLimit) { + UnicodeString *us=(UnicodeString *)ut->context; + + // Update to the latest length. + // For example, (utext_nativeLength(ut) != ut->nativeIndexingLimit). + int32_t newLength = us->length(); + + // Update the chunk description. + // The buffer may have switched between stack- and heap-based. + ut->chunkContents = us->getBuffer(); + ut->chunkLength = newLength; + ut->chunkNativeLimit = newLength; + ut->nativeIndexingLimit = newLength; + retVal = true; + } + + return retVal; +} + +//-------------------------------------------------------------------------------- +// +// lookingAt() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::lookingAt(UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + + if (fInputUniStrMaybeMutable) { + if (compat_SyncMutableUTextContents(fInputText)) { + fInputLength = utext_nativeLength(fInputText); + reset(); + } + } + else { + resetPreserveRegion(); + } + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + MatchChunkAt((int32_t)fActiveStart, false, status); + } else { + MatchAt(fActiveStart, false, status); + } + return fMatch; +} + + +UBool RegexMatcher::lookingAt(int64_t start, UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + reset(); + + if (start < 0) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + + if (fInputUniStrMaybeMutable) { + if (compat_SyncMutableUTextContents(fInputText)) { + fInputLength = utext_nativeLength(fInputText); + reset(); + } + } + + int64_t nativeStart; + nativeStart = start; + if (nativeStart < fActiveStart || nativeStart > fActiveLimit) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + MatchChunkAt((int32_t)nativeStart, false, status); + } else { + MatchAt(nativeStart, false, status); + } + return fMatch; +} + + + +//-------------------------------------------------------------------------------- +// +// matches() +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::matches(UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + + if (fInputUniStrMaybeMutable) { + if (compat_SyncMutableUTextContents(fInputText)) { + fInputLength = utext_nativeLength(fInputText); + reset(); + } + } + else { + resetPreserveRegion(); + } + + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + MatchChunkAt((int32_t)fActiveStart, true, status); + } else { + MatchAt(fActiveStart, true, status); + } + return fMatch; +} + + +UBool RegexMatcher::matches(int64_t start, UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return false; + } + reset(); + + if (start < 0) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + + if (fInputUniStrMaybeMutable) { + if (compat_SyncMutableUTextContents(fInputText)) { + fInputLength = utext_nativeLength(fInputText); + reset(); + } + } + + int64_t nativeStart; + nativeStart = start; + if (nativeStart < fActiveStart || nativeStart > fActiveLimit) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return false; + } + + if (UTEXT_FULL_TEXT_IN_CHUNK(fInputText, fInputLength)) { + MatchChunkAt((int32_t)nativeStart, true, status); + } else { + MatchAt(nativeStart, true, status); + } + return fMatch; +} + + + +//-------------------------------------------------------------------------------- +// +// pattern +// +//-------------------------------------------------------------------------------- +const RegexPattern &RegexMatcher::pattern() const { + return *fPattern; +} + + + +//-------------------------------------------------------------------------------- +// +// region +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::region(int64_t regionStart, int64_t regionLimit, int64_t startIndex, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + + if (regionStart>regionLimit || regionStart<0 || regionLimit<0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } + + int64_t nativeStart = regionStart; + int64_t nativeLimit = regionLimit; + if (nativeStart > fInputLength || nativeLimit > fInputLength) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } + + if (startIndex == -1) + this->reset(); + else + resetPreserveRegion(); + + fRegionStart = nativeStart; + fRegionLimit = nativeLimit; + fActiveStart = nativeStart; + fActiveLimit = nativeLimit; + + if (startIndex != -1) { + if (startIndex < fActiveStart || startIndex > fActiveLimit) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + } + fMatchEnd = startIndex; + } + + if (!fTransparentBounds) { + fLookStart = nativeStart; + fLookLimit = nativeLimit; + } + if (fAnchoringBounds) { + fAnchorStart = nativeStart; + fAnchorLimit = nativeLimit; + } + return *this; +} + +RegexMatcher &RegexMatcher::region(int64_t start, int64_t limit, UErrorCode &status) { + return region(start, limit, -1, status); +} + +//-------------------------------------------------------------------------------- +// +// regionEnd +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::regionEnd() const { + return (int32_t)fRegionLimit; +} + +int64_t RegexMatcher::regionEnd64() const { + return fRegionLimit; +} + +//-------------------------------------------------------------------------------- +// +// regionStart +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::regionStart() const { + return (int32_t)fRegionStart; +} + +int64_t RegexMatcher::regionStart64() const { + return fRegionStart; +} + + +//-------------------------------------------------------------------------------- +// +// replaceAll +// +//-------------------------------------------------------------------------------- +UnicodeString RegexMatcher::replaceAll(const UnicodeString &replacement, UErrorCode &status) { + UText replacementText = UTEXT_INITIALIZER; + UText resultText = UTEXT_INITIALIZER; + UnicodeString resultString; + if (U_FAILURE(status)) { + return resultString; + } + + utext_openConstUnicodeString(&replacementText, &replacement, &status); + utext_openUnicodeString(&resultText, &resultString, &status); + + replaceAll(&replacementText, &resultText, status); + + utext_close(&resultText); + utext_close(&replacementText); + + return resultString; +} + + +// +// replaceAll, UText mode +// +UText *RegexMatcher::replaceAll(UText *replacement, UText *dest, UErrorCode &status) { + if (U_FAILURE(status)) { + return dest; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return dest; + } + + if (dest == nullptr) { + UnicodeString emptyString; + UText empty = UTEXT_INITIALIZER; + + utext_openUnicodeString(&empty, &emptyString, &status); + dest = utext_clone(nullptr, &empty, true, false, &status); + utext_close(&empty); + } + + if (U_SUCCESS(status)) { + reset(); + while (find()) { + appendReplacement(dest, replacement, status); + if (U_FAILURE(status)) { + break; + } + } + appendTail(dest, status); + } + + return dest; +} + + +//-------------------------------------------------------------------------------- +// +// replaceFirst +// +//-------------------------------------------------------------------------------- +UnicodeString RegexMatcher::replaceFirst(const UnicodeString &replacement, UErrorCode &status) { + UText replacementText = UTEXT_INITIALIZER; + UText resultText = UTEXT_INITIALIZER; + UnicodeString resultString; + + utext_openConstUnicodeString(&replacementText, &replacement, &status); + utext_openUnicodeString(&resultText, &resultString, &status); + + replaceFirst(&replacementText, &resultText, status); + + utext_close(&resultText); + utext_close(&replacementText); + + return resultString; +} + +// +// replaceFirst, UText mode +// +UText *RegexMatcher::replaceFirst(UText *replacement, UText *dest, UErrorCode &status) { + if (U_FAILURE(status)) { + return dest; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return dest; + } + + reset(); + if (!find()) { + return getInput(dest, status); + } + + if (dest == nullptr) { + UnicodeString emptyString; + UText empty = UTEXT_INITIALIZER; + + utext_openUnicodeString(&empty, &emptyString, &status); + dest = utext_clone(nullptr, &empty, true, false, &status); + utext_close(&empty); + } + + appendReplacement(dest, replacement, status); + appendTail(dest, status); + + return dest; +} + + +//-------------------------------------------------------------------------------- +// +// requireEnd +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::requireEnd() const { + return fRequireEnd; +} + + +//-------------------------------------------------------------------------------- +// +// reset +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::reset() { + fRegionStart = 0; + fRegionLimit = fInputLength; + fActiveStart = 0; + fActiveLimit = fInputLength; + fAnchorStart = 0; + fAnchorLimit = fInputLength; + fLookStart = 0; + fLookLimit = fInputLength; + resetPreserveRegion(); + return *this; +} + + + +void RegexMatcher::resetPreserveRegion() { + fMatchStart = 0; + fMatchEnd = 0; + fLastMatchEnd = -1; + fAppendPosition = 0; + fMatch = false; + fHitEnd = false; + fRequireEnd = false; + fTime = 0; + fTickCounter = TIMER_INITIAL_VALUE; + //resetStack(); // more expensive than it looks... +} + + +RegexMatcher &RegexMatcher::reset(const UnicodeString &input) { + fInputText = utext_openConstUnicodeString(fInputText, &input, &fDeferredStatus); + if (fPattern->fNeedsAltInput) { + fAltInputText = utext_clone(fAltInputText, fInputText, false, true, &fDeferredStatus); + } + if (U_FAILURE(fDeferredStatus)) { + return *this; + } + fInputLength = utext_nativeLength(fInputText); + + reset(); + delete fInput; + fInput = nullptr; + + // Do the following for any UnicodeString. + // This is for compatibility for those clients who modify the input string "live" during regex operations. + fInputUniStrMaybeMutable = true; + +#if UCONFIG_NO_BREAK_ITERATION==0 + if (fWordBreakItr) { + fWordBreakItr->setText(fInputText, fDeferredStatus); + } + if (fGCBreakItr) { + fGCBreakItr->setText(fInputText, fDeferredStatus); + } +#endif + + return *this; +} + + +RegexMatcher &RegexMatcher::reset(UText *input) { + if (fInputText != input) { + fInputText = utext_clone(fInputText, input, false, true, &fDeferredStatus); + if (fPattern->fNeedsAltInput) fAltInputText = utext_clone(fAltInputText, fInputText, false, true, &fDeferredStatus); + if (U_FAILURE(fDeferredStatus)) { + return *this; + } + fInputLength = utext_nativeLength(fInputText); + + delete fInput; + fInput = nullptr; + +#if UCONFIG_NO_BREAK_ITERATION==0 + if (fWordBreakItr) { + fWordBreakItr->setText(input, fDeferredStatus); + } + if (fGCBreakItr) { + fGCBreakItr->setText(fInputText, fDeferredStatus); + } +#endif + } + reset(); + fInputUniStrMaybeMutable = false; + + return *this; +} + +/*RegexMatcher &RegexMatcher::reset(const char16_t *) { + fDeferredStatus = U_INTERNAL_PROGRAM_ERROR; + return *this; +}*/ + +RegexMatcher &RegexMatcher::reset(int64_t position, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + reset(); // Reset also resets the region to be the entire string. + + if (position < 0 || position > fActiveLimit) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return *this; + } + fMatchEnd = position; + return *this; +} + + +//-------------------------------------------------------------------------------- +// +// refresh +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::refreshInputText(UText *input, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (input == nullptr) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + if (utext_nativeLength(fInputText) != utext_nativeLength(input)) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + int64_t pos = utext_getNativeIndex(fInputText); + // Shallow read-only clone of the new UText into the existing input UText + fInputText = utext_clone(fInputText, input, false, true, &status); + if (U_FAILURE(status)) { + return *this; + } + utext_setNativeIndex(fInputText, pos); + + if (fAltInputText != nullptr) { + pos = utext_getNativeIndex(fAltInputText); + fAltInputText = utext_clone(fAltInputText, input, false, true, &status); + if (U_FAILURE(status)) { + return *this; + } + utext_setNativeIndex(fAltInputText, pos); + } + return *this; +} + + + +//-------------------------------------------------------------------------------- +// +// setTrace +// +//-------------------------------------------------------------------------------- +void RegexMatcher::setTrace(UBool state) { + fTraceDebug = state; +} + + + +/** + * UText, replace entire contents of the destination UText with a substring of the source UText. + * + * @param src The source UText + * @param dest The destination UText. Must be writable. + * May be nullptr, in which case a new UText will be allocated. + * @param start Start index of source substring. + * @param limit Limit index of source substring. + * @param status An error code. + */ +static UText *utext_extract_replace(UText *src, UText *dest, int64_t start, int64_t limit, UErrorCode *status) { + if (U_FAILURE(*status)) { + return dest; + } + if (start == limit) { + if (dest) { + utext_replace(dest, 0, utext_nativeLength(dest), nullptr, 0, status); + return dest; + } else { + return utext_openUChars(nullptr, nullptr, 0, status); + } + } + int32_t length = utext_extract(src, start, limit, nullptr, 0, status); + if (*status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(*status)) { + return dest; + } + *status = U_ZERO_ERROR; + MaybeStackArray buffer; + if (length >= buffer.getCapacity()) { + char16_t *newBuf = buffer.resize(length+1); // Leave space for terminating Nul. + if (newBuf == nullptr) { + *status = U_MEMORY_ALLOCATION_ERROR; + } + } + utext_extract(src, start, limit, buffer.getAlias(), length+1, status); + if (dest) { + utext_replace(dest, 0, utext_nativeLength(dest), buffer.getAlias(), length, status); + return dest; + } + + // Caller did not provide a preexisting UText. + // Open a new one, and have it adopt the text buffer storage. + if (U_FAILURE(*status)) { + return nullptr; + } + int32_t ownedLength = 0; + char16_t *ownedBuf = buffer.orphanOrClone(length+1, ownedLength); + if (ownedBuf == nullptr) { + *status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + UText *result = utext_openUChars(nullptr, ownedBuf, length, status); + if (U_FAILURE(*status)) { + uprv_free(ownedBuf); + return nullptr; + } + result->providerProperties |= (1 << UTEXT_PROVIDER_OWNS_TEXT); + return result; +} + + +//--------------------------------------------------------------------- +// +// split +// +//--------------------------------------------------------------------- +int32_t RegexMatcher::split(const UnicodeString &input, + UnicodeString dest[], + int32_t destCapacity, + UErrorCode &status) +{ + UText inputText = UTEXT_INITIALIZER; + utext_openConstUnicodeString(&inputText, &input, &status); + if (U_FAILURE(status)) { + return 0; + } + + UText **destText = (UText **)uprv_malloc(sizeof(UText*)*destCapacity); + if (destText == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + int32_t i; + for (i = 0; i < destCapacity; i++) { + destText[i] = utext_openUnicodeString(nullptr, &dest[i], &status); + } + + int32_t fieldCount = split(&inputText, destText, destCapacity, status); + + for (i = 0; i < destCapacity; i++) { + utext_close(destText[i]); + } + + uprv_free(destText); + utext_close(&inputText); + return fieldCount; +} + +// +// split, UText mode +// +int32_t RegexMatcher::split(UText *input, + UText *dest[], + int32_t destCapacity, + UErrorCode &status) +{ + // + // Check arguments for validity + // + if (U_FAILURE(status)) { + return 0; + } + + if (destCapacity < 1) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // + // Reset for the input text + // + reset(input); + int64_t nextOutputStringStart = 0; + if (fActiveLimit == 0) { + return 0; + } + + // + // Loop through the input text, searching for the delimiter pattern + // + int32_t i; + int32_t numCaptureGroups = fPattern->fGroupMap->size(); + for (i=0; ; i++) { + if (i>=destCapacity-1) { + // There is one or zero output string left. + // Fill the last output string with whatever is left from the input, then exit the loop. + // ( i will be == destCapacity if we filled the output array while processing + // capture groups of the delimiter expression, in which case we will discard the + // last capture group saved in favor of the unprocessed remainder of the + // input string.) + i = destCapacity-1; + if (fActiveLimit > nextOutputStringStart) { + if (UTEXT_FULL_TEXT_IN_CHUNK(input, fInputLength)) { + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), + input->chunkContents+nextOutputStringStart, + (int32_t)(fActiveLimit-nextOutputStringStart), &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, input->chunkContents+nextOutputStringStart, + fActiveLimit-nextOutputStringStart, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + int32_t remaining16Length = + utext_extract(input, nextOutputStringStart, fActiveLimit, nullptr, 0, &lengthStatus); + char16_t *remainingChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(remaining16Length+1)); + if (remainingChars == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + + utext_extract(input, nextOutputStringStart, fActiveLimit, remainingChars, remaining16Length+1, &status); + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), remainingChars, remaining16Length, &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, remainingChars, remaining16Length, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + + uprv_free(remainingChars); + } + } + break; + } + if (find()) { + // We found another delimiter. Move everything from where we started looking + // up until the start of the delimiter into the next output string. + if (UTEXT_FULL_TEXT_IN_CHUNK(input, fInputLength)) { + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), + input->chunkContents+nextOutputStringStart, + (int32_t)(fMatchStart-nextOutputStringStart), &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, input->chunkContents+nextOutputStringStart, + fMatchStart-nextOutputStringStart, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + int32_t remaining16Length = utext_extract(input, nextOutputStringStart, fMatchStart, nullptr, 0, &lengthStatus); + char16_t *remainingChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(remaining16Length+1)); + if (remainingChars == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + utext_extract(input, nextOutputStringStart, fMatchStart, remainingChars, remaining16Length+1, &status); + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), remainingChars, remaining16Length, &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, remainingChars, remaining16Length, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + + uprv_free(remainingChars); + } + nextOutputStringStart = fMatchEnd; + + // If the delimiter pattern has capturing parentheses, the captured + // text goes out into the next n destination strings. + int32_t groupNum; + for (groupNum=1; groupNum<=numCaptureGroups; groupNum++) { + if (i >= destCapacity-2) { + // Never fill the last available output string with capture group text. + // It will filled with the last field, the remainder of the + // unsplit input text. + break; + } + i++; + dest[i] = utext_extract_replace(fInputText, dest[i], + start64(groupNum, status), end64(groupNum, status), &status); + } + + if (nextOutputStringStart == fActiveLimit) { + // The delimiter was at the end of the string. We're done, but first + // we output one last empty string, for the empty field following + // the delimiter at the end of input. + if (i+1 < destCapacity) { + ++i; + if (dest[i] == nullptr) { + dest[i] = utext_openUChars(nullptr, nullptr, 0, &status); + } else { + static const char16_t emptyString[] = {(char16_t)0}; + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), emptyString, 0, &status); + } + } + break; + + } + } + else + { + // We ran off the end of the input while looking for the next delimiter. + // All the remaining text goes into the current output string. + if (UTEXT_FULL_TEXT_IN_CHUNK(input, fInputLength)) { + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), + input->chunkContents+nextOutputStringStart, + (int32_t)(fActiveLimit-nextOutputStringStart), &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, input->chunkContents+nextOutputStringStart, + fActiveLimit-nextOutputStringStart, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + } else { + UErrorCode lengthStatus = U_ZERO_ERROR; + int32_t remaining16Length = utext_extract(input, nextOutputStringStart, fActiveLimit, nullptr, 0, &lengthStatus); + char16_t *remainingChars = (char16_t *)uprv_malloc(sizeof(char16_t)*(remaining16Length+1)); + if (remainingChars == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + break; + } + + utext_extract(input, nextOutputStringStart, fActiveLimit, remainingChars, remaining16Length+1, &status); + if (dest[i]) { + utext_replace(dest[i], 0, utext_nativeLength(dest[i]), remainingChars, remaining16Length, &status); + } else { + UText remainingText = UTEXT_INITIALIZER; + utext_openUChars(&remainingText, remainingChars, remaining16Length, &status); + dest[i] = utext_clone(nullptr, &remainingText, true, false, &status); + utext_close(&remainingText); + } + + uprv_free(remainingChars); + } + break; + } + if (U_FAILURE(status)) { + break; + } + } // end of for loop + return i+1; +} + + +//-------------------------------------------------------------------------------- +// +// start +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::start(UErrorCode &status) const { + return start(0, status); +} + +int64_t RegexMatcher::start64(UErrorCode &status) const { + return start64(0, status); +} + +//-------------------------------------------------------------------------------- +// +// start(int32_t group, UErrorCode &status) +// +//-------------------------------------------------------------------------------- + +int64_t RegexMatcher::start64(int32_t group, UErrorCode &status) const { + if (U_FAILURE(status)) { + return -1; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return -1; + } + if (fMatch == false) { + status = U_REGEX_INVALID_STATE; + return -1; + } + if (group < 0 || group > fPattern->fGroupMap->size()) { + status = U_INDEX_OUTOFBOUNDS_ERROR; + return -1; + } + int64_t s; + if (group == 0) { + s = fMatchStart; + } else { + int32_t groupOffset = fPattern->fGroupMap->elementAti(group-1); + U_ASSERT(groupOffset < fPattern->fFrameSize); + U_ASSERT(groupOffset >= 0); + s = fFrame->fExtra[groupOffset]; + } + + return s; +} + + +int32_t RegexMatcher::start(int32_t group, UErrorCode &status) const { + return (int32_t)start64(group, status); +} + +//-------------------------------------------------------------------------------- +// +// useAnchoringBounds +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::useAnchoringBounds(UBool b) { + fAnchoringBounds = b; + fAnchorStart = (fAnchoringBounds ? fRegionStart : 0); + fAnchorLimit = (fAnchoringBounds ? fRegionLimit : fInputLength); + return *this; +} + + +//-------------------------------------------------------------------------------- +// +// useTransparentBounds +// +//-------------------------------------------------------------------------------- +RegexMatcher &RegexMatcher::useTransparentBounds(UBool b) { + fTransparentBounds = b; + fLookStart = (fTransparentBounds ? 0 : fRegionStart); + fLookLimit = (fTransparentBounds ? fInputLength : fRegionLimit); + return *this; +} + +//-------------------------------------------------------------------------------- +// +// setTimeLimit +// +//-------------------------------------------------------------------------------- +void RegexMatcher::setTimeLimit(int32_t limit, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return; + } + if (limit < 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + fTimeLimit = limit; +} + + +//-------------------------------------------------------------------------------- +// +// getTimeLimit +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::getTimeLimit() const { + return fTimeLimit; +} + + +//-------------------------------------------------------------------------------- +// +// setStackLimit +// +//-------------------------------------------------------------------------------- +void RegexMatcher::setStackLimit(int32_t limit, UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return; + } + if (limit < 0) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + // Reset the matcher. This is needed here in case there is a current match + // whose final stack frame (containing the match results, pointed to by fFrame) + // would be lost by resizing to a smaller stack size. + reset(); + + if (limit == 0) { + // Unlimited stack expansion + fStack->setMaxCapacity(0); + } else { + // Change the units of the limit from bytes to ints, and bump the size up + // to be big enough to hold at least one stack frame for the pattern, + // if it isn't there already. + int32_t adjustedLimit = limit / sizeof(int32_t); + if (adjustedLimit < fPattern->fFrameSize) { + adjustedLimit = fPattern->fFrameSize; + } + fStack->setMaxCapacity(adjustedLimit); + } + fStackLimit = limit; +} + + +//-------------------------------------------------------------------------------- +// +// getStackLimit +// +//-------------------------------------------------------------------------------- +int32_t RegexMatcher::getStackLimit() const { + return fStackLimit; +} + + +//-------------------------------------------------------------------------------- +// +// setMatchCallback +// +//-------------------------------------------------------------------------------- +void RegexMatcher::setMatchCallback(URegexMatchCallback *callback, + const void *context, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + fCallbackFn = callback; + fCallbackContext = context; +} + + +//-------------------------------------------------------------------------------- +// +// getMatchCallback +// +//-------------------------------------------------------------------------------- +void RegexMatcher::getMatchCallback(URegexMatchCallback *&callback, + const void *&context, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + callback = fCallbackFn; + context = fCallbackContext; +} + + +//-------------------------------------------------------------------------------- +// +// setMatchCallback +// +//-------------------------------------------------------------------------------- +void RegexMatcher::setFindProgressCallback(URegexFindProgressCallback *callback, + const void *context, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + fFindProgressCallbackFn = callback; + fFindProgressCallbackContext = context; +} + + +//-------------------------------------------------------------------------------- +// +// getMatchCallback +// +//-------------------------------------------------------------------------------- +void RegexMatcher::getFindProgressCallback(URegexFindProgressCallback *&callback, + const void *&context, + UErrorCode &status) { + if (U_FAILURE(status)) { + return; + } + callback = fFindProgressCallbackFn; + context = fFindProgressCallbackContext; +} + + +//================================================================================ +// +// Code following this point in this file is the internal +// Match Engine Implementation. +// +//================================================================================ + + +//-------------------------------------------------------------------------------- +// +// resetStack +// Discard any previous contents of the state save stack, and initialize a +// new stack frame to all -1. The -1s are needed for capture group limits, +// where they indicate that a group has not yet matched anything. +//-------------------------------------------------------------------------------- +REStackFrame *RegexMatcher::resetStack() { + // Discard any previous contents of the state save stack, and initialize a + // new stack frame with all -1 data. The -1s are needed for capture group limits, + // where they indicate that a group has not yet matched anything. + fStack->removeAllElements(); + + REStackFrame *iFrame = (REStackFrame *)fStack->reserveBlock(fPattern->fFrameSize, fDeferredStatus); + if(U_FAILURE(fDeferredStatus)) { + return nullptr; + } + + int32_t i; + for (i=0; ifFrameSize-RESTACKFRAME_HDRCOUNT; i++) { + iFrame->fExtra[i] = -1; + } + return iFrame; +} + + + +//-------------------------------------------------------------------------------- +// +// isWordBoundary +// in perl, "xab..cd..", \b is true at positions 0,3,5,7 +// For us, +// If the current char is a combining mark, +// \b is false. +// Else Scan backwards to the first non-combining char. +// We are at a boundary if the this char and the original chars are +// opposite in membership in \w set +// +// parameters: pos - the current position in the input buffer +// +// TODO: double-check edge cases at region boundaries. +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::isWordBoundary(int64_t pos) { + UBool isBoundary = false; + UBool cIsWord = false; + + if (pos >= fLookLimit) { + fHitEnd = true; + } else { + // Determine whether char c at current position is a member of the word set of chars. + // If we're off the end of the string, behave as though we're not at a word char. + UTEXT_SETNATIVEINDEX(fInputText, pos); + UChar32 c = UTEXT_CURRENT32(fInputText); + if (u_hasBinaryProperty(c, UCHAR_GRAPHEME_EXTEND) || u_charType(c) == U_FORMAT_CHAR) { + // Current char is a combining one. Not a boundary. + return false; + } + cIsWord = RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET].contains(c); + } + + // Back up until we come to a non-combining char, determine whether + // that char is a word char. + UBool prevCIsWord = false; + for (;;) { + if (UTEXT_GETNATIVEINDEX(fInputText) <= fLookStart) { + break; + } + UChar32 prevChar = UTEXT_PREVIOUS32(fInputText); + if (!(u_hasBinaryProperty(prevChar, UCHAR_GRAPHEME_EXTEND) + || u_charType(prevChar) == U_FORMAT_CHAR)) { + prevCIsWord = RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET].contains(prevChar); + break; + } + } + isBoundary = cIsWord ^ prevCIsWord; + return isBoundary; +} + +UBool RegexMatcher::isChunkWordBoundary(int32_t pos) { + UBool isBoundary = false; + UBool cIsWord = false; + + const char16_t *inputBuf = fInputText->chunkContents; + + if (pos >= fLookLimit) { + fHitEnd = true; + } else { + // Determine whether char c at current position is a member of the word set of chars. + // If we're off the end of the string, behave as though we're not at a word char. + UChar32 c; + U16_GET(inputBuf, fLookStart, pos, fLookLimit, c); + if (u_hasBinaryProperty(c, UCHAR_GRAPHEME_EXTEND) || u_charType(c) == U_FORMAT_CHAR) { + // Current char is a combining one. Not a boundary. + return false; + } + cIsWord = RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET].contains(c); + } + + // Back up until we come to a non-combining char, determine whether + // that char is a word char. + UBool prevCIsWord = false; + for (;;) { + if (pos <= fLookStart) { + break; + } + UChar32 prevChar; + U16_PREV(inputBuf, fLookStart, pos, prevChar); + if (!(u_hasBinaryProperty(prevChar, UCHAR_GRAPHEME_EXTEND) + || u_charType(prevChar) == U_FORMAT_CHAR)) { + prevCIsWord = RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET].contains(prevChar); + break; + } + } + isBoundary = cIsWord ^ prevCIsWord; + return isBoundary; +} + +//-------------------------------------------------------------------------------- +// +// isUWordBoundary +// +// Test for a word boundary using RBBI word break. +// +// parameters: pos - the current position in the input buffer +// +//-------------------------------------------------------------------------------- +UBool RegexMatcher::isUWordBoundary(int64_t pos, UErrorCode &status) { + UBool returnVal = false; + +#if UCONFIG_NO_BREAK_ITERATION==0 + // Note: this point will never be reached if break iteration is configured out. + // Regex patterns that would require this function will fail to compile. + + // If we haven't yet created a break iterator for this matcher, do it now. + if (fWordBreakItr == nullptr) { + fWordBreakItr = BreakIterator::createWordInstance(Locale::getEnglish(), status); + if (U_FAILURE(status)) { + return false; + } + fWordBreakItr->setText(fInputText, status); + } + + // Note: zero width boundary tests like \b see through transparent region bounds, + // which is why fLookLimit is used here, rather than fActiveLimit. + if (pos >= fLookLimit) { + fHitEnd = true; + returnVal = true; // With Unicode word rules, only positions within the interior of "real" + // words are not boundaries. All non-word chars stand by themselves, + // with word boundaries on both sides. + } else { + returnVal = fWordBreakItr->isBoundary((int32_t)pos); + } +#endif + return returnVal; +} + + +int64_t RegexMatcher::followingGCBoundary(int64_t pos, UErrorCode &status) { + int64_t result = pos; + +#if UCONFIG_NO_BREAK_ITERATION==0 + // Note: this point will never be reached if break iteration is configured out. + // Regex patterns that would require this function will fail to compile. + + // If we haven't yet created a break iterator for this matcher, do it now. + if (fGCBreakItr == nullptr) { + fGCBreakItr = BreakIterator::createCharacterInstance(Locale::getEnglish(), status); + if (U_FAILURE(status)) { + return pos; + } + fGCBreakItr->setText(fInputText, status); + } + result = fGCBreakItr->following(pos); + if (result == BreakIterator::DONE) { + result = pos; + } +#endif + return result; +} + +//-------------------------------------------------------------------------------- +// +// IncrementTime This function is called once each TIMER_INITIAL_VALUE state +// saves. Increment the "time" counter, and call the +// user callback function if there is one installed. +// +// If the match operation needs to be aborted, either for a time-out +// or because the user callback asked for it, just set an error status. +// The engine will pick that up and stop in its outer loop. +// +//-------------------------------------------------------------------------------- +void RegexMatcher::IncrementTime(UErrorCode &status) { + fTickCounter = TIMER_INITIAL_VALUE; + fTime++; + if (fCallbackFn != nullptr) { + if ((*fCallbackFn)(fCallbackContext, fTime) == false) { + status = U_REGEX_STOPPED_BY_CALLER; + return; + } + } + if (fTimeLimit > 0 && fTime >= fTimeLimit) { + status = U_REGEX_TIME_OUT; + } +} + +//-------------------------------------------------------------------------------- +// +// StateSave +// Make a new stack frame, initialized as a copy of the current stack frame. +// Set the pattern index in the original stack frame from the operand value +// in the opcode. Execution of the engine continues with the state in +// the newly created stack frame +// +// Note that reserveBlock() may grow the stack, resulting in the +// whole thing being relocated in memory. +// +// Parameters: +// fp The top frame pointer when called. At return, a new +// fame will be present +// savePatIdx An index into the compiled pattern. Goes into the original +// (not new) frame. If execution ever back-tracks out of the +// new frame, this will be where we continue from in the pattern. +// Return +// The new frame pointer. +// +//-------------------------------------------------------------------------------- +inline REStackFrame *RegexMatcher::StateSave(REStackFrame *fp, int64_t savePatIdx, UErrorCode &status) { + if (U_FAILURE(status)) { + return fp; + } + // push storage for a new frame. + int64_t *newFP = fStack->reserveBlock(fFrameSize, status); + if (U_FAILURE(status)) { + // Failure on attempted stack expansion. + // Stack function set some other error code, change it to a more + // specific one for regular expressions. + status = U_REGEX_STACK_OVERFLOW; + // We need to return a writable stack frame, so just return the + // previous frame. The match operation will stop quickly + // because of the error status, after which the frame will never + // be looked at again. + return fp; + } + fp = (REStackFrame *)(newFP - fFrameSize); // in case of realloc of stack. + + // New stack frame = copy of old top frame. + int64_t *source = (int64_t *)fp; + int64_t *dest = newFP; + for (;;) { + *dest++ = *source++; + if (source == newFP) { + break; + } + } + + fTickCounter--; + if (fTickCounter <= 0) { + IncrementTime(status); // Re-initializes fTickCounter + } + fp->fPatIdx = savePatIdx; + return (REStackFrame *)newFP; +} + +#if defined(REGEX_DEBUG) +namespace { +UnicodeString StringFromUText(UText *ut) { + UnicodeString result; + for (UChar32 c = utext_next32From(ut, 0); c != U_SENTINEL; c = UTEXT_NEXT32(ut)) { + result.append(c); + } + return result; +} +} +#endif // REGEX_DEBUG + + +//-------------------------------------------------------------------------------- +// +// MatchAt This is the actual matching engine. +// +// startIdx: begin matching a this index. +// toEnd: if true, match must extend to end of the input region +// +//-------------------------------------------------------------------------------- +void RegexMatcher::MatchAt(int64_t startIdx, UBool toEnd, UErrorCode &status) { + UBool isMatch = false; // True if the we have a match. + + int64_t backSearchIndex = U_INT64_MAX; // used after greedy single-character matches for searching backwards + + int32_t op; // Operation from the compiled pattern, split into + int32_t opType; // the opcode + int32_t opValue; // and the operand value. + +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + printf("MatchAt(startIdx=%ld)\n", startIdx); + printf("Original Pattern: \"%s\"\n", CStr(StringFromUText(fPattern->fPattern))()); + printf("Input String: \"%s\"\n\n", CStr(StringFromUText(fInputText))()); + } +#endif + + if (U_FAILURE(status)) { + return; + } + + // Cache frequently referenced items from the compiled pattern + // + int64_t *pat = fPattern->fCompiledPat->getBuffer(); + + const char16_t *litText = fPattern->fLiteralText.getBuffer(); + UVector *fSets = fPattern->fSets; + + fFrameSize = fPattern->fFrameSize; + REStackFrame *fp = resetStack(); + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return; + } + + fp->fPatIdx = 0; + fp->fInputIdx = startIdx; + + // Zero out the pattern's static data + int32_t i; + for (i = 0; ifDataSize; i++) { + fData[i] = 0; + } + + // + // Main loop for interpreting the compiled pattern. + // One iteration of the loop per pattern operation performed. + // + for (;;) { + op = (int32_t)pat[fp->fPatIdx]; + opType = URX_TYPE(op); + opValue = URX_VAL(op); +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + printf("inputIdx=%ld inputChar=%x sp=%3ld activeLimit=%ld ", fp->fInputIdx, + UTEXT_CURRENT32(fInputText), (int64_t *)fp-fStack->getBuffer(), fActiveLimit); + fPattern->dumpOp(fp->fPatIdx); + } +#endif + fp->fPatIdx++; + + switch (opType) { + + + case URX_NOP: + break; + + + case URX_BACKTRACK: + // Force a backtrack. In some circumstances, the pattern compiler + // will notice that the pattern can't possibly match anything, and will + // emit one of these at that point. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_ONECHAR: + if (fp->fInputIdx < fActiveLimit) { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + if (c == opValue) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } else { + fHitEnd = true; + } + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_STRING: + { + // Test input against a literal string. + // Strings require two slots in the compiled pattern, one for the + // offset to the string text, and one for the length. + + int32_t stringStartIdx = opValue; + op = (int32_t)pat[fp->fPatIdx]; // Fetch the second operand + fp->fPatIdx++; + opType = URX_TYPE(op); + int32_t stringLen = URX_VAL(op); + U_ASSERT(opType == URX_STRING_LEN); + U_ASSERT(stringLen >= 2); + + const char16_t *patternString = litText+stringStartIdx; + int32_t patternStringIndex = 0; + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 inputChar; + UChar32 patternChar; + UBool success = true; + while (patternStringIndex < stringLen) { + if (UTEXT_GETNATIVEINDEX(fInputText) >= fActiveLimit) { + success = false; + fHitEnd = true; + break; + } + inputChar = UTEXT_NEXT32(fInputText); + U16_NEXT(patternString, patternStringIndex, stringLen, patternChar); + if (patternChar != inputChar) { + success = false; + break; + } + } + + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_STATE_SAVE: + fp = StateSave(fp, opValue, status); + break; + + + case URX_END: + // The match loop will exit via this path on a successful match, + // when we reach the end of the pattern. + if (toEnd && fp->fInputIdx != fActiveLimit) { + // The pattern matched, but not to the end of input. Try some more. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + isMatch = true; + goto breakFromLoop; + + // Start and End Capture stack frame variables are laid out out like this: + // fp->fExtra[opValue] - The start of a completed capture group + // opValue+1 - The end of a completed capture group + // opValue+2 - the start of a capture group whose end + // has not yet been reached (and might not ever be). + case URX_START_CAPTURE: + U_ASSERT(opValue >= 0 && opValue < fFrameSize-3); + fp->fExtra[opValue+2] = fp->fInputIdx; + break; + + + case URX_END_CAPTURE: + U_ASSERT(opValue >= 0 && opValue < fFrameSize-3); + U_ASSERT(fp->fExtra[opValue+2] >= 0); // Start pos for this group must be set. + fp->fExtra[opValue] = fp->fExtra[opValue+2]; // Tentative start becomes real. + fp->fExtra[opValue+1] = fp->fInputIdx; // End position + U_ASSERT(fp->fExtra[opValue] <= fp->fExtra[opValue+1]); + break; + + + case URX_DOLLAR: // $, test for End of line + // or for position before new line at end of input + { + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // If we are positioned just before a new-line that is located at the + // end of input, succeed. + UChar32 c = UTEXT_NEXT32(fInputText); + if (UTEXT_GETNATIVEINDEX(fInputText) >= fAnchorLimit) { + if (isLineTerminator(c)) { + // If not in the middle of a CR/LF sequence + if ( !(c==0x0a && fp->fInputIdx>fAnchorStart && ((void)UTEXT_PREVIOUS32(fInputText), UTEXT_PREVIOUS32(fInputText))==0x0d)) { + // At new-line at end of input. Success + fHitEnd = true; + fRequireEnd = true; + + break; + } + } + } else { + UChar32 nextC = UTEXT_NEXT32(fInputText); + if (c == 0x0d && nextC == 0x0a && UTEXT_GETNATIVEINDEX(fInputText) >= fAnchorLimit) { + fHitEnd = true; + fRequireEnd = true; + break; // At CR/LF at end of input. Success + } + } + + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_DOLLAR_D: // $, test for End of Line, in UNIX_LINES mode. + if (fp->fInputIdx >= fAnchorLimit) { + // Off the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } else { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + // Either at the last character of input, or off the end. + if (c == 0x0a && UTEXT_GETNATIVEINDEX(fInputText) == fAnchorLimit) { + fHitEnd = true; + fRequireEnd = true; + break; + } + } + + // Not at end of input. Back-track out. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_DOLLAR_M: // $, test for End of line in multi-line mode + { + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } + // If we are positioned just before a new-line, succeed. + // It makes no difference where the new-line is within the input. + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_CURRENT32(fInputText); + if (isLineTerminator(c)) { + // At a line end, except for the odd chance of being in the middle of a CR/LF sequence + // In multi-line mode, hitting a new-line just before the end of input does not + // set the hitEnd or requireEnd flags + if ( !(c==0x0a && fp->fInputIdx>fAnchorStart && UTEXT_PREVIOUS32(fInputText)==0x0d)) { + break; + } + } + // not at a new line. Fail. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_DOLLAR_MD: // $, test for End of line in multi-line and UNIX_LINES mode + { + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; // Java set requireEnd in this case, even though + break; // adding a new-line would not lose the match. + } + // If we are not positioned just before a new-line, the test fails; backtrack out. + // It makes no difference where the new-line is within the input. + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + if (UTEXT_CURRENT32(fInputText) != 0x0a) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_CARET: // ^, test for start of line + if (fp->fInputIdx != fAnchorStart) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_CARET_M: // ^, test for start of line in mulit-line mode + { + if (fp->fInputIdx == fAnchorStart) { + // We are at the start input. Success. + break; + } + // Check whether character just before the current pos is a new-line + // unless we are at the end of input + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_PREVIOUS32(fInputText); + if ((fp->fInputIdx < fAnchorLimit) && isLineTerminator(c)) { + // It's a new-line. ^ is true. Success. + // TODO: what should be done with positions between a CR and LF? + break; + } + // Not at the start of a line. Fail. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_CARET_M_UNIX: // ^, test for start of line in mulit-line + Unix-line mode + { + U_ASSERT(fp->fInputIdx >= fAnchorStart); + if (fp->fInputIdx <= fAnchorStart) { + // We are at the start input. Success. + break; + } + // Check whether character just before the current pos is a new-line + U_ASSERT(fp->fInputIdx <= fAnchorLimit); + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_PREVIOUS32(fInputText); + if (c != 0x0a) { + // Not at the start of a line. Back-track out. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_BACKSLASH_B: // Test for word boundaries + { + UBool success = isWordBoundary(fp->fInputIdx); + success ^= (UBool)(opValue != 0); // flip sense for \B + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_BU: // Test for word boundaries, Unicode-style + { + UBool success = isUWordBoundary(fp->fInputIdx, status); + success ^= (UBool)(opValue != 0); // flip sense for \B + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_D: // Test for decimal digit + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + UChar32 c = UTEXT_NEXT32(fInputText); + int8_t ctype = u_charType(c); // TODO: make a unicode set for this. Will be faster. + UBool success = (ctype == U_DECIMAL_DIGIT_NUMBER); + success ^= (UBool)(opValue != 0); // flip sense for \D + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_G: // Test for position at end of previous match + if (!((fMatch && fp->fInputIdx==fMatchEnd) || (fMatch==false && fp->fInputIdx==fActiveStart))) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_BACKSLASH_H: // Test for \h, horizontal white space. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + int8_t ctype = u_charType(c); + UBool success = (ctype == U_SPACE_SEPARATOR || c == 9); // SPACE_SEPARATOR || TAB + success ^= (UBool)(opValue != 0); // flip sense for \H + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_R: // Test for \R, any line break sequence. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + if (isLineTerminator(c)) { + if (c == 0x0d && utext_current32(fInputText) == 0x0a) { + utext_next32(fInputText); + } + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_V: // \v, any single line ending character. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + UBool success = isLineTerminator(c); + success ^= (UBool)(opValue != 0); // flip sense for \V + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_X: + // Match a Grapheme, as defined by Unicode UAX 29. + + // Fail if at end of input + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + fp->fInputIdx = followingGCBoundary(fp->fInputIdx, status); + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp->fInputIdx = fActiveLimit; + } + break; + + + case URX_BACKSLASH_Z: // Test for end of Input + if (fp->fInputIdx < fAnchorLimit) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } else { + fHitEnd = true; + fRequireEnd = true; + } + break; + + + + case URX_STATIC_SETREF: + { + // Test input character against one of the predefined sets + // (Word Characters, for example) + // The high bit of the op value is a flag for the match polarity. + // 0: success if input char is in set. + // 1: success if input char is not in set. + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UBool success = ((opValue & URX_NEG_SET) == URX_NEG_SET); + opValue &= ~URX_NEG_SET; + U_ASSERT(opValue > 0 && opValue < URX_LAST_SET); + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 c = UTEXT_NEXT32(fInputText); + if (c < 256) { + Regex8BitSet &s8 = RegexStaticSets::gStaticSets->fPropSets8[opValue]; + if (s8.contains(c)) { + success = !success; + } + } else { + const UnicodeSet &s = RegexStaticSets::gStaticSets->fPropSets[opValue]; + if (s.contains(c)) { + success = !success; + } + } + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + // the character wasn't in the set. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_STAT_SETREF_N: + { + // Test input character for NOT being a member of one of + // the predefined sets (Word Characters, for example) + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + U_ASSERT(opValue > 0 && opValue < URX_LAST_SET); + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + UChar32 c = UTEXT_NEXT32(fInputText); + if (c < 256) { + Regex8BitSet &s8 = RegexStaticSets::gStaticSets->fPropSets8[opValue]; + if (s8.contains(c) == false) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } else { + const UnicodeSet &s = RegexStaticSets::gStaticSets->fPropSets[opValue]; + if (s.contains(c) == false) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } + // the character wasn't in the set. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_SETREF: + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } else { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // There is input left. Pick up one char and test it for set membership. + UChar32 c = UTEXT_NEXT32(fInputText); + U_ASSERT(opValue > 0 && opValue < fSets->size()); + if (c<256) { + Regex8BitSet *s8 = &fPattern->fSets8[opValue]; + if (s8->contains(c)) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } else { + UnicodeSet *s = (UnicodeSet *)fSets->elementAt(opValue); + if (s->contains(c)) { + // The character is in the set. A Match. + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } + + // the character wasn't in the set. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_DOTANY: + { + // . matches anything, but stops at end-of-line. + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // There is input left. Advance over one char, unless we've hit end-of-line + UChar32 c = UTEXT_NEXT32(fInputText); + if (isLineTerminator(c)) { + // End of line in normal mode. . does not match. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + break; + + + case URX_DOTANY_ALL: + { + // ., in dot-matches-all (including new lines) mode + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // There is input left. Advance over one char, except if we are + // at a cr/lf, advance over both of them. + UChar32 c; + c = UTEXT_NEXT32(fInputText); + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + if (c==0x0d && fp->fInputIdx < fActiveLimit) { + // In the case of a CR/LF, we need to advance over both. + UChar32 nextc = UTEXT_CURRENT32(fInputText); + if (nextc == 0x0a) { + (void)UTEXT_NEXT32(fInputText); + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } + } + break; + + + case URX_DOTANY_UNIX: + { + // '.' operator, matches all, but stops at end-of-line. + // UNIX_LINES mode, so 0x0a is the only recognized line ending. + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // There is input left. Advance over one char, unless we've hit end-of-line + UChar32 c = UTEXT_NEXT32(fInputText); + if (c == 0x0a) { + // End of line in normal mode. '.' does not match the \n + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } else { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } + break; + + + case URX_JMP: + fp->fPatIdx = opValue; + break; + + case URX_FAIL: + isMatch = false; + goto breakFromLoop; + + case URX_JMP_SAV: + U_ASSERT(opValue < fPattern->fCompiledPat->size()); + fp = StateSave(fp, fp->fPatIdx, status); // State save to loc following current + fp->fPatIdx = opValue; // Then JMP. + break; + + case URX_JMP_SAV_X: + // This opcode is used with (x)+, when x can match a zero length string. + // Same as JMP_SAV, except conditional on the match having made forward progress. + // Destination of the JMP must be a URX_STO_INP_LOC, from which we get the + // data address of the input position at the start of the loop. + { + U_ASSERT(opValue > 0 && opValue < fPattern->fCompiledPat->size()); + int32_t stoOp = (int32_t)pat[opValue-1]; + U_ASSERT(URX_TYPE(stoOp) == URX_STO_INP_LOC); + int32_t frameLoc = URX_VAL(stoOp); + U_ASSERT(frameLoc >= 0 && frameLoc < fFrameSize); + int64_t prevInputIdx = fp->fExtra[frameLoc]; + U_ASSERT(prevInputIdx <= fp->fInputIdx); + if (prevInputIdx < fp->fInputIdx) { + // The match did make progress. Repeat the loop. + fp = StateSave(fp, fp->fPatIdx, status); // State save to loc following current + fp->fPatIdx = opValue; + fp->fExtra[frameLoc] = fp->fInputIdx; + } + // If the input position did not advance, we do nothing here, + // execution will fall out of the loop. + } + break; + + case URX_CTR_INIT: + { + U_ASSERT(opValue >= 0 && opValue < fFrameSize-2); + fp->fExtra[opValue] = 0; // Set the loop counter variable to zero + + // Pick up the three extra operands that CTR_INIT has, and + // skip the pattern location counter past + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 3; + int32_t loopLoc = URX_VAL(pat[instrOperandLoc]); + int32_t minCount = (int32_t)pat[instrOperandLoc+1]; + int32_t maxCount = (int32_t)pat[instrOperandLoc+2]; + U_ASSERT(minCount>=0); + U_ASSERT(maxCount>=minCount || maxCount==-1); + U_ASSERT(loopLoc>=fp->fPatIdx); + + if (minCount == 0) { + fp = StateSave(fp, loopLoc+1, status); + } + if (maxCount == -1) { + fp->fExtra[opValue+1] = fp->fInputIdx; // For loop breaking. + } else if (maxCount == 0) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_CTR_LOOP: + { + U_ASSERT(opValue>0 && opValue < fp->fPatIdx-2); + int32_t initOp = (int32_t)pat[opValue]; + U_ASSERT(URX_TYPE(initOp) == URX_CTR_INIT); + int64_t *pCounter = &fp->fExtra[URX_VAL(initOp)]; + int32_t minCount = (int32_t)pat[opValue+2]; + int32_t maxCount = (int32_t)pat[opValue+3]; + (*pCounter)++; + if ((uint64_t)*pCounter >= (uint32_t)maxCount && maxCount != -1) { + U_ASSERT(*pCounter == maxCount); + break; + } + if (*pCounter >= minCount) { + if (maxCount == -1) { + // Loop has no hard upper bound. + // Check that it is progressing through the input, break if it is not. + int64_t *pLastInputIdx = &fp->fExtra[URX_VAL(initOp) + 1]; + if (fp->fInputIdx == *pLastInputIdx) { + break; + } else { + *pLastInputIdx = fp->fInputIdx; + } + } + fp = StateSave(fp, fp->fPatIdx, status); + } else { + // Increment time-out counter. (StateSave() does it if count >= minCount) + fTickCounter--; + if (fTickCounter <= 0) { + IncrementTime(status); // Re-initializes fTickCounter + } + } + + fp->fPatIdx = opValue + 4; // Loop back. + } + break; + + case URX_CTR_INIT_NG: + { + // Initialize a non-greedy loop + U_ASSERT(opValue >= 0 && opValue < fFrameSize-2); + fp->fExtra[opValue] = 0; // Set the loop counter variable to zero + + // Pick up the three extra operands that CTR_INIT_NG has, and + // skip the pattern location counter past + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 3; + int32_t loopLoc = URX_VAL(pat[instrOperandLoc]); + int32_t minCount = (int32_t)pat[instrOperandLoc+1]; + int32_t maxCount = (int32_t)pat[instrOperandLoc+2]; + U_ASSERT(minCount>=0); + U_ASSERT(maxCount>=minCount || maxCount==-1); + U_ASSERT(loopLoc>fp->fPatIdx); + if (maxCount == -1) { + fp->fExtra[opValue+1] = fp->fInputIdx; // Save initial input index for loop breaking. + } + + if (minCount == 0) { + if (maxCount != 0) { + fp = StateSave(fp, fp->fPatIdx, status); + } + fp->fPatIdx = loopLoc+1; // Continue with stuff after repeated block + } + } + break; + + case URX_CTR_LOOP_NG: + { + // Non-greedy {min, max} loops + U_ASSERT(opValue>0 && opValue < fp->fPatIdx-2); + int32_t initOp = (int32_t)pat[opValue]; + U_ASSERT(URX_TYPE(initOp) == URX_CTR_INIT_NG); + int64_t *pCounter = &fp->fExtra[URX_VAL(initOp)]; + int32_t minCount = (int32_t)pat[opValue+2]; + int32_t maxCount = (int32_t)pat[opValue+3]; + + (*pCounter)++; + if ((uint64_t)*pCounter >= (uint32_t)maxCount && maxCount != -1) { + // The loop has matched the maximum permitted number of times. + // Break out of here with no action. Matching will + // continue with the following pattern. + U_ASSERT(*pCounter == maxCount); + break; + } + + if (*pCounter < minCount) { + // We haven't met the minimum number of matches yet. + // Loop back for another one. + fp->fPatIdx = opValue + 4; // Loop back. + // Increment time-out counter. (StateSave() does it if count >= minCount) + fTickCounter--; + if (fTickCounter <= 0) { + IncrementTime(status); // Re-initializes fTickCounter + } + } else { + // We do have the minimum number of matches. + + // If there is no upper bound on the loop iterations, check that the input index + // is progressing, and stop the loop if it is not. + if (maxCount == -1) { + int64_t *pLastInputIdx = &fp->fExtra[URX_VAL(initOp) + 1]; + if (fp->fInputIdx == *pLastInputIdx) { + break; + } + *pLastInputIdx = fp->fInputIdx; + } + + // Loop Continuation: we will fall into the pattern following the loop + // (non-greedy, don't execute loop body first), but first do + // a state save to the top of the loop, so that a match failure + // in the following pattern will try another iteration of the loop. + fp = StateSave(fp, opValue + 4, status); + } + } + break; + + case URX_STO_SP: + U_ASSERT(opValue >= 0 && opValue < fPattern->fDataSize); + fData[opValue] = fStack->size(); + break; + + case URX_LD_SP: + { + U_ASSERT(opValue >= 0 && opValue < fPattern->fDataSize); + int32_t newStackSize = (int32_t)fData[opValue]; + U_ASSERT(newStackSize <= fStack->size()); + int64_t *newFP = fStack->getBuffer() + newStackSize - fFrameSize; + if (newFP == (int64_t *)fp) { + break; + } + int32_t j; + for (j=0; jsetSize(newStackSize); + } + break; + + case URX_BACKREF: + { + U_ASSERT(opValue < fFrameSize); + int64_t groupStartIdx = fp->fExtra[opValue]; + int64_t groupEndIdx = fp->fExtra[opValue+1]; + U_ASSERT(groupStartIdx <= groupEndIdx); + if (groupStartIdx < 0) { + // This capture group has not participated in the match thus far, + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no match. + break; + } + UTEXT_SETNATIVEINDEX(fAltInputText, groupStartIdx); + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + // Note: if the capture group match was of an empty string the backref + // match succeeds. Verified by testing: Perl matches succeed + // in this case, so we do too. + + UBool success = true; + for (;;) { + if (utext_getNativeIndex(fAltInputText) >= groupEndIdx) { + success = true; + break; + } + if (utext_getNativeIndex(fInputText) >= fActiveLimit) { + success = false; + fHitEnd = true; + break; + } + UChar32 captureGroupChar = utext_next32(fAltInputText); + UChar32 inputChar = utext_next32(fInputText); + if (inputChar != captureGroupChar) { + success = false; + break; + } + } + + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + + case URX_BACKREF_I: + { + U_ASSERT(opValue < fFrameSize); + int64_t groupStartIdx = fp->fExtra[opValue]; + int64_t groupEndIdx = fp->fExtra[opValue+1]; + U_ASSERT(groupStartIdx <= groupEndIdx); + if (groupStartIdx < 0) { + // This capture group has not participated in the match thus far, + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no match. + break; + } + utext_setNativeIndex(fAltInputText, groupStartIdx); + utext_setNativeIndex(fInputText, fp->fInputIdx); + CaseFoldingUTextIterator captureGroupItr(*fAltInputText); + CaseFoldingUTextIterator inputItr(*fInputText); + + // Note: if the capture group match was of an empty string the backref + // match succeeds. Verified by testing: Perl matches succeed + // in this case, so we do too. + + UBool success = true; + for (;;) { + if (!captureGroupItr.inExpansion() && utext_getNativeIndex(fAltInputText) >= groupEndIdx) { + success = true; + break; + } + if (!inputItr.inExpansion() && utext_getNativeIndex(fInputText) >= fActiveLimit) { + success = false; + fHitEnd = true; + break; + } + UChar32 captureGroupChar = captureGroupItr.next(); + UChar32 inputChar = inputItr.next(); + if (inputChar != captureGroupChar) { + success = false; + break; + } + } + + if (success && inputItr.inExpansion()) { + // We obtained a match by consuming part of a string obtained from + // case-folding a single code point of the input text. + // This does not count as an overall match. + success = false; + } + + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + + } + break; + + case URX_STO_INP_LOC: + { + U_ASSERT(opValue >= 0 && opValue < fFrameSize); + fp->fExtra[opValue] = fp->fInputIdx; + } + break; + + case URX_JMPX: + { + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 1; + int32_t dataLoc = URX_VAL(pat[instrOperandLoc]); + U_ASSERT(dataLoc >= 0 && dataLoc < fFrameSize); + int64_t savedInputIdx = fp->fExtra[dataLoc]; + U_ASSERT(savedInputIdx <= fp->fInputIdx); + if (savedInputIdx < fp->fInputIdx) { + fp->fPatIdx = opValue; // JMP + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no progress in loop. + } + } + break; + + case URX_LA_START: + { + // Entering a look around block. + // Save Stack Ptr, Input Pos. + U_ASSERT(opValue>=0 && opValue+3fDataSize); + fData[opValue] = fStack->size(); + fData[opValue+1] = fp->fInputIdx; + fData[opValue+2] = fActiveStart; + fData[opValue+3] = fActiveLimit; + fActiveStart = fLookStart; // Set the match region change for + fActiveLimit = fLookLimit; // transparent bounds. + } + break; + + case URX_LA_END: + { + // Leaving a look-ahead block. + // restore Stack Ptr, Input Pos to positions they had on entry to block. + U_ASSERT(opValue>=0 && opValue+3fDataSize); + int32_t stackSize = fStack->size(); + int32_t newStackSize =(int32_t)fData[opValue]; + U_ASSERT(stackSize >= newStackSize); + if (stackSize > newStackSize) { + // Copy the current top frame back to the new (cut back) top frame. + // This makes the capture groups from within the look-ahead + // expression available. + int64_t *newFP = fStack->getBuffer() + newStackSize - fFrameSize; + int32_t j; + for (j=0; jsetSize(newStackSize); + } + fp->fInputIdx = fData[opValue+1]; + + // Restore the active region bounds in the input string; they may have + // been changed because of transparent bounds on a Region. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + } + break; + + case URX_ONECHAR_I: + // Case insensitive one char. The char from the pattern is already case folded. + // Input text is not, but case folding the input can not reduce two or more code + // points to one. + if (fp->fInputIdx < fActiveLimit) { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + + UChar32 c = UTEXT_NEXT32(fInputText); + if (u_foldCase(c, U_FOLD_CASE_DEFAULT) == opValue) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + break; + } + } else { + fHitEnd = true; + } + + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + case URX_STRING_I: + { + // Case-insensitive test input against a literal string. + // Strings require two slots in the compiled pattern, one for the + // offset to the string text, and one for the length. + // The compiled string has already been case folded. + { + const char16_t *patternString = litText + opValue; + int32_t patternStringIdx = 0; + + op = (int32_t)pat[fp->fPatIdx]; + fp->fPatIdx++; + opType = URX_TYPE(op); + opValue = URX_VAL(op); + U_ASSERT(opType == URX_STRING_LEN); + int32_t patternStringLen = opValue; // Length of the string from the pattern. + + + UChar32 cPattern; + UChar32 cText; + UBool success = true; + + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + CaseFoldingUTextIterator inputIterator(*fInputText); + while (patternStringIdx < patternStringLen) { + if (!inputIterator.inExpansion() && UTEXT_GETNATIVEINDEX(fInputText) >= fActiveLimit) { + success = false; + fHitEnd = true; + break; + } + U16_NEXT(patternString, patternStringIdx, patternStringLen, cPattern); + cText = inputIterator.next(); + if (cText != cPattern) { + success = false; + break; + } + } + if (inputIterator.inExpansion()) { + success = false; + } + + if (success) { + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + } + break; + + case URX_LB_START: + { + // Entering a look-behind block. + // Save Stack Ptr, Input Pos and active input region. + // TODO: implement transparent bounds. Ticket #6067 + U_ASSERT(opValue>=0 && opValue+4fDataSize); + fData[opValue] = fStack->size(); + fData[opValue+1] = fp->fInputIdx; + // Save input string length, then reset to pin any matches to end at + // the current position. + fData[opValue+2] = fActiveStart; + fData[opValue+3] = fActiveLimit; + fActiveStart = fRegionStart; + fActiveLimit = fp->fInputIdx; + // Init the variable containing the start index for attempted matches. + fData[opValue+4] = -1; + } + break; + + + case URX_LB_CONT: + { + // Positive Look-Behind, at top of loop checking for matches of LB expression + // at all possible input starting positions. + + // Fetch the min and max possible match lengths. They are the operands + // of this op in the pattern. + int32_t minML = (int32_t)pat[fp->fPatIdx++]; + int32_t maxML = (int32_t)pat[fp->fPatIdx++]; + if (!UTEXT_USES_U16(fInputText)) { + // utf-8 fix to maximum match length. The pattern compiler assumes utf-16. + // The max length need not be exact; it just needs to be >= actual maximum. + maxML *= 3; + } + U_ASSERT(minML <= maxML); + U_ASSERT(minML >= 0); + + // Fetch (from data) the last input index where a match was attempted. + U_ASSERT(opValue>=0 && opValue+4fDataSize); + int64_t &lbStartIdx = fData[opValue+4]; + if (lbStartIdx < 0) { + // First time through loop. + lbStartIdx = fp->fInputIdx - minML; + if (lbStartIdx > 0) { + // move index to a code point boundary, if it's not on one already. + UTEXT_SETNATIVEINDEX(fInputText, lbStartIdx); + lbStartIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } else { + // 2nd through nth time through the loop. + // Back up start position for match by one. + if (lbStartIdx == 0) { + (lbStartIdx)--; + } else { + UTEXT_SETNATIVEINDEX(fInputText, lbStartIdx); + (void)UTEXT_PREVIOUS32(fInputText); + lbStartIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } + + if (lbStartIdx < 0 || lbStartIdx < fp->fInputIdx - maxML) { + // We have tried all potential match starting points without + // getting a match. Backtrack out, and out of the + // Look Behind altogether. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + break; + } + + // Save state to this URX_LB_CONT op, so failure to match will repeat the loop. + // (successful match will fall off the end of the loop.) + fp = StateSave(fp, fp->fPatIdx-3, status); + fp->fInputIdx = lbStartIdx; + } + break; + + case URX_LB_END: + // End of a look-behind block, after a successful match. + { + U_ASSERT(opValue>=0 && opValue+4fDataSize); + if (fp->fInputIdx != fActiveLimit) { + // The look-behind expression matched, but the match did not + // extend all the way to the point that we are looking behind from. + // FAIL out of here, which will take us back to the LB_CONT, which + // will retry the match starting at another position or fail + // the look-behind altogether, whichever is appropriate. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // Look-behind match is good. Restore the original input string region, + // which had been truncated to pin the end of the lookbehind match to the + // position being looked-behind. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + } + break; + + + case URX_LBN_CONT: + { + // Negative Look-Behind, at top of loop checking for matches of LB expression + // at all possible input starting positions. + + // Fetch the extra parameters of this op. + int32_t minML = (int32_t)pat[fp->fPatIdx++]; + int32_t maxML = (int32_t)pat[fp->fPatIdx++]; + if (!UTEXT_USES_U16(fInputText)) { + // utf-8 fix to maximum match length. The pattern compiler assumes utf-16. + // The max length need not be exact; it just needs to be >= actual maximum. + maxML *= 3; + } + int32_t continueLoc = (int32_t)pat[fp->fPatIdx++]; + continueLoc = URX_VAL(continueLoc); + U_ASSERT(minML <= maxML); + U_ASSERT(minML >= 0); + U_ASSERT(continueLoc > fp->fPatIdx); + + // Fetch (from data) the last input index where a match was attempted. + U_ASSERT(opValue>=0 && opValue+4fDataSize); + int64_t &lbStartIdx = fData[opValue+4]; + if (lbStartIdx < 0) { + // First time through loop. + lbStartIdx = fp->fInputIdx - minML; + if (lbStartIdx > 0) { + // move index to a code point boundary, if it's not on one already. + UTEXT_SETNATIVEINDEX(fInputText, lbStartIdx); + lbStartIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } else { + // 2nd through nth time through the loop. + // Back up start position for match by one. + if (lbStartIdx == 0) { + (lbStartIdx)--; + } else { + UTEXT_SETNATIVEINDEX(fInputText, lbStartIdx); + (void)UTEXT_PREVIOUS32(fInputText); + lbStartIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } + + if (lbStartIdx < 0 || lbStartIdx < fp->fInputIdx - maxML) { + // We have tried all potential match starting points without + // getting a match, which means that the negative lookbehind as + // a whole has succeeded. Jump forward to the continue location + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + fp->fPatIdx = continueLoc; + break; + } + + // Save state to this URX_LB_CONT op, so failure to match will repeat the loop. + // (successful match will cause a FAIL out of the loop altogether.) + fp = StateSave(fp, fp->fPatIdx-4, status); + fp->fInputIdx = lbStartIdx; + } + break; + + case URX_LBN_END: + // End of a negative look-behind block, after a successful match. + { + U_ASSERT(opValue>=0 && opValue+4fDataSize); + if (fp->fInputIdx != fActiveLimit) { + // The look-behind expression matched, but the match did not + // extend all the way to the point that we are looking behind from. + // FAIL out of here, which will take us back to the LB_CONT, which + // will retry the match starting at another position or succeed + // the look-behind altogether, whichever is appropriate. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // Look-behind expression matched, which means look-behind test as + // a whole Fails + + // Restore the original input string length, which had been truncated + // inorder to pin the end of the lookbehind match + // to the position being looked-behind. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + + // Restore original stack position, discarding any state saved + // by the successful pattern match. + U_ASSERT(opValue>=0 && opValue+1fDataSize); + int32_t newStackSize = (int32_t)fData[opValue]; + U_ASSERT(fStack->size() > newStackSize); + fStack->setSize(newStackSize); + + // FAIL, which will take control back to someplace + // prior to entering the look-behind test. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_LOOP_SR_I: + // Loop Initialization for the optimized implementation of + // [some character set]* + // This op scans through all matching input. + // The following LOOP_C op emulates stack unwinding if the following pattern fails. + { + U_ASSERT(opValue > 0 && opValue < fSets->size()); + Regex8BitSet *s8 = &fPattern->fSets8[opValue]; + UnicodeSet *s = (UnicodeSet *)fSets->elementAt(opValue); + + // Loop through input, until either the input is exhausted or + // we reach a character that is not a member of the set. + int64_t ix = fp->fInputIdx; + UTEXT_SETNATIVEINDEX(fInputText, ix); + for (;;) { + if (ix >= fActiveLimit) { + fHitEnd = true; + break; + } + UChar32 c = UTEXT_NEXT32(fInputText); + if (c<256) { + if (s8->contains(c) == false) { + break; + } + } else { + if (s->contains(c) == false) { + break; + } + } + ix = UTEXT_GETNATIVEINDEX(fInputText); + } + + // If there were no matching characters, skip over the loop altogether. + // The loop doesn't run at all, a * op always succeeds. + if (ix == fp->fInputIdx) { + fp->fPatIdx++; // skip the URX_LOOP_C op. + break; + } + + // Peek ahead in the compiled pattern, to the URX_LOOP_C that + // must follow. It's operand is the stack location + // that holds the starting input index for the match of this [set]* + int32_t loopcOp = (int32_t)pat[fp->fPatIdx]; + U_ASSERT(URX_TYPE(loopcOp) == URX_LOOP_C); + int32_t stackLoc = URX_VAL(loopcOp); + U_ASSERT(stackLoc >= 0 && stackLoc < fFrameSize); + fp->fExtra[stackLoc] = fp->fInputIdx; + fp->fInputIdx = ix; + + // Save State to the URX_LOOP_C op that follows this one, + // so that match failures in the following code will return to there. + // Then bump the pattern idx so the LOOP_C is skipped on the way out of here. + fp = StateSave(fp, fp->fPatIdx, status); + fp->fPatIdx++; + } + break; + + + case URX_LOOP_DOT_I: + // Loop Initialization for the optimized implementation of .* + // This op scans through all remaining input. + // The following LOOP_C op emulates stack unwinding if the following pattern fails. + { + // Loop through input until the input is exhausted (we reach an end-of-line) + // In DOTALL mode, we can just go straight to the end of the input. + int64_t ix; + if ((opValue & 1) == 1) { + // Dot-matches-All mode. Jump straight to the end of the string. + ix = fActiveLimit; + fHitEnd = true; + } else { + // NOT DOT ALL mode. Line endings do not match '.' + // Scan forward until a line ending or end of input. + ix = fp->fInputIdx; + UTEXT_SETNATIVEINDEX(fInputText, ix); + for (;;) { + if (ix >= fActiveLimit) { + fHitEnd = true; + break; + } + UChar32 c = UTEXT_NEXT32(fInputText); + if ((c & 0x7f) <= 0x29) { // Fast filter of non-new-line-s + if ((c == 0x0a) || // 0x0a is newline in both modes. + (((opValue & 2) == 0) && // IF not UNIX_LINES mode + isLineTerminator(c))) { + // char is a line ending. Exit the scanning loop. + break; + } + } + ix = UTEXT_GETNATIVEINDEX(fInputText); + } + } + + // If there were no matching characters, skip over the loop altogether. + // The loop doesn't run at all, a * op always succeeds. + if (ix == fp->fInputIdx) { + fp->fPatIdx++; // skip the URX_LOOP_C op. + break; + } + + // Peek ahead in the compiled pattern, to the URX_LOOP_C that + // must follow. It's operand is the stack location + // that holds the starting input index for the match of this .* + int32_t loopcOp = (int32_t)pat[fp->fPatIdx]; + U_ASSERT(URX_TYPE(loopcOp) == URX_LOOP_C); + int32_t stackLoc = URX_VAL(loopcOp); + U_ASSERT(stackLoc >= 0 && stackLoc < fFrameSize); + fp->fExtra[stackLoc] = fp->fInputIdx; + fp->fInputIdx = ix; + + // Save State to the URX_LOOP_C op that follows this one, + // so that match failures in the following code will return to there. + // Then bump the pattern idx so the LOOP_C is skipped on the way out of here. + fp = StateSave(fp, fp->fPatIdx, status); + fp->fPatIdx++; + } + break; + + + case URX_LOOP_C: + { + U_ASSERT(opValue>=0 && opValuefExtra[opValue]; + U_ASSERT(backSearchIndex <= fp->fInputIdx); + if (backSearchIndex == fp->fInputIdx) { + // We've backed up the input idx to the point that the loop started. + // The loop is done. Leave here without saving state. + // Subsequent failures won't come back here. + break; + } + // Set up for the next iteration of the loop, with input index + // backed up by one from the last time through, + // and a state save to this instruction in case the following code fails again. + // (We're going backwards because this loop emulates stack unwinding, not + // the initial scan forward.) + U_ASSERT(fp->fInputIdx > 0); + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + UChar32 prevC = UTEXT_PREVIOUS32(fInputText); + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + + UChar32 twoPrevC = UTEXT_PREVIOUS32(fInputText); + if (prevC == 0x0a && + fp->fInputIdx > backSearchIndex && + twoPrevC == 0x0d) { + int32_t prevOp = (int32_t)pat[fp->fPatIdx-2]; + if (URX_TYPE(prevOp) == URX_LOOP_DOT_I) { + // .*, stepping back over CRLF pair. + fp->fInputIdx = UTEXT_GETNATIVEINDEX(fInputText); + } + } + + + fp = StateSave(fp, fp->fPatIdx-1, status); + } + break; + + + + default: + // Trouble. The compiled pattern contains an entry with an + // unrecognized type tag. + UPRV_UNREACHABLE_ASSERT; + // Unknown opcode type in opType = URX_TYPE(pat[fp->fPatIdx]). But we have + // reports of this in production code, don't use UPRV_UNREACHABLE_EXIT. + // See ICU-21669. + status = U_INTERNAL_PROGRAM_ERROR; + } + + if (U_FAILURE(status)) { + isMatch = false; + break; + } + } + +breakFromLoop: + fMatch = isMatch; + if (isMatch) { + fLastMatchEnd = fMatchEnd; + fMatchStart = startIdx; + fMatchEnd = fp->fInputIdx; + } + +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + if (isMatch) { + printf("Match. start=%ld end=%ld\n\n", fMatchStart, fMatchEnd); + } else { + printf("No match\n\n"); + } + } +#endif + + fFrame = fp; // The active stack frame when the engine stopped. + // Contains the capture group results that we need to + // access later. + return; +} + + +//-------------------------------------------------------------------------------- +// +// MatchChunkAt This is the actual matching engine. Like MatchAt, but with the +// assumption that the entire string is available in the UText's +// chunk buffer. For now, that means we can use int32_t indexes, +// except for anything that needs to be saved (like group starts +// and ends). +// +// startIdx: begin matching a this index. +// toEnd: if true, match must extend to end of the input region +// +//-------------------------------------------------------------------------------- +void RegexMatcher::MatchChunkAt(int32_t startIdx, UBool toEnd, UErrorCode &status) { + UBool isMatch = false; // True if the we have a match. + + int32_t backSearchIndex = INT32_MAX; // used after greedy single-character matches for searching backwards + + int32_t op; // Operation from the compiled pattern, split into + int32_t opType; // the opcode + int32_t opValue; // and the operand value. + +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + printf("MatchAt(startIdx=%d)\n", startIdx); + printf("Original Pattern: \"%s\"\n", CStr(StringFromUText(fPattern->fPattern))()); + printf("Input String: \"%s\"\n\n", CStr(StringFromUText(fInputText))()); + } +#endif + + if (U_FAILURE(status)) { + return; + } + + // Cache frequently referenced items from the compiled pattern + // + int64_t *pat = fPattern->fCompiledPat->getBuffer(); + + const char16_t *litText = fPattern->fLiteralText.getBuffer(); + UVector *fSets = fPattern->fSets; + + const char16_t *inputBuf = fInputText->chunkContents; + + fFrameSize = fPattern->fFrameSize; + REStackFrame *fp = resetStack(); + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return; + } + + fp->fPatIdx = 0; + fp->fInputIdx = startIdx; + + // Zero out the pattern's static data + int32_t i; + for (i = 0; ifDataSize; i++) { + fData[i] = 0; + } + + // + // Main loop for interpreting the compiled pattern. + // One iteration of the loop per pattern operation performed. + // + for (;;) { + op = (int32_t)pat[fp->fPatIdx]; + opType = URX_TYPE(op); + opValue = URX_VAL(op); +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + UTEXT_SETNATIVEINDEX(fInputText, fp->fInputIdx); + printf("inputIdx=%ld inputChar=%x sp=%3ld activeLimit=%ld ", fp->fInputIdx, + UTEXT_CURRENT32(fInputText), (int64_t *)fp-fStack->getBuffer(), fActiveLimit); + fPattern->dumpOp(fp->fPatIdx); + } +#endif + fp->fPatIdx++; + + switch (opType) { + + + case URX_NOP: + break; + + + case URX_BACKTRACK: + // Force a backtrack. In some circumstances, the pattern compiler + // will notice that the pattern can't possibly match anything, and will + // emit one of these at that point. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_ONECHAR: + if (fp->fInputIdx < fActiveLimit) { + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c == opValue) { + break; + } + } else { + fHitEnd = true; + } + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_STRING: + { + // Test input against a literal string. + // Strings require two slots in the compiled pattern, one for the + // offset to the string text, and one for the length. + int32_t stringStartIdx = opValue; + int32_t stringLen; + + op = (int32_t)pat[fp->fPatIdx]; // Fetch the second operand + fp->fPatIdx++; + opType = URX_TYPE(op); + stringLen = URX_VAL(op); + U_ASSERT(opType == URX_STRING_LEN); + U_ASSERT(stringLen >= 2); + + const char16_t * pInp = inputBuf + fp->fInputIdx; + const char16_t * pInpLimit = inputBuf + fActiveLimit; + const char16_t * pPat = litText+stringStartIdx; + const char16_t * pEnd = pInp + stringLen; + UBool success = true; + while (pInp < pEnd) { + if (pInp >= pInpLimit) { + fHitEnd = true; + success = false; + break; + } + if (*pInp++ != *pPat++) { + success = false; + break; + } + } + + if (success) { + fp->fInputIdx += stringLen; + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_STATE_SAVE: + fp = StateSave(fp, opValue, status); + break; + + + case URX_END: + // The match loop will exit via this path on a successful match, + // when we reach the end of the pattern. + if (toEnd && fp->fInputIdx != fActiveLimit) { + // The pattern matched, but not to the end of input. Try some more. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + isMatch = true; + goto breakFromLoop; + + // Start and End Capture stack frame variables are laid out out like this: + // fp->fExtra[opValue] - The start of a completed capture group + // opValue+1 - The end of a completed capture group + // opValue+2 - the start of a capture group whose end + // has not yet been reached (and might not ever be). + case URX_START_CAPTURE: + U_ASSERT(opValue >= 0 && opValue < fFrameSize-3); + fp->fExtra[opValue+2] = fp->fInputIdx; + break; + + + case URX_END_CAPTURE: + U_ASSERT(opValue >= 0 && opValue < fFrameSize-3); + U_ASSERT(fp->fExtra[opValue+2] >= 0); // Start pos for this group must be set. + fp->fExtra[opValue] = fp->fExtra[opValue+2]; // Tentative start becomes real. + fp->fExtra[opValue+1] = fp->fInputIdx; // End position + U_ASSERT(fp->fExtra[opValue] <= fp->fExtra[opValue+1]); + break; + + + case URX_DOLLAR: // $, test for End of line + // or for position before new line at end of input + if (fp->fInputIdx < fAnchorLimit-2) { + // We are no where near the end of input. Fail. + // This is the common case. Keep it first. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } + + // If we are positioned just before a new-line that is located at the + // end of input, succeed. + if (fp->fInputIdx == fAnchorLimit-1) { + UChar32 c; + U16_GET(inputBuf, fAnchorStart, fp->fInputIdx, fAnchorLimit, c); + + if (isLineTerminator(c)) { + if ( !(c==0x0a && fp->fInputIdx>fAnchorStart && inputBuf[fp->fInputIdx-1]==0x0d)) { + // At new-line at end of input. Success + fHitEnd = true; + fRequireEnd = true; + break; + } + } + } else if (fp->fInputIdx == fAnchorLimit-2 && + inputBuf[fp->fInputIdx]==0x0d && inputBuf[fp->fInputIdx+1]==0x0a) { + fHitEnd = true; + fRequireEnd = true; + break; // At CR/LF at end of input. Success + } + + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + + break; + + + case URX_DOLLAR_D: // $, test for End of Line, in UNIX_LINES mode. + if (fp->fInputIdx >= fAnchorLimit-1) { + // Either at the last character of input, or off the end. + if (fp->fInputIdx == fAnchorLimit-1) { + // At last char of input. Success if it's a new line. + if (inputBuf[fp->fInputIdx] == 0x0a) { + fHitEnd = true; + fRequireEnd = true; + break; + } + } else { + // Off the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } + } + + // Not at end of input. Back-track out. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + + case URX_DOLLAR_M: // $, test for End of line in multi-line mode + { + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; + break; + } + // If we are positioned just before a new-line, succeed. + // It makes no difference where the new-line is within the input. + UChar32 c = inputBuf[fp->fInputIdx]; + if (isLineTerminator(c)) { + // At a line end, except for the odd chance of being in the middle of a CR/LF sequence + // In multi-line mode, hitting a new-line just before the end of input does not + // set the hitEnd or requireEnd flags + if ( !(c==0x0a && fp->fInputIdx>fAnchorStart && inputBuf[fp->fInputIdx-1]==0x0d)) { + break; + } + } + // not at a new line. Fail. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_DOLLAR_MD: // $, test for End of line in multi-line and UNIX_LINES mode + { + if (fp->fInputIdx >= fAnchorLimit) { + // We really are at the end of input. Success. + fHitEnd = true; + fRequireEnd = true; // Java set requireEnd in this case, even though + break; // adding a new-line would not lose the match. + } + // If we are not positioned just before a new-line, the test fails; backtrack out. + // It makes no difference where the new-line is within the input. + if (inputBuf[fp->fInputIdx] != 0x0a) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_CARET: // ^, test for start of line + if (fp->fInputIdx != fAnchorStart) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_CARET_M: // ^, test for start of line in mulit-line mode + { + if (fp->fInputIdx == fAnchorStart) { + // We are at the start input. Success. + break; + } + // Check whether character just before the current pos is a new-line + // unless we are at the end of input + char16_t c = inputBuf[fp->fInputIdx - 1]; + if ((fp->fInputIdx < fAnchorLimit) && + isLineTerminator(c)) { + // It's a new-line. ^ is true. Success. + // TODO: what should be done with positions between a CR and LF? + break; + } + // Not at the start of a line. Fail. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_CARET_M_UNIX: // ^, test for start of line in mulit-line + Unix-line mode + { + U_ASSERT(fp->fInputIdx >= fAnchorStart); + if (fp->fInputIdx <= fAnchorStart) { + // We are at the start input. Success. + break; + } + // Check whether character just before the current pos is a new-line + U_ASSERT(fp->fInputIdx <= fAnchorLimit); + char16_t c = inputBuf[fp->fInputIdx - 1]; + if (c != 0x0a) { + // Not at the start of a line. Back-track out. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_BACKSLASH_B: // Test for word boundaries + { + UBool success = isChunkWordBoundary((int32_t)fp->fInputIdx); + success ^= (UBool)(opValue != 0); // flip sense for \B + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_BU: // Test for word boundaries, Unicode-style + { + UBool success = isUWordBoundary(fp->fInputIdx, status); + success ^= (UBool)(opValue != 0); // flip sense for \B + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_D: // Test for decimal digit + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + int8_t ctype = u_charType(c); // TODO: make a unicode set for this. Will be faster. + UBool success = (ctype == U_DECIMAL_DIGIT_NUMBER); + success ^= (UBool)(opValue != 0); // flip sense for \D + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_G: // Test for position at end of previous match + if (!((fMatch && fp->fInputIdx==fMatchEnd) || (fMatch==false && fp->fInputIdx==fActiveStart))) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_BACKSLASH_H: // Test for \h, horizontal white space. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + int8_t ctype = u_charType(c); + UBool success = (ctype == U_SPACE_SEPARATOR || c == 9); // SPACE_SEPARATOR || TAB + success ^= (UBool)(opValue != 0); // flip sense for \H + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_R: // Test for \R, any line break sequence. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (isLineTerminator(c)) { + if (c == 0x0d && fp->fInputIdx < fActiveLimit) { + // Check for CR/LF sequence. Consume both together when found. + char16_t c2; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c2); + if (c2 != 0x0a) { + U16_PREV(inputBuf, 0, fp->fInputIdx, c2); + } + } + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_V: // Any single code point line ending. + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + UBool success = isLineTerminator(c); + success ^= (UBool)(opValue != 0); // flip sense for \V + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_BACKSLASH_X: + // Match a Grapheme, as defined by Unicode UAX 29. + + // Fail if at end of input + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + fp->fInputIdx = followingGCBoundary(fp->fInputIdx, status); + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp->fInputIdx = fActiveLimit; + } + break; + + + case URX_BACKSLASH_Z: // Test for end of Input + if (fp->fInputIdx < fAnchorLimit) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } else { + fHitEnd = true; + fRequireEnd = true; + } + break; + + + + case URX_STATIC_SETREF: + { + // Test input character against one of the predefined sets + // (Word Characters, for example) + // The high bit of the op value is a flag for the match polarity. + // 0: success if input char is in set. + // 1: success if input char is not in set. + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + UBool success = ((opValue & URX_NEG_SET) == URX_NEG_SET); + opValue &= ~URX_NEG_SET; + U_ASSERT(opValue > 0 && opValue < URX_LAST_SET); + + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c < 256) { + Regex8BitSet &s8 = RegexStaticSets::gStaticSets->fPropSets8[opValue]; + if (s8.contains(c)) { + success = !success; + } + } else { + const UnicodeSet &s = RegexStaticSets::gStaticSets->fPropSets[opValue]; + if (s.contains(c)) { + success = !success; + } + } + if (!success) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_STAT_SETREF_N: + { + // Test input character for NOT being a member of one of + // the predefined sets (Word Characters, for example) + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + U_ASSERT(opValue > 0 && opValue < URX_LAST_SET); + + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c < 256) { + Regex8BitSet &s8 = RegexStaticSets::gStaticSets->fPropSets8[opValue]; + if (s8.contains(c) == false) { + break; + } + } else { + const UnicodeSet &s = RegexStaticSets::gStaticSets->fPropSets[opValue]; + if (s.contains(c) == false) { + break; + } + } + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_SETREF: + { + if (fp->fInputIdx >= fActiveLimit) { + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + U_ASSERT(opValue > 0 && opValue < fSets->size()); + + // There is input left. Pick up one char and test it for set membership. + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c<256) { + Regex8BitSet *s8 = &fPattern->fSets8[opValue]; + if (s8->contains(c)) { + // The character is in the set. A Match. + break; + } + } else { + UnicodeSet *s = (UnicodeSet *)fSets->elementAt(opValue); + if (s->contains(c)) { + // The character is in the set. A Match. + break; + } + } + + // the character wasn't in the set. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_DOTANY: + { + // . matches anything, but stops at end-of-line. + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // There is input left. Advance over one char, unless we've hit end-of-line + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (isLineTerminator(c)) { + // End of line in normal mode. . does not match. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + } + break; + + + case URX_DOTANY_ALL: + { + // . in dot-matches-all (including new lines) mode + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // There is input left. Advance over one char, except if we are + // at a cr/lf, advance over both of them. + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c==0x0d && fp->fInputIdx < fActiveLimit) { + // In the case of a CR/LF, we need to advance over both. + if (inputBuf[fp->fInputIdx] == 0x0a) { + U16_FWD_1(inputBuf, fp->fInputIdx, fActiveLimit); + } + } + } + break; + + + case URX_DOTANY_UNIX: + { + // '.' operator, matches all, but stops at end-of-line. + // UNIX_LINES mode, so 0x0a is the only recognized line ending. + if (fp->fInputIdx >= fActiveLimit) { + // At end of input. Match failed. Backtrack out. + fHitEnd = true; + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // There is input left. Advance over one char, unless we've hit end-of-line + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (c == 0x0a) { + // End of line in normal mode. '.' does not match the \n + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + + case URX_JMP: + fp->fPatIdx = opValue; + break; + + case URX_FAIL: + isMatch = false; + goto breakFromLoop; + + case URX_JMP_SAV: + U_ASSERT(opValue < fPattern->fCompiledPat->size()); + fp = StateSave(fp, fp->fPatIdx, status); // State save to loc following current + fp->fPatIdx = opValue; // Then JMP. + break; + + case URX_JMP_SAV_X: + // This opcode is used with (x)+, when x can match a zero length string. + // Same as JMP_SAV, except conditional on the match having made forward progress. + // Destination of the JMP must be a URX_STO_INP_LOC, from which we get the + // data address of the input position at the start of the loop. + { + U_ASSERT(opValue > 0 && opValue < fPattern->fCompiledPat->size()); + int32_t stoOp = (int32_t)pat[opValue-1]; + U_ASSERT(URX_TYPE(stoOp) == URX_STO_INP_LOC); + int32_t frameLoc = URX_VAL(stoOp); + U_ASSERT(frameLoc >= 0 && frameLoc < fFrameSize); + int32_t prevInputIdx = (int32_t)fp->fExtra[frameLoc]; + U_ASSERT(prevInputIdx <= fp->fInputIdx); + if (prevInputIdx < fp->fInputIdx) { + // The match did make progress. Repeat the loop. + fp = StateSave(fp, fp->fPatIdx, status); // State save to loc following current + fp->fPatIdx = opValue; + fp->fExtra[frameLoc] = fp->fInputIdx; + } + // If the input position did not advance, we do nothing here, + // execution will fall out of the loop. + } + break; + + case URX_CTR_INIT: + { + U_ASSERT(opValue >= 0 && opValue < fFrameSize-2); + fp->fExtra[opValue] = 0; // Set the loop counter variable to zero + + // Pick up the three extra operands that CTR_INIT has, and + // skip the pattern location counter past + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 3; + int32_t loopLoc = URX_VAL(pat[instrOperandLoc]); + int32_t minCount = (int32_t)pat[instrOperandLoc+1]; + int32_t maxCount = (int32_t)pat[instrOperandLoc+2]; + U_ASSERT(minCount>=0); + U_ASSERT(maxCount>=minCount || maxCount==-1); + U_ASSERT(loopLoc>=fp->fPatIdx); + + if (minCount == 0) { + fp = StateSave(fp, loopLoc+1, status); + } + if (maxCount == -1) { + fp->fExtra[opValue+1] = fp->fInputIdx; // For loop breaking. + } else if (maxCount == 0) { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_CTR_LOOP: + { + U_ASSERT(opValue>0 && opValue < fp->fPatIdx-2); + int32_t initOp = (int32_t)pat[opValue]; + U_ASSERT(URX_TYPE(initOp) == URX_CTR_INIT); + int64_t *pCounter = &fp->fExtra[URX_VAL(initOp)]; + int32_t minCount = (int32_t)pat[opValue+2]; + int32_t maxCount = (int32_t)pat[opValue+3]; + (*pCounter)++; + if ((uint64_t)*pCounter >= (uint32_t)maxCount && maxCount != -1) { + U_ASSERT(*pCounter == maxCount); + break; + } + if (*pCounter >= minCount) { + if (maxCount == -1) { + // Loop has no hard upper bound. + // Check that it is progressing through the input, break if it is not. + int64_t *pLastInputIdx = &fp->fExtra[URX_VAL(initOp) + 1]; + if (fp->fInputIdx == *pLastInputIdx) { + break; + } else { + *pLastInputIdx = fp->fInputIdx; + } + } + fp = StateSave(fp, fp->fPatIdx, status); + } else { + // Increment time-out counter. (StateSave() does it if count >= minCount) + fTickCounter--; + if (fTickCounter <= 0) { + IncrementTime(status); // Re-initializes fTickCounter + } + } + fp->fPatIdx = opValue + 4; // Loop back. + } + break; + + case URX_CTR_INIT_NG: + { + // Initialize a non-greedy loop + U_ASSERT(opValue >= 0 && opValue < fFrameSize-2); + fp->fExtra[opValue] = 0; // Set the loop counter variable to zero + + // Pick up the three extra operands that CTR_INIT_NG has, and + // skip the pattern location counter past + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 3; + int32_t loopLoc = URX_VAL(pat[instrOperandLoc]); + int32_t minCount = (int32_t)pat[instrOperandLoc+1]; + int32_t maxCount = (int32_t)pat[instrOperandLoc+2]; + U_ASSERT(minCount>=0); + U_ASSERT(maxCount>=minCount || maxCount==-1); + U_ASSERT(loopLoc>fp->fPatIdx); + if (maxCount == -1) { + fp->fExtra[opValue+1] = fp->fInputIdx; // Save initial input index for loop breaking. + } + + if (minCount == 0) { + if (maxCount != 0) { + fp = StateSave(fp, fp->fPatIdx, status); + } + fp->fPatIdx = loopLoc+1; // Continue with stuff after repeated block + } + } + break; + + case URX_CTR_LOOP_NG: + { + // Non-greedy {min, max} loops + U_ASSERT(opValue>0 && opValue < fp->fPatIdx-2); + int32_t initOp = (int32_t)pat[opValue]; + U_ASSERT(URX_TYPE(initOp) == URX_CTR_INIT_NG); + int64_t *pCounter = &fp->fExtra[URX_VAL(initOp)]; + int32_t minCount = (int32_t)pat[opValue+2]; + int32_t maxCount = (int32_t)pat[opValue+3]; + + (*pCounter)++; + if ((uint64_t)*pCounter >= (uint32_t)maxCount && maxCount != -1) { + // The loop has matched the maximum permitted number of times. + // Break out of here with no action. Matching will + // continue with the following pattern. + U_ASSERT(*pCounter == maxCount); + break; + } + + if (*pCounter < minCount) { + // We haven't met the minimum number of matches yet. + // Loop back for another one. + fp->fPatIdx = opValue + 4; // Loop back. + fTickCounter--; + if (fTickCounter <= 0) { + IncrementTime(status); // Re-initializes fTickCounter + } + } else { + // We do have the minimum number of matches. + + // If there is no upper bound on the loop iterations, check that the input index + // is progressing, and stop the loop if it is not. + if (maxCount == -1) { + int64_t *pLastInputIdx = &fp->fExtra[URX_VAL(initOp) + 1]; + if (fp->fInputIdx == *pLastInputIdx) { + break; + } + *pLastInputIdx = fp->fInputIdx; + } + + // Loop Continuation: we will fall into the pattern following the loop + // (non-greedy, don't execute loop body first), but first do + // a state save to the top of the loop, so that a match failure + // in the following pattern will try another iteration of the loop. + fp = StateSave(fp, opValue + 4, status); + } + } + break; + + case URX_STO_SP: + U_ASSERT(opValue >= 0 && opValue < fPattern->fDataSize); + fData[opValue] = fStack->size(); + break; + + case URX_LD_SP: + { + U_ASSERT(opValue >= 0 && opValue < fPattern->fDataSize); + int32_t newStackSize = (int32_t)fData[opValue]; + U_ASSERT(newStackSize <= fStack->size()); + int64_t *newFP = fStack->getBuffer() + newStackSize - fFrameSize; + if (newFP == (int64_t *)fp) { + break; + } + int32_t j; + for (j=0; jsetSize(newStackSize); + } + break; + + case URX_BACKREF: + { + U_ASSERT(opValue < fFrameSize); + int64_t groupStartIdx = fp->fExtra[opValue]; + int64_t groupEndIdx = fp->fExtra[opValue+1]; + U_ASSERT(groupStartIdx <= groupEndIdx); + int64_t inputIndex = fp->fInputIdx; + if (groupStartIdx < 0) { + // This capture group has not participated in the match thus far, + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no match. + break; + } + UBool success = true; + for (int64_t groupIndex = groupStartIdx; groupIndex < groupEndIdx; ++groupIndex,++inputIndex) { + if (inputIndex >= fActiveLimit) { + success = false; + fHitEnd = true; + break; + } + if (inputBuf[groupIndex] != inputBuf[inputIndex]) { + success = false; + break; + } + } + if (success && groupStartIdx < groupEndIdx && U16_IS_LEAD(inputBuf[groupEndIdx-1]) && + inputIndex < fActiveLimit && U16_IS_TRAIL(inputBuf[inputIndex])) { + // Capture group ended with an unpaired lead surrogate. + // Back reference is not permitted to match lead only of a surrogatge pair. + success = false; + } + if (success) { + fp->fInputIdx = inputIndex; + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_BACKREF_I: + { + U_ASSERT(opValue < fFrameSize); + int64_t groupStartIdx = fp->fExtra[opValue]; + int64_t groupEndIdx = fp->fExtra[opValue+1]; + U_ASSERT(groupStartIdx <= groupEndIdx); + if (groupStartIdx < 0) { + // This capture group has not participated in the match thus far, + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no match. + break; + } + CaseFoldingUCharIterator captureGroupItr(inputBuf, groupStartIdx, groupEndIdx); + CaseFoldingUCharIterator inputItr(inputBuf, fp->fInputIdx, fActiveLimit); + + // Note: if the capture group match was of an empty string the backref + // match succeeds. Verified by testing: Perl matches succeed + // in this case, so we do too. + + UBool success = true; + for (;;) { + UChar32 captureGroupChar = captureGroupItr.next(); + if (captureGroupChar == U_SENTINEL) { + success = true; + break; + } + UChar32 inputChar = inputItr.next(); + if (inputChar == U_SENTINEL) { + success = false; + fHitEnd = true; + break; + } + if (inputChar != captureGroupChar) { + success = false; + break; + } + } + + if (success && inputItr.inExpansion()) { + // We obtained a match by consuming part of a string obtained from + // case-folding a single code point of the input text. + // This does not count as an overall match. + success = false; + } + + if (success) { + fp->fInputIdx = inputItr.getIndex(); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_STO_INP_LOC: + { + U_ASSERT(opValue >= 0 && opValue < fFrameSize); + fp->fExtra[opValue] = fp->fInputIdx; + } + break; + + case URX_JMPX: + { + int32_t instrOperandLoc = (int32_t)fp->fPatIdx; + fp->fPatIdx += 1; + int32_t dataLoc = URX_VAL(pat[instrOperandLoc]); + U_ASSERT(dataLoc >= 0 && dataLoc < fFrameSize); + int32_t savedInputIdx = (int32_t)fp->fExtra[dataLoc]; + U_ASSERT(savedInputIdx <= fp->fInputIdx); + if (savedInputIdx < fp->fInputIdx) { + fp->fPatIdx = opValue; // JMP + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); // FAIL, no progress in loop. + } + } + break; + + case URX_LA_START: + { + // Entering a look around block. + // Save Stack Ptr, Input Pos. + U_ASSERT(opValue>=0 && opValue+3fDataSize); + fData[opValue] = fStack->size(); + fData[opValue+1] = fp->fInputIdx; + fData[opValue+2] = fActiveStart; + fData[opValue+3] = fActiveLimit; + fActiveStart = fLookStart; // Set the match region change for + fActiveLimit = fLookLimit; // transparent bounds. + } + break; + + case URX_LA_END: + { + // Leaving a look around block. + // restore Stack Ptr, Input Pos to positions they had on entry to block. + U_ASSERT(opValue>=0 && opValue+3fDataSize); + int32_t stackSize = fStack->size(); + int32_t newStackSize = (int32_t)fData[opValue]; + U_ASSERT(stackSize >= newStackSize); + if (stackSize > newStackSize) { + // Copy the current top frame back to the new (cut back) top frame. + // This makes the capture groups from within the look-ahead + // expression available. + int64_t *newFP = fStack->getBuffer() + newStackSize - fFrameSize; + int32_t j; + for (j=0; jsetSize(newStackSize); + } + fp->fInputIdx = fData[opValue+1]; + + // Restore the active region bounds in the input string; they may have + // been changed because of transparent bounds on a Region. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + } + break; + + case URX_ONECHAR_I: + if (fp->fInputIdx < fActiveLimit) { + UChar32 c; + U16_NEXT(inputBuf, fp->fInputIdx, fActiveLimit, c); + if (u_foldCase(c, U_FOLD_CASE_DEFAULT) == opValue) { + break; + } + } else { + fHitEnd = true; + } + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + + case URX_STRING_I: + // Case-insensitive test input against a literal string. + // Strings require two slots in the compiled pattern, one for the + // offset to the string text, and one for the length. + // The compiled string has already been case folded. + { + const char16_t *patternString = litText + opValue; + + op = (int32_t)pat[fp->fPatIdx]; + fp->fPatIdx++; + opType = URX_TYPE(op); + opValue = URX_VAL(op); + U_ASSERT(opType == URX_STRING_LEN); + int32_t patternStringLen = opValue; // Length of the string from the pattern. + + UChar32 cText; + UChar32 cPattern; + UBool success = true; + int32_t patternStringIdx = 0; + CaseFoldingUCharIterator inputIterator(inputBuf, fp->fInputIdx, fActiveLimit); + while (patternStringIdx < patternStringLen) { + U16_NEXT(patternString, patternStringIdx, patternStringLen, cPattern); + cText = inputIterator.next(); + if (cText != cPattern) { + success = false; + if (cText == U_SENTINEL) { + fHitEnd = true; + } + break; + } + } + if (inputIterator.inExpansion()) { + success = false; + } + + if (success) { + fp->fInputIdx = inputIterator.getIndex(); + } else { + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + } + break; + + case URX_LB_START: + { + // Entering a look-behind block. + // Save Stack Ptr, Input Pos and active input region. + // TODO: implement transparent bounds. Ticket #6067 + U_ASSERT(opValue>=0 && opValue+4fDataSize); + fData[opValue] = fStack->size(); + fData[opValue+1] = fp->fInputIdx; + // Save input string length, then reset to pin any matches to end at + // the current position. + fData[opValue+2] = fActiveStart; + fData[opValue+3] = fActiveLimit; + fActiveStart = fRegionStart; + fActiveLimit = fp->fInputIdx; + // Init the variable containing the start index for attempted matches. + fData[opValue+4] = -1; + } + break; + + + case URX_LB_CONT: + { + // Positive Look-Behind, at top of loop checking for matches of LB expression + // at all possible input starting positions. + + // Fetch the min and max possible match lengths. They are the operands + // of this op in the pattern. + int32_t minML = (int32_t)pat[fp->fPatIdx++]; + int32_t maxML = (int32_t)pat[fp->fPatIdx++]; + U_ASSERT(minML <= maxML); + U_ASSERT(minML >= 0); + + // Fetch (from data) the last input index where a match was attempted. + U_ASSERT(opValue>=0 && opValue+4fDataSize); + int64_t &lbStartIdx = fData[opValue+4]; + if (lbStartIdx < 0) { + // First time through loop. + lbStartIdx = fp->fInputIdx - minML; + if (lbStartIdx > 0 && lbStartIdx < fInputLength) { + U16_SET_CP_START(inputBuf, 0, lbStartIdx); + } + } else { + // 2nd through nth time through the loop. + // Back up start position for match by one. + if (lbStartIdx == 0) { + lbStartIdx--; + } else { + U16_BACK_1(inputBuf, 0, lbStartIdx); + } + } + + if (lbStartIdx < 0 || lbStartIdx < fp->fInputIdx - maxML) { + // We have tried all potential match starting points without + // getting a match. Backtrack out, and out of the + // Look Behind altogether. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + break; + } + + // Save state to this URX_LB_CONT op, so failure to match will repeat the loop. + // (successful match will fall off the end of the loop.) + fp = StateSave(fp, fp->fPatIdx-3, status); + fp->fInputIdx = lbStartIdx; + } + break; + + case URX_LB_END: + // End of a look-behind block, after a successful match. + { + U_ASSERT(opValue>=0 && opValue+4fDataSize); + if (fp->fInputIdx != fActiveLimit) { + // The look-behind expression matched, but the match did not + // extend all the way to the point that we are looking behind from. + // FAIL out of here, which will take us back to the LB_CONT, which + // will retry the match starting at another position or fail + // the look-behind altogether, whichever is appropriate. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // Look-behind match is good. Restore the original input string region, + // which had been truncated to pin the end of the lookbehind match to the + // position being looked-behind. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + } + break; + + + case URX_LBN_CONT: + { + // Negative Look-Behind, at top of loop checking for matches of LB expression + // at all possible input starting positions. + + // Fetch the extra parameters of this op. + int32_t minML = (int32_t)pat[fp->fPatIdx++]; + int32_t maxML = (int32_t)pat[fp->fPatIdx++]; + int32_t continueLoc = (int32_t)pat[fp->fPatIdx++]; + continueLoc = URX_VAL(continueLoc); + U_ASSERT(minML <= maxML); + U_ASSERT(minML >= 0); + U_ASSERT(continueLoc > fp->fPatIdx); + + // Fetch (from data) the last input index where a match was attempted. + U_ASSERT(opValue>=0 && opValue+4fDataSize); + int64_t &lbStartIdx = fData[opValue+4]; + if (lbStartIdx < 0) { + // First time through loop. + lbStartIdx = fp->fInputIdx - minML; + if (lbStartIdx > 0 && lbStartIdx < fInputLength) { + U16_SET_CP_START(inputBuf, 0, lbStartIdx); + } + } else { + // 2nd through nth time through the loop. + // Back up start position for match by one. + if (lbStartIdx == 0) { + lbStartIdx--; // Because U16_BACK is unsafe starting at 0. + } else { + U16_BACK_1(inputBuf, 0, lbStartIdx); + } + } + + if (lbStartIdx < 0 || lbStartIdx < fp->fInputIdx - maxML) { + // We have tried all potential match starting points without + // getting a match, which means that the negative lookbehind as + // a whole has succeeded. Jump forward to the continue location + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + fp->fPatIdx = continueLoc; + break; + } + + // Save state to this URX_LB_CONT op, so failure to match will repeat the loop. + // (successful match will cause a FAIL out of the loop altogether.) + fp = StateSave(fp, fp->fPatIdx-4, status); + fp->fInputIdx = lbStartIdx; + } + break; + + case URX_LBN_END: + // End of a negative look-behind block, after a successful match. + { + U_ASSERT(opValue>=0 && opValue+4fDataSize); + if (fp->fInputIdx != fActiveLimit) { + // The look-behind expression matched, but the match did not + // extend all the way to the point that we are looking behind from. + // FAIL out of here, which will take us back to the LB_CONT, which + // will retry the match starting at another position or succeed + // the look-behind altogether, whichever is appropriate. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + break; + } + + // Look-behind expression matched, which means look-behind test as + // a whole Fails + + // Restore the original input string length, which had been truncated + // inorder to pin the end of the lookbehind match + // to the position being looked-behind. + fActiveStart = fData[opValue+2]; + fActiveLimit = fData[opValue+3]; + U_ASSERT(fActiveStart >= 0); + U_ASSERT(fActiveLimit <= fInputLength); + + // Restore original stack position, discarding any state saved + // by the successful pattern match. + U_ASSERT(opValue>=0 && opValue+1fDataSize); + int32_t newStackSize = (int32_t)fData[opValue]; + U_ASSERT(fStack->size() > newStackSize); + fStack->setSize(newStackSize); + + // FAIL, which will take control back to someplace + // prior to entering the look-behind test. + fp = (REStackFrame *)fStack->popFrame(fFrameSize); + } + break; + + + case URX_LOOP_SR_I: + // Loop Initialization for the optimized implementation of + // [some character set]* + // This op scans through all matching input. + // The following LOOP_C op emulates stack unwinding if the following pattern fails. + { + U_ASSERT(opValue > 0 && opValue < fSets->size()); + Regex8BitSet *s8 = &fPattern->fSets8[opValue]; + UnicodeSet *s = (UnicodeSet *)fSets->elementAt(opValue); + + // Loop through input, until either the input is exhausted or + // we reach a character that is not a member of the set. + int32_t ix = (int32_t)fp->fInputIdx; + for (;;) { + if (ix >= fActiveLimit) { + fHitEnd = true; + break; + } + UChar32 c; + U16_NEXT(inputBuf, ix, fActiveLimit, c); + if (c<256) { + if (s8->contains(c) == false) { + U16_BACK_1(inputBuf, 0, ix); + break; + } + } else { + if (s->contains(c) == false) { + U16_BACK_1(inputBuf, 0, ix); + break; + } + } + } + + // If there were no matching characters, skip over the loop altogether. + // The loop doesn't run at all, a * op always succeeds. + if (ix == fp->fInputIdx) { + fp->fPatIdx++; // skip the URX_LOOP_C op. + break; + } + + // Peek ahead in the compiled pattern, to the URX_LOOP_C that + // must follow. It's operand is the stack location + // that holds the starting input index for the match of this [set]* + int32_t loopcOp = (int32_t)pat[fp->fPatIdx]; + U_ASSERT(URX_TYPE(loopcOp) == URX_LOOP_C); + int32_t stackLoc = URX_VAL(loopcOp); + U_ASSERT(stackLoc >= 0 && stackLoc < fFrameSize); + fp->fExtra[stackLoc] = fp->fInputIdx; + fp->fInputIdx = ix; + + // Save State to the URX_LOOP_C op that follows this one, + // so that match failures in the following code will return to there. + // Then bump the pattern idx so the LOOP_C is skipped on the way out of here. + fp = StateSave(fp, fp->fPatIdx, status); + fp->fPatIdx++; + } + break; + + + case URX_LOOP_DOT_I: + // Loop Initialization for the optimized implementation of .* + // This op scans through all remaining input. + // The following LOOP_C op emulates stack unwinding if the following pattern fails. + { + // Loop through input until the input is exhausted (we reach an end-of-line) + // In DOTALL mode, we can just go straight to the end of the input. + int32_t ix; + if ((opValue & 1) == 1) { + // Dot-matches-All mode. Jump straight to the end of the string. + ix = (int32_t)fActiveLimit; + fHitEnd = true; + } else { + // NOT DOT ALL mode. Line endings do not match '.' + // Scan forward until a line ending or end of input. + ix = (int32_t)fp->fInputIdx; + for (;;) { + if (ix >= fActiveLimit) { + fHitEnd = true; + break; + } + UChar32 c; + U16_NEXT(inputBuf, ix, fActiveLimit, c); // c = inputBuf[ix++] + if ((c & 0x7f) <= 0x29) { // Fast filter of non-new-line-s + if ((c == 0x0a) || // 0x0a is newline in both modes. + (((opValue & 2) == 0) && // IF not UNIX_LINES mode + isLineTerminator(c))) { + // char is a line ending. Put the input pos back to the + // line ending char, and exit the scanning loop. + U16_BACK_1(inputBuf, 0, ix); + break; + } + } + } + } + + // If there were no matching characters, skip over the loop altogether. + // The loop doesn't run at all, a * op always succeeds. + if (ix == fp->fInputIdx) { + fp->fPatIdx++; // skip the URX_LOOP_C op. + break; + } + + // Peek ahead in the compiled pattern, to the URX_LOOP_C that + // must follow. It's operand is the stack location + // that holds the starting input index for the match of this .* + int32_t loopcOp = (int32_t)pat[fp->fPatIdx]; + U_ASSERT(URX_TYPE(loopcOp) == URX_LOOP_C); + int32_t stackLoc = URX_VAL(loopcOp); + U_ASSERT(stackLoc >= 0 && stackLoc < fFrameSize); + fp->fExtra[stackLoc] = fp->fInputIdx; + fp->fInputIdx = ix; + + // Save State to the URX_LOOP_C op that follows this one, + // so that match failures in the following code will return to there. + // Then bump the pattern idx so the LOOP_C is skipped on the way out of here. + fp = StateSave(fp, fp->fPatIdx, status); + fp->fPatIdx++; + } + break; + + + case URX_LOOP_C: + { + U_ASSERT(opValue>=0 && opValuefExtra[opValue]; + U_ASSERT(backSearchIndex <= fp->fInputIdx); + if (backSearchIndex == fp->fInputIdx) { + // We've backed up the input idx to the point that the loop started. + // The loop is done. Leave here without saving state. + // Subsequent failures won't come back here. + break; + } + // Set up for the next iteration of the loop, with input index + // backed up by one from the last time through, + // and a state save to this instruction in case the following code fails again. + // (We're going backwards because this loop emulates stack unwinding, not + // the initial scan forward.) + U_ASSERT(fp->fInputIdx > 0); + UChar32 prevC; + U16_PREV(inputBuf, 0, fp->fInputIdx, prevC); // !!!: should this 0 be one of f*Limit? + + if (prevC == 0x0a && + fp->fInputIdx > backSearchIndex && + inputBuf[fp->fInputIdx-1] == 0x0d) { + int32_t prevOp = (int32_t)pat[fp->fPatIdx-2]; + if (URX_TYPE(prevOp) == URX_LOOP_DOT_I) { + // .*, stepping back over CRLF pair. + U16_BACK_1(inputBuf, 0, fp->fInputIdx); + } + } + + + fp = StateSave(fp, fp->fPatIdx-1, status); + } + break; + + + + default: + // Trouble. The compiled pattern contains an entry with an + // unrecognized type tag. + UPRV_UNREACHABLE_ASSERT; + // Unknown opcode type in opType = URX_TYPE(pat[fp->fPatIdx]). But we have + // reports of this in production code, don't use UPRV_UNREACHABLE_EXIT. + // See ICU-21669. + status = U_INTERNAL_PROGRAM_ERROR; + } + + if (U_FAILURE(status)) { + isMatch = false; + break; + } + } + +breakFromLoop: + fMatch = isMatch; + if (isMatch) { + fLastMatchEnd = fMatchEnd; + fMatchStart = startIdx; + fMatchEnd = fp->fInputIdx; + } + +#ifdef REGEX_RUN_DEBUG + if (fTraceDebug) { + if (isMatch) { + printf("Match. start=%ld end=%ld\n\n", fMatchStart, fMatchEnd); + } else { + printf("No match\n\n"); + } + } +#endif + + fFrame = fp; // The active stack frame when the engine stopped. + // Contains the capture group results that we need to + // access later. + + return; +} + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RegexMatcher) + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS + diff --git a/intl/icu/source/i18n/remtrans.cpp b/intl/icu/source/i18n/remtrans.cpp new file mode 100644 index 0000000000..40af1845c7 --- /dev/null +++ b/intl/icu/source/i18n/remtrans.cpp @@ -0,0 +1,71 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2001-2011, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 04/02/2001 aliu Creation. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "remtrans.h" +#include "unicode/unifilt.h" + +static const char16_t CURR_ID[] = {65, 110, 121, 45, 0x52, 0x65, 0x6D, 0x6F, 0x76, 0x65, 0x00}; /* "Any-Remove" */ + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RemoveTransliterator) + +/** + * Factory method + */ +static Transliterator* RemoveTransliterator_create(const UnicodeString& /*ID*/, + Transliterator::Token /*context*/) { + /* We don't need the ID or context. We just remove data */ + return new RemoveTransliterator(); +} + +/** + * System registration hook. + */ +void RemoveTransliterator::registerIDs() { + + Transliterator::_registerFactory(UnicodeString(true, ::CURR_ID, -1), + RemoveTransliterator_create, integerToken(0)); + + Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("Remove"), + UNICODE_STRING_SIMPLE("Null"), false); +} + +RemoveTransliterator::RemoveTransliterator() : Transliterator(UnicodeString(true, ::CURR_ID, -1), 0) {} + +RemoveTransliterator::~RemoveTransliterator() {} + +RemoveTransliterator* RemoveTransliterator::clone() const { + RemoveTransliterator* result = new RemoveTransliterator(); + if (result != nullptr && getFilter() != 0) { + result->adoptFilter(getFilter()->clone()); + } + return result; +} + +void RemoveTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index, + UBool /*isIncremental*/) const { + // Our caller (filteredTransliterate) has already narrowed us + // to an unfiltered run. Delete it. + UnicodeString empty; + text.handleReplaceBetween(index.start, index.limit, empty); + int32_t len = index.limit - index.start; + index.contextLimit -= len; + index.limit -= len; +} +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ diff --git a/intl/icu/source/i18n/remtrans.h b/intl/icu/source/i18n/remtrans.h new file mode 100644 index 0000000000..398cc5177c --- /dev/null +++ b/intl/icu/source/i18n/remtrans.h @@ -0,0 +1,80 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2001-2007, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 04/02/2001 aliu Creation. +********************************************************************** +*/ +#ifndef REMTRANS_H +#define REMTRANS_H + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/translit.h" + +U_NAMESPACE_BEGIN + +/** + * A transliterator that removes text. + * @author Alan Liu + */ +class RemoveTransliterator : public Transliterator { + +public: + + /** + * Constructs a transliterator. + */ + RemoveTransliterator(); + + /** + * Destructor. + */ + virtual ~RemoveTransliterator(); + + /** + * System registration hook. + */ + static void registerIDs(); + + /** + * Transliterator API. + * @return A copy of the object. + */ + virtual RemoveTransliterator* clone() const override; + + /** + * Implements {@link Transliterator#handleTransliterate}. + * @param text the buffer holding transliterated and + * untransliterated text + * @param offset the start and limit of the text, the position + * of the cursor, and the start and limit of transliteration. + * @param incremental if true, assume more text may be coming after + * pos.contextLimit. Otherwise, assume the text is complete. + */ + virtual void handleTransliterate(Replaceable& text, UTransPosition& offset, + UBool isIncremental) const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const override; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(); + +}; + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif diff --git a/intl/icu/source/i18n/repattrn.cpp b/intl/icu/source/i18n/repattrn.cpp new file mode 100644 index 0000000000..c0a88f70d9 --- /dev/null +++ b/intl/icu/source/i18n/repattrn.cpp @@ -0,0 +1,875 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +// +// file: repattrn.cpp +// +/* +*************************************************************************** +* Copyright (C) 2002-2016 International Business Machines Corporation +* and others. All rights reserved. +*************************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + +#include "unicode/regex.h" +#include "unicode/uclean.h" +#include "cmemory.h" +#include "cstr.h" +#include "uassert.h" +#include "uhash.h" +#include "uvector.h" +#include "uvectr32.h" +#include "uvectr64.h" +#include "regexcmp.h" +#include "regeximp.h" +#include "regexst.h" + +U_NAMESPACE_BEGIN + +//-------------------------------------------------------------------------- +// +// RegexPattern Default Constructor +// +//-------------------------------------------------------------------------- +RegexPattern::RegexPattern() { + // Init all of this instances data. + init(); +} + + +//-------------------------------------------------------------------------- +// +// Copy Constructor Note: This is a rather inefficient implementation, +// but it probably doesn't matter. +// +//-------------------------------------------------------------------------- +RegexPattern::RegexPattern(const RegexPattern &other) : UObject(other) { + init(); + *this = other; +} + + + +//-------------------------------------------------------------------------- +// +// Assignment Operator +// +//-------------------------------------------------------------------------- +RegexPattern &RegexPattern::operator = (const RegexPattern &other) { + if (this == &other) { + // Source and destination are the same. Don't do anything. + return *this; + } + + // Clean out any previous contents of object being assigned to. + zap(); + + // Give target object a default initialization + init(); + + // Copy simple fields + fDeferredStatus = other.fDeferredStatus; + + if (U_FAILURE(fDeferredStatus)) { + return *this; + } + + if (other.fPatternString == nullptr) { + fPatternString = nullptr; + fPattern = utext_clone(fPattern, other.fPattern, false, true, &fDeferredStatus); + } else { + fPatternString = new UnicodeString(*(other.fPatternString)); + if (fPatternString == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + } else { + fPattern = utext_openConstUnicodeString(nullptr, fPatternString, &fDeferredStatus); + } + } + if (U_FAILURE(fDeferredStatus)) { + return *this; + } + + fFlags = other.fFlags; + fLiteralText = other.fLiteralText; + fMinMatchLen = other.fMinMatchLen; + fFrameSize = other.fFrameSize; + fDataSize = other.fDataSize; + + fStartType = other.fStartType; + fInitialStringIdx = other.fInitialStringIdx; + fInitialStringLen = other.fInitialStringLen; + *fInitialChars = *other.fInitialChars; + fInitialChar = other.fInitialChar; + *fInitialChars8 = *other.fInitialChars8; + fNeedsAltInput = other.fNeedsAltInput; + + // Copy the pattern. It's just values, nothing deep to copy. + fCompiledPat->assign(*other.fCompiledPat, fDeferredStatus); + fGroupMap->assign(*other.fGroupMap, fDeferredStatus); + + // Copy the Unicode Sets. + // Could be made more efficient if the sets were reference counted and shared, + // but I doubt that pattern copying will be particularly common. + // Note: init() already added an empty element zero to fSets + int32_t i; + int32_t numSets = other.fSets->size(); + fSets8 = new Regex8BitSet[numSets]; + if (fSets8 == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + for (i=1; ielementAt(i); + UnicodeSet *newSet = new UnicodeSet(*sourceSet); + if (newSet == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + break; + } + fSets->addElement(newSet, fDeferredStatus); + fSets8[i] = other.fSets8[i]; + } + + // Copy the named capture group hash map. + if (other.fNamedCaptureMap != nullptr && initNamedCaptureMap()) { + int32_t hashPos = UHASH_FIRST; + while (const UHashElement *hashEl = uhash_nextElement(other.fNamedCaptureMap, &hashPos)) { + if (U_FAILURE(fDeferredStatus)) { + break; + } + const UnicodeString *name = (const UnicodeString *)hashEl->key.pointer; + UnicodeString *key = new UnicodeString(*name); + int32_t val = hashEl->value.integer; + if (key == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + } else { + uhash_puti(fNamedCaptureMap, key, val, &fDeferredStatus); + } + } + } + return *this; +} + + +//-------------------------------------------------------------------------- +// +// init Shared initialization for use by constructors. +// Bring an uninitialized RegexPattern up to a default state. +// +//-------------------------------------------------------------------------- +void RegexPattern::init() { + fFlags = 0; + fCompiledPat = 0; + fLiteralText.remove(); + fSets = nullptr; + fSets8 = nullptr; + fDeferredStatus = U_ZERO_ERROR; + fMinMatchLen = 0; + fFrameSize = 0; + fDataSize = 0; + fGroupMap = nullptr; + fStartType = START_NO_INFO; + fInitialStringIdx = 0; + fInitialStringLen = 0; + fInitialChars = nullptr; + fInitialChar = 0; + fInitialChars8 = nullptr; + fNeedsAltInput = false; + fNamedCaptureMap = nullptr; + + fPattern = nullptr; // will be set later + fPatternString = nullptr; // may be set later + fCompiledPat = new UVector64(fDeferredStatus); + fGroupMap = new UVector32(fDeferredStatus); + fSets = new UVector(fDeferredStatus); + fInitialChars = new UnicodeSet; + fInitialChars8 = new Regex8BitSet; + if (U_FAILURE(fDeferredStatus)) { + return; + } + if (fCompiledPat == nullptr || fGroupMap == nullptr || fSets == nullptr || + fInitialChars == nullptr || fInitialChars8 == nullptr) { + fDeferredStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + + // Slot zero of the vector of sets is reserved. Fill it here. + fSets->addElement((int32_t)0, fDeferredStatus); +} + + +bool RegexPattern::initNamedCaptureMap() { + if (fNamedCaptureMap) { + return true; + } + fNamedCaptureMap = uhash_openSize(uhash_hashUnicodeString, // Key hash function + uhash_compareUnicodeString, // Key comparator function + uhash_compareLong, // Value comparator function + 7, // Initial table capacity + &fDeferredStatus); + if (U_FAILURE(fDeferredStatus)) { + return false; + } + + // fNamedCaptureMap owns its key strings, type (UnicodeString *) + uhash_setKeyDeleter(fNamedCaptureMap, uprv_deleteUObject); + return true; +} + +//-------------------------------------------------------------------------- +// +// zap Delete everything owned by this RegexPattern. +// +//-------------------------------------------------------------------------- +void RegexPattern::zap() { + delete fCompiledPat; + fCompiledPat = nullptr; + int i; + for (i=1; isize(); i++) { + UnicodeSet *s; + s = (UnicodeSet *)fSets->elementAt(i); + if (s != nullptr) { + delete s; + } + } + delete fSets; + fSets = nullptr; + delete[] fSets8; + fSets8 = nullptr; + delete fGroupMap; + fGroupMap = nullptr; + delete fInitialChars; + fInitialChars = nullptr; + delete fInitialChars8; + fInitialChars8 = nullptr; + if (fPattern != nullptr) { + utext_close(fPattern); + fPattern = nullptr; + } + if (fPatternString != nullptr) { + delete fPatternString; + fPatternString = nullptr; + } + if (fNamedCaptureMap != nullptr) { + uhash_close(fNamedCaptureMap); + fNamedCaptureMap = nullptr; + } +} + + +//-------------------------------------------------------------------------- +// +// Destructor +// +//-------------------------------------------------------------------------- +RegexPattern::~RegexPattern() { + zap(); +} + + +//-------------------------------------------------------------------------- +// +// Clone +// +//-------------------------------------------------------------------------- +RegexPattern *RegexPattern::clone() const { + RegexPattern *copy = new RegexPattern(*this); + return copy; +} + + +//-------------------------------------------------------------------------- +// +// operator == (comparison) Consider to patterns to be == if the +// pattern strings and the flags are the same. +// Note that pattern strings with the same +// characters can still be considered different. +// +//-------------------------------------------------------------------------- +bool RegexPattern::operator ==(const RegexPattern &other) const { + if (this->fFlags == other.fFlags && this->fDeferredStatus == other.fDeferredStatus) { + if (this->fPatternString != nullptr && other.fPatternString != nullptr) { + return *(this->fPatternString) == *(other.fPatternString); + } else if (this->fPattern == nullptr) { + if (other.fPattern == nullptr) { + return true; + } + } else if (other.fPattern != nullptr) { + UTEXT_SETNATIVEINDEX(this->fPattern, 0); + UTEXT_SETNATIVEINDEX(other.fPattern, 0); + return utext_equals(this->fPattern, other.fPattern); + } + } + return false; +} + +//--------------------------------------------------------------------- +// +// compile +// +//--------------------------------------------------------------------- +RegexPattern * U_EXPORT2 +RegexPattern::compile(const UnicodeString ®ex, + uint32_t flags, + UParseError &pe, + UErrorCode &status) +{ + if (U_FAILURE(status)) { + return nullptr; + } + + const uint32_t allFlags = UREGEX_CANON_EQ | UREGEX_CASE_INSENSITIVE | UREGEX_COMMENTS | + UREGEX_DOTALL | UREGEX_MULTILINE | UREGEX_UWORD | + UREGEX_ERROR_ON_UNKNOWN_ESCAPES | UREGEX_UNIX_LINES | UREGEX_LITERAL; + + if ((flags & ~allFlags) != 0) { + status = U_REGEX_INVALID_FLAG; + return nullptr; + } + + if ((flags & UREGEX_CANON_EQ) != 0) { + status = U_REGEX_UNIMPLEMENTED; + return nullptr; + } + + RegexPattern *This = new RegexPattern; + if (This == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if (U_FAILURE(This->fDeferredStatus)) { + status = This->fDeferredStatus; + delete This; + return nullptr; + } + This->fFlags = flags; + + RegexCompile compiler(This, status); + compiler.compile(regex, pe, status); + + if (U_FAILURE(status)) { + delete This; + This = nullptr; + } + + return This; +} + + +// +// compile, UText mode +// +RegexPattern * U_EXPORT2 +RegexPattern::compile(UText *regex, + uint32_t flags, + UParseError &pe, + UErrorCode &status) +{ + if (U_FAILURE(status)) { + return nullptr; + } + + const uint32_t allFlags = UREGEX_CANON_EQ | UREGEX_CASE_INSENSITIVE | UREGEX_COMMENTS | + UREGEX_DOTALL | UREGEX_MULTILINE | UREGEX_UWORD | + UREGEX_ERROR_ON_UNKNOWN_ESCAPES | UREGEX_UNIX_LINES | UREGEX_LITERAL; + + if ((flags & ~allFlags) != 0) { + status = U_REGEX_INVALID_FLAG; + return nullptr; + } + + if ((flags & UREGEX_CANON_EQ) != 0) { + status = U_REGEX_UNIMPLEMENTED; + return nullptr; + } + + RegexPattern *This = new RegexPattern; + if (This == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if (U_FAILURE(This->fDeferredStatus)) { + status = This->fDeferredStatus; + delete This; + return nullptr; + } + This->fFlags = flags; + + RegexCompile compiler(This, status); + compiler.compile(regex, pe, status); + + if (U_FAILURE(status)) { + delete This; + This = nullptr; + } + + return This; +} + +// +// compile with default flags. +// +RegexPattern * U_EXPORT2 +RegexPattern::compile(const UnicodeString ®ex, + UParseError &pe, + UErrorCode &err) +{ + return compile(regex, 0, pe, err); +} + + +// +// compile with default flags, UText mode +// +RegexPattern * U_EXPORT2 +RegexPattern::compile(UText *regex, + UParseError &pe, + UErrorCode &err) +{ + return compile(regex, 0, pe, err); +} + + +// +// compile with no UParseErr parameter. +// +RegexPattern * U_EXPORT2 +RegexPattern::compile(const UnicodeString ®ex, + uint32_t flags, + UErrorCode &err) +{ + UParseError pe; + return compile(regex, flags, pe, err); +} + + +// +// compile with no UParseErr parameter, UText mode +// +RegexPattern * U_EXPORT2 +RegexPattern::compile(UText *regex, + uint32_t flags, + UErrorCode &err) +{ + UParseError pe; + return compile(regex, flags, pe, err); +} + + +//--------------------------------------------------------------------- +// +// flags +// +//--------------------------------------------------------------------- +uint32_t RegexPattern::flags() const { + return fFlags; +} + + +//--------------------------------------------------------------------- +// +// matcher(UnicodeString, err) +// +//--------------------------------------------------------------------- +RegexMatcher *RegexPattern::matcher(const UnicodeString &input, + UErrorCode &status) const { + RegexMatcher *retMatcher = matcher(status); + if (retMatcher != nullptr) { + retMatcher->fDeferredStatus = status; + retMatcher->reset(input); + } + return retMatcher; +} + + +//--------------------------------------------------------------------- +// +// matcher(status) +// +//--------------------------------------------------------------------- +RegexMatcher *RegexPattern::matcher(UErrorCode &status) const { + RegexMatcher *retMatcher = nullptr; + + if (U_FAILURE(status)) { + return nullptr; + } + if (U_FAILURE(fDeferredStatus)) { + status = fDeferredStatus; + return nullptr; + } + + retMatcher = new RegexMatcher(this); + if (retMatcher == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + return retMatcher; +} + + + +//--------------------------------------------------------------------- +// +// matches Convenience function to test for a match, starting +// with a pattern string and a data string. +// +//--------------------------------------------------------------------- +UBool U_EXPORT2 RegexPattern::matches(const UnicodeString ®ex, + const UnicodeString &input, + UParseError &pe, + UErrorCode &status) { + + if (U_FAILURE(status)) {return false;} + + UBool retVal; + RegexPattern *pat = nullptr; + RegexMatcher *matcher = nullptr; + + pat = RegexPattern::compile(regex, 0, pe, status); + matcher = pat->matcher(input, status); + retVal = matcher->matches(status); + + delete matcher; + delete pat; + return retVal; +} + + +// +// matches, UText mode +// +UBool U_EXPORT2 RegexPattern::matches(UText *regex, + UText *input, + UParseError &pe, + UErrorCode &status) { + + if (U_FAILURE(status)) {return false;} + + UBool retVal = false; + RegexPattern *pat = nullptr; + RegexMatcher *matcher = nullptr; + + pat = RegexPattern::compile(regex, 0, pe, status); + matcher = pat->matcher(status); + if (U_SUCCESS(status)) { + matcher->reset(input); + retVal = matcher->matches(status); + } + + delete matcher; + delete pat; + return retVal; +} + + + + + +//--------------------------------------------------------------------- +// +// pattern +// +//--------------------------------------------------------------------- +UnicodeString RegexPattern::pattern() const { + if (fPatternString != nullptr) { + return *fPatternString; + } else if (fPattern == nullptr) { + return UnicodeString(); + } else { + UErrorCode status = U_ZERO_ERROR; + int64_t nativeLen = utext_nativeLength(fPattern); + int32_t len16 = utext_extract(fPattern, 0, nativeLen, nullptr, 0, &status); // buffer overflow error + UnicodeString result; + + status = U_ZERO_ERROR; + char16_t *resultChars = result.getBuffer(len16); + utext_extract(fPattern, 0, nativeLen, resultChars, len16, &status); // unterminated warning + result.releaseBuffer(len16); + + return result; + } +} + + + + +//--------------------------------------------------------------------- +// +// patternText +// +//--------------------------------------------------------------------- +UText *RegexPattern::patternText(UErrorCode &status) const { + if (U_FAILURE(status)) {return nullptr;} + status = U_ZERO_ERROR; + + if (fPattern != nullptr) { + return fPattern; + } else { + RegexStaticSets::initGlobals(&status); + return RegexStaticSets::gStaticSets->fEmptyText; + } +} + + +//-------------------------------------------------------------------------------- +// +// groupNumberFromName() +// +//-------------------------------------------------------------------------------- +int32_t RegexPattern::groupNumberFromName(const UnicodeString &groupName, UErrorCode &status) const { + if (U_FAILURE(status)) { + return 0; + } + + // No need to explicitly check for syntactically valid names. + // Invalid ones will never be in the map, and the lookup will fail. + + int32_t number = fNamedCaptureMap ? uhash_geti(fNamedCaptureMap, &groupName) : 0; + if (number == 0) { + status = U_REGEX_INVALID_CAPTURE_GROUP_NAME; + } + return number; +} + +int32_t RegexPattern::groupNumberFromName(const char *groupName, int32_t nameLength, UErrorCode &status) const { + if (U_FAILURE(status)) { + return 0; + } + UnicodeString name(groupName, nameLength, US_INV); + return groupNumberFromName(name, status); +} + + +//--------------------------------------------------------------------- +// +// split +// +//--------------------------------------------------------------------- +int32_t RegexPattern::split(const UnicodeString &input, + UnicodeString dest[], + int32_t destCapacity, + UErrorCode &status) const +{ + if (U_FAILURE(status)) { + return 0; + } + + RegexMatcher m(this); + int32_t r = 0; + // Check m's status to make sure all is ok. + if (U_SUCCESS(m.fDeferredStatus)) { + r = m.split(input, dest, destCapacity, status); + } + return r; +} + +// +// split, UText mode +// +int32_t RegexPattern::split(UText *input, + UText *dest[], + int32_t destCapacity, + UErrorCode &status) const +{ + if (U_FAILURE(status)) { + return 0; + } + + RegexMatcher m(this); + int32_t r = 0; + // Check m's status to make sure all is ok. + if (U_SUCCESS(m.fDeferredStatus)) { + r = m.split(input, dest, destCapacity, status); + } + return r; +} + + +//--------------------------------------------------------------------- +// +// dump Output the compiled form of the pattern. +// Debugging function only. +// +//--------------------------------------------------------------------- +void RegexPattern::dumpOp(int32_t index) const { + (void)index; // Suppress warnings in non-debug build. +#if defined(REGEX_DEBUG) + static const char * const opNames[] = {URX_OPCODE_NAMES}; + int32_t op = fCompiledPat->elementAti(index); + int32_t val = URX_VAL(op); + int32_t type = URX_TYPE(op); + int32_t pinnedType = type; + if ((uint32_t)pinnedType >= UPRV_LENGTHOF(opNames)) { + pinnedType = 0; + } + + printf("%4d %08x %-15s ", index, op, opNames[pinnedType]); + switch (type) { + case URX_NOP: + case URX_DOTANY: + case URX_DOTANY_ALL: + case URX_FAIL: + case URX_CARET: + case URX_DOLLAR: + case URX_BACKSLASH_G: + case URX_BACKSLASH_X: + case URX_END: + case URX_DOLLAR_M: + case URX_CARET_M: + // Types with no operand field of interest. + break; + + case URX_RESERVED_OP: + case URX_START_CAPTURE: + case URX_END_CAPTURE: + case URX_STATE_SAVE: + case URX_JMP: + case URX_JMP_SAV: + case URX_JMP_SAV_X: + case URX_BACKSLASH_B: + case URX_BACKSLASH_BU: + case URX_BACKSLASH_D: + case URX_BACKSLASH_Z: + case URX_STRING_LEN: + case URX_CTR_INIT: + case URX_CTR_INIT_NG: + case URX_CTR_LOOP: + case URX_CTR_LOOP_NG: + case URX_RELOC_OPRND: + case URX_STO_SP: + case URX_LD_SP: + case URX_BACKREF: + case URX_STO_INP_LOC: + case URX_JMPX: + case URX_LA_START: + case URX_LA_END: + case URX_BACKREF_I: + case URX_LB_START: + case URX_LB_CONT: + case URX_LB_END: + case URX_LBN_CONT: + case URX_LBN_END: + case URX_LOOP_C: + case URX_LOOP_DOT_I: + case URX_BACKSLASH_H: + case URX_BACKSLASH_R: + case URX_BACKSLASH_V: + // types with an integer operand field. + printf("%d", val); + break; + + case URX_ONECHAR: + case URX_ONECHAR_I: + if (val < 0x20) { + printf("%#x", val); + } else { + printf("'%s'", CStr(UnicodeString(val))()); + } + break; + + case URX_STRING: + case URX_STRING_I: + { + int32_t lengthOp = fCompiledPat->elementAti(index+1); + U_ASSERT(URX_TYPE(lengthOp) == URX_STRING_LEN); + int32_t length = URX_VAL(lengthOp); + UnicodeString str(fLiteralText, val, length); + printf("%s", CStr(str)()); + } + break; + + case URX_SETREF: + case URX_LOOP_SR_I: + { + UnicodeString s; + UnicodeSet *set = (UnicodeSet *)fSets->elementAt(val); + set->toPattern(s, true); + printf("%s", CStr(s)()); + } + break; + + case URX_STATIC_SETREF: + case URX_STAT_SETREF_N: + { + UnicodeString s; + if (val & URX_NEG_SET) { + printf("NOT "); + val &= ~URX_NEG_SET; + } + UnicodeSet &set = RegexStaticSets::gStaticSets->fPropSets[val]; + set.toPattern(s, true); + printf("%s", CStr(s)()); + } + break; + + + default: + printf("??????"); + break; + } + printf("\n"); +#endif +} + + +void RegexPattern::dumpPattern() const { +#if defined(REGEX_DEBUG) + int index; + + UnicodeString patStr; + for (UChar32 c = utext_next32From(fPattern, 0); c != U_SENTINEL; c = utext_next32(fPattern)) { + patStr.append(c); + } + printf("Original Pattern: \"%s\"\n", CStr(patStr)()); + printf(" Min Match Length: %d\n", fMinMatchLen); + printf(" Match Start Type: %s\n", START_OF_MATCH_STR(fStartType)); + if (fStartType == START_STRING) { + UnicodeString initialString(fLiteralText,fInitialStringIdx, fInitialStringLen); + printf(" Initial match string: \"%s\"\n", CStr(initialString)()); + } else if (fStartType == START_SET) { + UnicodeString s; + fInitialChars->toPattern(s, true); + printf(" Match First Chars: %s\n", CStr(s)()); + + } else if (fStartType == START_CHAR) { + printf(" First char of Match: "); + if (fInitialChar > 0x20) { + printf("'%s'\n", CStr(UnicodeString(fInitialChar))()); + } else { + printf("%#x\n", fInitialChar); + } + } + + printf("Named Capture Groups:\n"); + if (!fNamedCaptureMap || uhash_count(fNamedCaptureMap) == 0) { + printf(" None\n"); + } else { + int32_t pos = UHASH_FIRST; + const UHashElement *el = nullptr; + while ((el = uhash_nextElement(fNamedCaptureMap, &pos))) { + const UnicodeString *name = (const UnicodeString *)el->key.pointer; + int32_t number = el->value.integer; + printf(" %d\t%s\n", number, CStr(*name)()); + } + } + + printf("\nIndex Binary Type Operand\n" \ + "-------------------------------------------\n"); + for (index = 0; indexsize(); index++) { + dumpOp(index); + } + printf("\n\n"); +#endif +} + + + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RegexPattern) + +U_NAMESPACE_END +#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS diff --git a/intl/icu/source/i18n/rulebasedcollator.cpp b/intl/icu/source/i18n/rulebasedcollator.cpp new file mode 100644 index 0000000000..e9482628d9 --- /dev/null +++ b/intl/icu/source/i18n/rulebasedcollator.cpp @@ -0,0 +1,1656 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* Copyright (C) 1996-2015, International Business Machines +* Corporation and others. All Rights Reserved. +******************************************************************************* +* rulebasedcollator.cpp +* +* (replaced the former tblcoll.cpp) +* +* created on: 2012feb14 with new and old collation code +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_COLLATION + +#include "unicode/coll.h" +#include "unicode/coleitr.h" +#include "unicode/localpointer.h" +#include "unicode/locid.h" +#include "unicode/sortkey.h" +#include "unicode/tblcoll.h" +#include "unicode/ucol.h" +#include "unicode/uiter.h" +#include "unicode/uloc.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/usetiter.h" +#include "unicode/utf8.h" +#include "unicode/uversion.h" +#include "bocsu.h" +#include "charstr.h" +#include "cmemory.h" +#include "collation.h" +#include "collationcompare.h" +#include "collationdata.h" +#include "collationdatareader.h" +#include "collationfastlatin.h" +#include "collationiterator.h" +#include "collationkeys.h" +#include "collationroot.h" +#include "collationsets.h" +#include "collationsettings.h" +#include "collationtailoring.h" +#include "cstring.h" +#include "uassert.h" +#include "ucol_imp.h" +#include "uhash.h" +#include "uitercollationiterator.h" +#include "ustr_imp.h" +#include "utf16collationiterator.h" +#include "utf8collationiterator.h" +#include "uvectr64.h" + +U_NAMESPACE_BEGIN + +namespace { + +class FixedSortKeyByteSink : public SortKeyByteSink { +public: + FixedSortKeyByteSink(char *dest, int32_t destCapacity) + : SortKeyByteSink(dest, destCapacity) {} + virtual ~FixedSortKeyByteSink(); + +private: + virtual void AppendBeyondCapacity(const char *bytes, int32_t n, int32_t length) override; + virtual UBool Resize(int32_t appendCapacity, int32_t length) override; +}; + +FixedSortKeyByteSink::~FixedSortKeyByteSink() {} + +void +FixedSortKeyByteSink::AppendBeyondCapacity(const char *bytes, int32_t /*n*/, int32_t length) { + // buffer_ != nullptr && bytes != nullptr && n > 0 && appended_ > capacity_ + // Fill the buffer completely. + int32_t available = capacity_ - length; + if (available > 0) { + uprv_memcpy(buffer_ + length, bytes, available); + } +} + +UBool +FixedSortKeyByteSink::Resize(int32_t /*appendCapacity*/, int32_t /*length*/) { + return false; +} + +} // namespace + +// Not in an anonymous namespace, so that it can be a friend of CollationKey. +class CollationKeyByteSink : public SortKeyByteSink { +public: + CollationKeyByteSink(CollationKey &key) + : SortKeyByteSink(reinterpret_cast(key.getBytes()), key.getCapacity()), + key_(key) {} + virtual ~CollationKeyByteSink(); + +private: + virtual void AppendBeyondCapacity(const char *bytes, int32_t n, int32_t length) override; + virtual UBool Resize(int32_t appendCapacity, int32_t length) override; + + CollationKey &key_; +}; + +CollationKeyByteSink::~CollationKeyByteSink() {} + +void +CollationKeyByteSink::AppendBeyondCapacity(const char *bytes, int32_t n, int32_t length) { + // buffer_ != nullptr && bytes != nullptr && n > 0 && appended_ > capacity_ + if (Resize(n, length)) { + uprv_memcpy(buffer_ + length, bytes, n); + } +} + +UBool +CollationKeyByteSink::Resize(int32_t appendCapacity, int32_t length) { + if (buffer_ == nullptr) { + return false; // allocation failed before already + } + int32_t newCapacity = 2 * capacity_; + int32_t altCapacity = length + 2 * appendCapacity; + if (newCapacity < altCapacity) { + newCapacity = altCapacity; + } + if (newCapacity < 200) { + newCapacity = 200; + } + uint8_t *newBuffer = key_.reallocate(newCapacity, length); + if (newBuffer == nullptr) { + SetNotOk(); + return false; + } + buffer_ = reinterpret_cast(newBuffer); + capacity_ = newCapacity; + return true; +} + +RuleBasedCollator::RuleBasedCollator(const RuleBasedCollator &other) + : Collator(other), + data(other.data), + settings(other.settings), + tailoring(other.tailoring), + cacheEntry(other.cacheEntry), + validLocale(other.validLocale), + explicitlySetAttributes(other.explicitlySetAttributes), + actualLocaleIsSameAsValid(other.actualLocaleIsSameAsValid) { + settings->addRef(); + cacheEntry->addRef(); +} + +RuleBasedCollator::RuleBasedCollator(const uint8_t *bin, int32_t length, + const RuleBasedCollator *base, UErrorCode &errorCode) + : data(nullptr), + settings(nullptr), + tailoring(nullptr), + cacheEntry(nullptr), + validLocale(""), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + if(U_FAILURE(errorCode)) { return; } + if(bin == nullptr || length == 0 || base == nullptr) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + const CollationTailoring *root = CollationRoot::getRoot(errorCode); + if(U_FAILURE(errorCode)) { return; } + if(base->tailoring != root) { + errorCode = U_UNSUPPORTED_ERROR; + return; + } + LocalPointer t(new CollationTailoring(base->tailoring->settings)); + if(t.isNull() || t->isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + CollationDataReader::read(base->tailoring, bin, length, *t, errorCode); + if(U_FAILURE(errorCode)) { return; } + t->actualLocale.setToBogus(); + adoptTailoring(t.orphan(), errorCode); +} + +RuleBasedCollator::RuleBasedCollator(const CollationCacheEntry *entry) + : data(entry->tailoring->data), + settings(entry->tailoring->settings), + tailoring(entry->tailoring), + cacheEntry(entry), + validLocale(entry->validLocale), + explicitlySetAttributes(0), + actualLocaleIsSameAsValid(false) { + settings->addRef(); + cacheEntry->addRef(); +} + +RuleBasedCollator::~RuleBasedCollator() { + SharedObject::clearPtr(settings); + SharedObject::clearPtr(cacheEntry); +} + +void +RuleBasedCollator::adoptTailoring(CollationTailoring *t, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { + t->deleteIfZeroRefCount(); + return; + } + U_ASSERT(settings == nullptr && data == nullptr && tailoring == nullptr && cacheEntry == nullptr); + cacheEntry = new CollationCacheEntry(t->actualLocale, t); + if(cacheEntry == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + t->deleteIfZeroRefCount(); + return; + } + data = t->data; + settings = t->settings; + settings->addRef(); + tailoring = t; + cacheEntry->addRef(); + validLocale = t->actualLocale; + actualLocaleIsSameAsValid = false; +} + +RuleBasedCollator * +RuleBasedCollator::clone() const { + return new RuleBasedCollator(*this); +} + +RuleBasedCollator &RuleBasedCollator::operator=(const RuleBasedCollator &other) { + if(this == &other) { return *this; } + SharedObject::copyPtr(other.settings, settings); + tailoring = other.tailoring; + SharedObject::copyPtr(other.cacheEntry, cacheEntry); + data = tailoring->data; + validLocale = other.validLocale; + explicitlySetAttributes = other.explicitlySetAttributes; + actualLocaleIsSameAsValid = other.actualLocaleIsSameAsValid; + return *this; +} + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedCollator) + +bool +RuleBasedCollator::operator==(const Collator& other) const { + if(this == &other) { return true; } + if(!Collator::operator==(other)) { return false; } + const RuleBasedCollator &o = static_cast(other); + if(*settings != *o.settings) { return false; } + if(data == o.data) { return true; } + UBool thisIsRoot = data->base == nullptr; + UBool otherIsRoot = o.data->base == nullptr; + U_ASSERT(!thisIsRoot || !otherIsRoot); // otherwise their data pointers should be == + if(thisIsRoot != otherIsRoot) { return false; } + if((thisIsRoot || !tailoring->rules.isEmpty()) && + (otherIsRoot || !o.tailoring->rules.isEmpty())) { + // Shortcut: If both collators have valid rule strings, then compare those. + if(tailoring->rules == o.tailoring->rules) { return true; } + } + // Different rule strings can result in the same or equivalent tailoring. + // The rule strings are optional in ICU resource bundles, although included by default. + // cloneBinary() drops the rule string. + UErrorCode errorCode = U_ZERO_ERROR; + LocalPointer thisTailored(getTailoredSet(errorCode)); + LocalPointer otherTailored(o.getTailoredSet(errorCode)); + if(U_FAILURE(errorCode)) { return false; } + if(*thisTailored != *otherTailored) { return false; } + // For completeness, we should compare all of the mappings; + // or we should create a list of strings, sort it with one collator, + // and check if both collators compare adjacent strings the same + // (order & strength, down to quaternary); or similar. + // Testing equality of collators seems unusual. + return true; +} + +int32_t +RuleBasedCollator::hashCode() const { + int32_t h = settings->hashCode(); + if(data->base == nullptr) { return h; } // root collator + // Do not rely on the rule string, see comments in operator==(). + UErrorCode errorCode = U_ZERO_ERROR; + LocalPointer set(getTailoredSet(errorCode)); + if(U_FAILURE(errorCode)) { return 0; } + UnicodeSetIterator iter(*set); + while(iter.next() && !iter.isString()) { + h ^= data->getCE32(iter.getCodepoint()); + } + return h; +} + +void +RuleBasedCollator::setLocales(const Locale &requested, const Locale &valid, + const Locale &actual) { + if(actual == tailoring->actualLocale) { + actualLocaleIsSameAsValid = false; + } else { + U_ASSERT(actual == valid); + actualLocaleIsSameAsValid = true; + } + // Do not modify tailoring.actualLocale: + // We cannot be sure that that would be thread-safe. + validLocale = valid; + (void)requested; // Ignore, see also ticket #10477. +} + +Locale +RuleBasedCollator::getLocale(ULocDataLocaleType type, UErrorCode& errorCode) const { + if(U_FAILURE(errorCode)) { + return Locale::getRoot(); + } + switch(type) { + case ULOC_ACTUAL_LOCALE: + return actualLocaleIsSameAsValid ? validLocale : tailoring->actualLocale; + case ULOC_VALID_LOCALE: + return validLocale; + case ULOC_REQUESTED_LOCALE: + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return Locale::getRoot(); + } +} + +const char * +RuleBasedCollator::internalGetLocaleID(ULocDataLocaleType type, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { + return nullptr; + } + const Locale *result; + switch(type) { + case ULOC_ACTUAL_LOCALE: + result = actualLocaleIsSameAsValid ? &validLocale : &tailoring->actualLocale; + break; + case ULOC_VALID_LOCALE: + result = &validLocale; + break; + case ULOC_REQUESTED_LOCALE: + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return nullptr; + } + if(result->isBogus()) { return nullptr; } + const char *id = result->getName(); + return id[0] == 0 ? "root" : id; +} + +const UnicodeString& +RuleBasedCollator::getRules() const { + return tailoring->rules; +} + +void +RuleBasedCollator::getRules(UColRuleOption delta, UnicodeString &buffer) const { + if(delta == UCOL_TAILORING_ONLY) { + buffer = tailoring->rules; + return; + } + // UCOL_FULL_RULES + buffer.remove(); + CollationLoader::appendRootRules(buffer); + buffer.append(tailoring->rules).getTerminatedBuffer(); +} + +void +RuleBasedCollator::getVersion(UVersionInfo version) const { + uprv_memcpy(version, tailoring->version, U_MAX_VERSION_LENGTH); + version[0] += (UCOL_RUNTIME_VERSION << 4) + (UCOL_RUNTIME_VERSION >> 4); +} + +UnicodeSet * +RuleBasedCollator::getTailoredSet(UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return nullptr; } + UnicodeSet *tailored = new UnicodeSet(); + if(tailored == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return nullptr; + } + if(data->base != nullptr) { + TailoredSet(tailored).forData(data, errorCode); + if(U_FAILURE(errorCode)) { + delete tailored; + return nullptr; + } + } + return tailored; +} + +void +RuleBasedCollator::internalGetContractionsAndExpansions( + UnicodeSet *contractions, UnicodeSet *expansions, + UBool addPrefixes, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return; } + if(contractions != nullptr) { + contractions->clear(); + } + if(expansions != nullptr) { + expansions->clear(); + } + ContractionsAndExpansions(contractions, expansions, nullptr, addPrefixes).forData(data, errorCode); +} + +void +RuleBasedCollator::internalAddContractions(UChar32 c, UnicodeSet &set, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return; } + ContractionsAndExpansions(&set, nullptr, nullptr, false).forCodePoint(data, c, errorCode); +} + +const CollationSettings & +RuleBasedCollator::getDefaultSettings() const { + return *tailoring->settings; +} + +UColAttributeValue +RuleBasedCollator::getAttribute(UColAttribute attr, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; } + int32_t option; + switch(attr) { + case UCOL_FRENCH_COLLATION: + option = CollationSettings::BACKWARD_SECONDARY; + break; + case UCOL_ALTERNATE_HANDLING: + return settings->getAlternateHandling(); + case UCOL_CASE_FIRST: + return settings->getCaseFirst(); + case UCOL_CASE_LEVEL: + option = CollationSettings::CASE_LEVEL; + break; + case UCOL_NORMALIZATION_MODE: + option = CollationSettings::CHECK_FCD; + break; + case UCOL_STRENGTH: + return (UColAttributeValue)settings->getStrength(); + case UCOL_HIRAGANA_QUATERNARY_MODE: + // Deprecated attribute, unsettable. + return UCOL_OFF; + case UCOL_NUMERIC_COLLATION: + option = CollationSettings::NUMERIC; + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_DEFAULT; + } + return ((settings->options & option) == 0) ? UCOL_OFF : UCOL_ON; +} + +void +RuleBasedCollator::setAttribute(UColAttribute attr, UColAttributeValue value, + UErrorCode &errorCode) { + UColAttributeValue oldValue = getAttribute(attr, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(value == oldValue) { + setAttributeExplicitly(attr); + return; + } + const CollationSettings &defaultSettings = getDefaultSettings(); + if(settings == &defaultSettings) { + if(value == UCOL_DEFAULT) { + setAttributeDefault(attr); + return; + } + } + CollationSettings *ownedSettings = SharedObject::copyOnWrite(settings); + if(ownedSettings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + + switch(attr) { + case UCOL_FRENCH_COLLATION: + ownedSettings->setFlag(CollationSettings::BACKWARD_SECONDARY, value, + defaultSettings.options, errorCode); + break; + case UCOL_ALTERNATE_HANDLING: + ownedSettings->setAlternateHandling(value, defaultSettings.options, errorCode); + break; + case UCOL_CASE_FIRST: + ownedSettings->setCaseFirst(value, defaultSettings.options, errorCode); + break; + case UCOL_CASE_LEVEL: + ownedSettings->setFlag(CollationSettings::CASE_LEVEL, value, + defaultSettings.options, errorCode); + break; + case UCOL_NORMALIZATION_MODE: + ownedSettings->setFlag(CollationSettings::CHECK_FCD, value, + defaultSettings.options, errorCode); + break; + case UCOL_STRENGTH: + ownedSettings->setStrength(value, defaultSettings.options, errorCode); + break; + case UCOL_HIRAGANA_QUATERNARY_MODE: + // Deprecated attribute. Check for valid values but do not change anything. + if(value != UCOL_OFF && value != UCOL_ON && value != UCOL_DEFAULT) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + } + break; + case UCOL_NUMERIC_COLLATION: + ownedSettings->setFlag(CollationSettings::NUMERIC, value, defaultSettings.options, errorCode); + break; + default: + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + break; + } + if(U_FAILURE(errorCode)) { return; } + setFastLatinOptions(*ownedSettings); + if(value == UCOL_DEFAULT) { + setAttributeDefault(attr); + } else { + setAttributeExplicitly(attr); + } +} + +Collator & +RuleBasedCollator::setMaxVariable(UColReorderCode group, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return *this; } + // Convert the reorder code into a MaxVariable number, or UCOL_DEFAULT=-1. + int32_t value; + if(group == UCOL_REORDER_CODE_DEFAULT) { + value = UCOL_DEFAULT; + } else if(UCOL_REORDER_CODE_FIRST <= group && group <= UCOL_REORDER_CODE_CURRENCY) { + value = group - UCOL_REORDER_CODE_FIRST; + } else { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + CollationSettings::MaxVariable oldValue = settings->getMaxVariable(); + if(value == oldValue) { + setAttributeExplicitly(ATTR_VARIABLE_TOP); + return *this; + } + const CollationSettings &defaultSettings = getDefaultSettings(); + if(settings == &defaultSettings) { + if(value == UCOL_DEFAULT) { + setAttributeDefault(ATTR_VARIABLE_TOP); + return *this; + } + } + CollationSettings *ownedSettings = SharedObject::copyOnWrite(settings); + if(ownedSettings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return *this; + } + + if(group == UCOL_REORDER_CODE_DEFAULT) { + group = (UColReorderCode)( + UCOL_REORDER_CODE_FIRST + int32_t{defaultSettings.getMaxVariable()}); + } + uint32_t varTop = data->getLastPrimaryForGroup(group); + U_ASSERT(varTop != 0); + ownedSettings->setMaxVariable(value, defaultSettings.options, errorCode); + if(U_FAILURE(errorCode)) { return *this; } + ownedSettings->variableTop = varTop; + setFastLatinOptions(*ownedSettings); + if(value == UCOL_DEFAULT) { + setAttributeDefault(ATTR_VARIABLE_TOP); + } else { + setAttributeExplicitly(ATTR_VARIABLE_TOP); + } + return *this; +} + +UColReorderCode +RuleBasedCollator::getMaxVariable() const { + return (UColReorderCode)(UCOL_REORDER_CODE_FIRST + int32_t{settings->getMaxVariable()}); +} + +uint32_t +RuleBasedCollator::getVariableTop(UErrorCode & /*errorCode*/) const { + return settings->variableTop; +} + +uint32_t +RuleBasedCollator::setVariableTop(const char16_t *varTop, int32_t len, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return 0; } + if(varTop == nullptr && len !=0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(len < 0) { len = u_strlen(varTop); } + if(len == 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UBool numeric = settings->isNumeric(); + int64_t ce1, ce2; + if(settings->dontCheckFCD()) { + UTF16CollationIterator ci(data, numeric, varTop, varTop, varTop + len); + ce1 = ci.nextCE(errorCode); + ce2 = ci.nextCE(errorCode); + } else { + FCDUTF16CollationIterator ci(data, numeric, varTop, varTop, varTop + len); + ce1 = ci.nextCE(errorCode); + ce2 = ci.nextCE(errorCode); + } + if(ce1 == Collation::NO_CE || ce2 != Collation::NO_CE) { + errorCode = U_CE_NOT_FOUND_ERROR; + return 0; + } + setVariableTop((uint32_t)(ce1 >> 32), errorCode); + return settings->variableTop; +} + +uint32_t +RuleBasedCollator::setVariableTop(const UnicodeString &varTop, UErrorCode &errorCode) { + return setVariableTop(varTop.getBuffer(), varTop.length(), errorCode); +} + +void +RuleBasedCollator::setVariableTop(uint32_t varTop, UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(varTop != settings->variableTop) { + // Pin the variable top to the end of the reordering group which contains it. + // Only a few special groups are supported. + int32_t group = data->getGroupForPrimary(varTop); + if(group < UCOL_REORDER_CODE_FIRST || UCOL_REORDER_CODE_CURRENCY < group) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + uint32_t v = data->getLastPrimaryForGroup(group); + U_ASSERT(v != 0 && v >= varTop); + varTop = v; + if(varTop != settings->variableTop) { + CollationSettings *ownedSettings = SharedObject::copyOnWrite(settings); + if(ownedSettings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + ownedSettings->setMaxVariable(group - UCOL_REORDER_CODE_FIRST, + getDefaultSettings().options, errorCode); + if(U_FAILURE(errorCode)) { return; } + ownedSettings->variableTop = varTop; + setFastLatinOptions(*ownedSettings); + } + } + if(varTop == getDefaultSettings().variableTop) { + setAttributeDefault(ATTR_VARIABLE_TOP); + } else { + setAttributeExplicitly(ATTR_VARIABLE_TOP); + } +} + +int32_t +RuleBasedCollator::getReorderCodes(int32_t *dest, int32_t capacity, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + if(capacity < 0 || (dest == nullptr && capacity > 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + int32_t length = settings->reorderCodesLength; + if(length == 0) { return 0; } + if(length > capacity) { + errorCode = U_BUFFER_OVERFLOW_ERROR; + return length; + } + uprv_memcpy(dest, settings->reorderCodes, length * 4); + return length; +} + +void +RuleBasedCollator::setReorderCodes(const int32_t *reorderCodes, int32_t length, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(length < 0 || (reorderCodes == nullptr && length > 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + if(length == 1 && reorderCodes[0] == UCOL_REORDER_CODE_NONE) { + length = 0; + } + if(length == settings->reorderCodesLength && + uprv_memcmp(reorderCodes, settings->reorderCodes, length * 4) == 0) { + return; + } + const CollationSettings &defaultSettings = getDefaultSettings(); + if(length == 1 && reorderCodes[0] == UCOL_REORDER_CODE_DEFAULT) { + if(settings != &defaultSettings) { + CollationSettings *ownedSettings = SharedObject::copyOnWrite(settings); + if(ownedSettings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + ownedSettings->copyReorderingFrom(defaultSettings, errorCode); + setFastLatinOptions(*ownedSettings); + } + return; + } + CollationSettings *ownedSettings = SharedObject::copyOnWrite(settings); + if(ownedSettings == nullptr) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return; + } + ownedSettings->setReordering(*data, reorderCodes, length, errorCode); + setFastLatinOptions(*ownedSettings); +} + +void +RuleBasedCollator::setFastLatinOptions(CollationSettings &ownedSettings) const { + ownedSettings.fastLatinOptions = CollationFastLatin::getOptions( + data, ownedSettings, + ownedSettings.fastLatinPrimaries, UPRV_LENGTHOF(ownedSettings.fastLatinPrimaries)); +} + +UCollationResult +RuleBasedCollator::compare(const UnicodeString &left, const UnicodeString &right, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + return doCompare(left.getBuffer(), left.length(), + right.getBuffer(), right.length(), errorCode); +} + +UCollationResult +RuleBasedCollator::compare(const UnicodeString &left, const UnicodeString &right, + int32_t length, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode) || length == 0) { return UCOL_EQUAL; } + if(length < 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_EQUAL; + } + int32_t leftLength = left.length(); + int32_t rightLength = right.length(); + if(leftLength > length) { leftLength = length; } + if(rightLength > length) { rightLength = length; } + return doCompare(left.getBuffer(), leftLength, + right.getBuffer(), rightLength, errorCode); +} + +UCollationResult +RuleBasedCollator::compare(const char16_t *left, int32_t leftLength, + const char16_t *right, int32_t rightLength, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + if((left == nullptr && leftLength != 0) || (right == nullptr && rightLength != 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_EQUAL; + } + // Make sure both or neither strings have a known length. + // We do not optimize for mixed length/termination. + if(leftLength >= 0) { + if(rightLength < 0) { rightLength = u_strlen(right); } + } else { + if(rightLength >= 0) { leftLength = u_strlen(left); } + } + return doCompare(left, leftLength, right, rightLength, errorCode); +} + +UCollationResult +RuleBasedCollator::compareUTF8(const StringPiece &left, const StringPiece &right, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + const uint8_t *leftBytes = reinterpret_cast(left.data()); + const uint8_t *rightBytes = reinterpret_cast(right.data()); + if((leftBytes == nullptr && !left.empty()) || (rightBytes == nullptr && !right.empty())) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_EQUAL; + } + return doCompare(leftBytes, left.length(), rightBytes, right.length(), errorCode); +} + +UCollationResult +RuleBasedCollator::internalCompareUTF8(const char *left, int32_t leftLength, + const char *right, int32_t rightLength, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return UCOL_EQUAL; } + if((left == nullptr && leftLength != 0) || (right == nullptr && rightLength != 0)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return UCOL_EQUAL; + } + // Make sure both or neither strings have a known length. + // We do not optimize for mixed length/termination. + if(leftLength >= 0) { + if(rightLength < 0) { rightLength = static_cast(uprv_strlen(right)); } + } else { + if(rightLength >= 0) { leftLength = static_cast(uprv_strlen(left)); } + } + return doCompare(reinterpret_cast(left), leftLength, + reinterpret_cast(right), rightLength, errorCode); +} + +namespace { + +/** + * Abstract iterator for identical-level string comparisons. + * Returns FCD code points and handles temporary switching to NFD. + */ +class NFDIterator : public UObject { +public: + NFDIterator() : index(-1), length(0) {} + virtual ~NFDIterator() {} + /** + * Returns the next code point from the internal normalization buffer, + * or else the next text code point. + * Returns -1 at the end of the text. + */ + UChar32 nextCodePoint() { + if(index >= 0) { + if(index == length) { + index = -1; + } else { + UChar32 c; + U16_NEXT_UNSAFE(decomp, index, c); + return c; + } + } + return nextRawCodePoint(); + } + /** + * @param nfcImpl + * @param c the last code point returned by nextCodePoint() or nextDecomposedCodePoint() + * @return the first code point in c's decomposition, + * or c itself if it was decomposed already or if it does not decompose + */ + UChar32 nextDecomposedCodePoint(const Normalizer2Impl &nfcImpl, UChar32 c) { + if(index >= 0) { return c; } + decomp = nfcImpl.getDecomposition(c, buffer, length); + if(decomp == nullptr) { return c; } + index = 0; + U16_NEXT_UNSAFE(decomp, index, c); + return c; + } +protected: + /** + * Returns the next text code point in FCD order. + * Returns -1 at the end of the text. + */ + virtual UChar32 nextRawCodePoint() = 0; +private: + const char16_t *decomp; + char16_t buffer[4]; + int32_t index; + int32_t length; +}; + +class UTF16NFDIterator : public NFDIterator { +public: + UTF16NFDIterator(const char16_t *text, const char16_t *textLimit) : s(text), limit(textLimit) {} +protected: + virtual UChar32 nextRawCodePoint() override { + if(s == limit) { return U_SENTINEL; } + UChar32 c = *s++; + if(limit == nullptr && c == 0) { + s = nullptr; + return U_SENTINEL; + } + char16_t trail; + if(U16_IS_LEAD(c) && s != limit && U16_IS_TRAIL(trail = *s)) { + ++s; + c = U16_GET_SUPPLEMENTARY(c, trail); + } + return c; + } + + const char16_t *s; + const char16_t *limit; +}; + +class FCDUTF16NFDIterator : public UTF16NFDIterator { +public: + FCDUTF16NFDIterator(const Normalizer2Impl &nfcImpl, const char16_t *text, const char16_t *textLimit) + : UTF16NFDIterator(nullptr, nullptr) { + UErrorCode errorCode = U_ZERO_ERROR; + const char16_t *spanLimit = nfcImpl.makeFCD(text, textLimit, nullptr, errorCode); + if(U_FAILURE(errorCode)) { return; } + if(spanLimit == textLimit || (textLimit == nullptr && *spanLimit == 0)) { + s = text; + limit = spanLimit; + } else { + str.setTo(text, (int32_t)(spanLimit - text)); + { + ReorderingBuffer r_buffer(nfcImpl, str); + if(r_buffer.init(str.length(), errorCode)) { + nfcImpl.makeFCD(spanLimit, textLimit, &r_buffer, errorCode); + } + } + if(U_SUCCESS(errorCode)) { + s = str.getBuffer(); + limit = s + str.length(); + } + } + } +private: + UnicodeString str; +}; + +class UTF8NFDIterator : public NFDIterator { +public: + UTF8NFDIterator(const uint8_t *text, int32_t textLength) + : s(text), pos(0), length(textLength) {} +protected: + virtual UChar32 nextRawCodePoint() override { + if(pos == length || (s[pos] == 0 && length < 0)) { return U_SENTINEL; } + UChar32 c; + U8_NEXT_OR_FFFD(s, pos, length, c); + return c; + } + + const uint8_t *s; + int32_t pos; + int32_t length; +}; + +class FCDUTF8NFDIterator : public NFDIterator { +public: + FCDUTF8NFDIterator(const CollationData *data, const uint8_t *text, int32_t textLength) + : u8ci(data, false, text, 0, textLength) {} +protected: + virtual UChar32 nextRawCodePoint() override { + UErrorCode errorCode = U_ZERO_ERROR; + return u8ci.nextCodePoint(errorCode); + } +private: + FCDUTF8CollationIterator u8ci; +}; + +class UIterNFDIterator : public NFDIterator { +public: + UIterNFDIterator(UCharIterator &it) : iter(it) {} +protected: + virtual UChar32 nextRawCodePoint() override { + return uiter_next32(&iter); + } +private: + UCharIterator &iter; +}; + +class FCDUIterNFDIterator : public NFDIterator { +public: + FCDUIterNFDIterator(const CollationData *data, UCharIterator &it, int32_t startIndex) + : uici(data, false, it, startIndex) {} +protected: + virtual UChar32 nextRawCodePoint() override { + UErrorCode errorCode = U_ZERO_ERROR; + return uici.nextCodePoint(errorCode); + } +private: + FCDUIterCollationIterator uici; +}; + +UCollationResult compareNFDIter(const Normalizer2Impl &nfcImpl, + NFDIterator &left, NFDIterator &right) { + for(;;) { + // Fetch the next FCD code point from each string. + UChar32 leftCp = left.nextCodePoint(); + UChar32 rightCp = right.nextCodePoint(); + if(leftCp == rightCp) { + if(leftCp < 0) { break; } + continue; + } + // If they are different, then decompose each and compare again. + if(leftCp < 0) { + leftCp = -2; // end of string + } else if(leftCp == 0xfffe) { + leftCp = -1; // U+FFFE: merge separator + } else { + leftCp = left.nextDecomposedCodePoint(nfcImpl, leftCp); + } + if(rightCp < 0) { + rightCp = -2; // end of string + } else if(rightCp == 0xfffe) { + rightCp = -1; // U+FFFE: merge separator + } else { + rightCp = right.nextDecomposedCodePoint(nfcImpl, rightCp); + } + if(leftCp < rightCp) { return UCOL_LESS; } + if(leftCp > rightCp) { return UCOL_GREATER; } + } + return UCOL_EQUAL; +} + +} // namespace + +UCollationResult +RuleBasedCollator::doCompare(const char16_t *left, int32_t leftLength, + const char16_t *right, int32_t rightLength, + UErrorCode &errorCode) const { + // U_FAILURE(errorCode) checked by caller. + if(left == right && leftLength == rightLength) { + return UCOL_EQUAL; + } + + // Identical-prefix test. + const char16_t *leftLimit; + const char16_t *rightLimit; + int32_t equalPrefixLength = 0; + if(leftLength < 0) { + leftLimit = nullptr; + rightLimit = nullptr; + char16_t c; + while((c = left[equalPrefixLength]) == right[equalPrefixLength]) { + if(c == 0) { return UCOL_EQUAL; } + ++equalPrefixLength; + } + } else { + leftLimit = left + leftLength; + rightLimit = right + rightLength; + for(;;) { + if(equalPrefixLength == leftLength) { + if(equalPrefixLength == rightLength) { return UCOL_EQUAL; } + break; + } else if(equalPrefixLength == rightLength || + left[equalPrefixLength] != right[equalPrefixLength]) { + break; + } + ++equalPrefixLength; + } + } + + UBool numeric = settings->isNumeric(); + if(equalPrefixLength > 0) { + if((equalPrefixLength != leftLength && + data->isUnsafeBackward(left[equalPrefixLength], numeric)) || + (equalPrefixLength != rightLength && + data->isUnsafeBackward(right[equalPrefixLength], numeric))) { + // Identical prefix: Back up to the start of a contraction or reordering sequence. + while(--equalPrefixLength > 0 && + data->isUnsafeBackward(left[equalPrefixLength], numeric)) {} + } + // Notes: + // - A longer string can compare equal to a prefix of it if only ignorables follow. + // - With a backward level, a longer string can compare less-than a prefix of it. + + // Pass the actual start of each string into the CollationIterators, + // plus the equalPrefixLength position, + // so that prefix matches back into the equal prefix work. + } + + int32_t result; + int32_t fastLatinOptions = settings->fastLatinOptions; + if(fastLatinOptions >= 0 && + (equalPrefixLength == leftLength || + left[equalPrefixLength] <= CollationFastLatin::LATIN_MAX) && + (equalPrefixLength == rightLength || + right[equalPrefixLength] <= CollationFastLatin::LATIN_MAX)) { + if(leftLength >= 0) { + result = CollationFastLatin::compareUTF16(data->fastLatinTable, + settings->fastLatinPrimaries, + fastLatinOptions, + left + equalPrefixLength, + leftLength - equalPrefixLength, + right + equalPrefixLength, + rightLength - equalPrefixLength); + } else { + result = CollationFastLatin::compareUTF16(data->fastLatinTable, + settings->fastLatinPrimaries, + fastLatinOptions, + left + equalPrefixLength, -1, + right + equalPrefixLength, -1); + } + } else { + result = CollationFastLatin::BAIL_OUT_RESULT; + } + + if(result == CollationFastLatin::BAIL_OUT_RESULT) { + if(settings->dontCheckFCD()) { + UTF16CollationIterator leftIter(data, numeric, + left, left + equalPrefixLength, leftLimit); + UTF16CollationIterator rightIter(data, numeric, + right, right + equalPrefixLength, rightLimit); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } else { + FCDUTF16CollationIterator leftIter(data, numeric, + left, left + equalPrefixLength, leftLimit); + FCDUTF16CollationIterator rightIter(data, numeric, + right, right + equalPrefixLength, rightLimit); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } + } + if(result != UCOL_EQUAL || settings->getStrength() < UCOL_IDENTICAL || U_FAILURE(errorCode)) { + return (UCollationResult)result; + } + + // Note: If NUL-terminated, we could get the actual limits from the iterators now. + // That would complicate the iterators a bit, NUL-terminated strings are only a C convenience, + // and the benefit seems unlikely to be measurable. + + // Compare identical level. + const Normalizer2Impl &nfcImpl = data->nfcImpl; + left += equalPrefixLength; + right += equalPrefixLength; + if(settings->dontCheckFCD()) { + UTF16NFDIterator leftIter(left, leftLimit); + UTF16NFDIterator rightIter(right, rightLimit); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } else { + FCDUTF16NFDIterator leftIter(nfcImpl, left, leftLimit); + FCDUTF16NFDIterator rightIter(nfcImpl, right, rightLimit); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } +} + +UCollationResult +RuleBasedCollator::doCompare(const uint8_t *left, int32_t leftLength, + const uint8_t *right, int32_t rightLength, + UErrorCode &errorCode) const { + // U_FAILURE(errorCode) checked by caller. + if(left == right && leftLength == rightLength) { + return UCOL_EQUAL; + } + + // Identical-prefix test. + int32_t equalPrefixLength = 0; + if(leftLength < 0) { + uint8_t c; + while((c = left[equalPrefixLength]) == right[equalPrefixLength]) { + if(c == 0) { return UCOL_EQUAL; } + ++equalPrefixLength; + } + } else { + for(;;) { + if(equalPrefixLength == leftLength) { + if(equalPrefixLength == rightLength) { return UCOL_EQUAL; } + break; + } else if(equalPrefixLength == rightLength || + left[equalPrefixLength] != right[equalPrefixLength]) { + break; + } + ++equalPrefixLength; + } + } + // Back up to the start of a partially-equal code point. + if(equalPrefixLength > 0 && + ((equalPrefixLength != leftLength && U8_IS_TRAIL(left[equalPrefixLength])) || + (equalPrefixLength != rightLength && U8_IS_TRAIL(right[equalPrefixLength])))) { + while(--equalPrefixLength > 0 && U8_IS_TRAIL(left[equalPrefixLength])) {} + } + + UBool numeric = settings->isNumeric(); + if(equalPrefixLength > 0) { + UBool unsafe = false; + if(equalPrefixLength != leftLength) { + int32_t i = equalPrefixLength; + UChar32 c; + U8_NEXT_OR_FFFD(left, i, leftLength, c); + unsafe = data->isUnsafeBackward(c, numeric); + } + if(!unsafe && equalPrefixLength != rightLength) { + int32_t i = equalPrefixLength; + UChar32 c; + U8_NEXT_OR_FFFD(right, i, rightLength, c); + unsafe = data->isUnsafeBackward(c, numeric); + } + if(unsafe) { + // Identical prefix: Back up to the start of a contraction or reordering sequence. + UChar32 c; + do { + U8_PREV_OR_FFFD(left, 0, equalPrefixLength, c); + } while(equalPrefixLength > 0 && data->isUnsafeBackward(c, numeric)); + } + // See the notes in the UTF-16 version. + + // Pass the actual start of each string into the CollationIterators, + // plus the equalPrefixLength position, + // so that prefix matches back into the equal prefix work. + } + + int32_t result; + int32_t fastLatinOptions = settings->fastLatinOptions; + if(fastLatinOptions >= 0 && + (equalPrefixLength == leftLength || + left[equalPrefixLength] <= CollationFastLatin::LATIN_MAX_UTF8_LEAD) && + (equalPrefixLength == rightLength || + right[equalPrefixLength] <= CollationFastLatin::LATIN_MAX_UTF8_LEAD)) { + if(leftLength >= 0) { + result = CollationFastLatin::compareUTF8(data->fastLatinTable, + settings->fastLatinPrimaries, + fastLatinOptions, + left + equalPrefixLength, + leftLength - equalPrefixLength, + right + equalPrefixLength, + rightLength - equalPrefixLength); + } else { + result = CollationFastLatin::compareUTF8(data->fastLatinTable, + settings->fastLatinPrimaries, + fastLatinOptions, + left + equalPrefixLength, -1, + right + equalPrefixLength, -1); + } + } else { + result = CollationFastLatin::BAIL_OUT_RESULT; + } + + if(result == CollationFastLatin::BAIL_OUT_RESULT) { + if(settings->dontCheckFCD()) { + UTF8CollationIterator leftIter(data, numeric, left, equalPrefixLength, leftLength); + UTF8CollationIterator rightIter(data, numeric, right, equalPrefixLength, rightLength); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } else { + FCDUTF8CollationIterator leftIter(data, numeric, left, equalPrefixLength, leftLength); + FCDUTF8CollationIterator rightIter(data, numeric, right, equalPrefixLength, rightLength); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } + } + if(result != UCOL_EQUAL || settings->getStrength() < UCOL_IDENTICAL || U_FAILURE(errorCode)) { + return (UCollationResult)result; + } + + // Note: If NUL-terminated, we could get the actual limits from the iterators now. + // That would complicate the iterators a bit, NUL-terminated strings are only a C convenience, + // and the benefit seems unlikely to be measurable. + + // Compare identical level. + const Normalizer2Impl &nfcImpl = data->nfcImpl; + left += equalPrefixLength; + right += equalPrefixLength; + if(leftLength > 0) { + leftLength -= equalPrefixLength; + rightLength -= equalPrefixLength; + } + if(settings->dontCheckFCD()) { + UTF8NFDIterator leftIter(left, leftLength); + UTF8NFDIterator rightIter(right, rightLength); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } else { + FCDUTF8NFDIterator leftIter(data, left, leftLength); + FCDUTF8NFDIterator rightIter(data, right, rightLength); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } +} + +UCollationResult +RuleBasedCollator::compare(UCharIterator &left, UCharIterator &right, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode) || &left == &right) { return UCOL_EQUAL; } + UBool numeric = settings->isNumeric(); + + // Identical-prefix test. + int32_t equalPrefixLength = 0; + { + UChar32 leftUnit; + UChar32 rightUnit; + while((leftUnit = left.next(&left)) == (rightUnit = right.next(&right))) { + if(leftUnit < 0) { return UCOL_EQUAL; } + ++equalPrefixLength; + } + + // Back out the code units that differed, for the real collation comparison. + if(leftUnit >= 0) { left.previous(&left); } + if(rightUnit >= 0) { right.previous(&right); } + + if(equalPrefixLength > 0) { + if((leftUnit >= 0 && data->isUnsafeBackward(leftUnit, numeric)) || + (rightUnit >= 0 && data->isUnsafeBackward(rightUnit, numeric))) { + // Identical prefix: Back up to the start of a contraction or reordering sequence. + do { + --equalPrefixLength; + leftUnit = left.previous(&left); + right.previous(&right); + } while(equalPrefixLength > 0 && data->isUnsafeBackward(leftUnit, numeric)); + } + // See the notes in the UTF-16 version. + } + } + + UCollationResult result; + if(settings->dontCheckFCD()) { + UIterCollationIterator leftIter(data, numeric, left); + UIterCollationIterator rightIter(data, numeric, right); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } else { + FCDUIterCollationIterator leftIter(data, numeric, left, equalPrefixLength); + FCDUIterCollationIterator rightIter(data, numeric, right, equalPrefixLength); + result = CollationCompare::compareUpToQuaternary(leftIter, rightIter, *settings, errorCode); + } + if(result != UCOL_EQUAL || settings->getStrength() < UCOL_IDENTICAL || U_FAILURE(errorCode)) { + return result; + } + + // Compare identical level. + left.move(&left, equalPrefixLength, UITER_ZERO); + right.move(&right, equalPrefixLength, UITER_ZERO); + const Normalizer2Impl &nfcImpl = data->nfcImpl; + if(settings->dontCheckFCD()) { + UIterNFDIterator leftIter(left); + UIterNFDIterator rightIter(right); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } else { + FCDUIterNFDIterator leftIter(data, left, equalPrefixLength); + FCDUIterNFDIterator rightIter(data, right, equalPrefixLength); + return compareNFDIter(nfcImpl, leftIter, rightIter); + } +} + +CollationKey & +RuleBasedCollator::getCollationKey(const UnicodeString &s, CollationKey &key, + UErrorCode &errorCode) const { + return getCollationKey(s.getBuffer(), s.length(), key, errorCode); +} + +CollationKey & +RuleBasedCollator::getCollationKey(const char16_t *s, int32_t length, CollationKey& key, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { + return key.setToBogus(); + } + if(s == nullptr && length != 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return key.setToBogus(); + } + key.reset(); // resets the "bogus" state + CollationKeyByteSink sink(key); + writeSortKey(s, length, sink, errorCode); + if(U_FAILURE(errorCode)) { + key.setToBogus(); + } else if(key.isBogus()) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + } else { + key.setLength(sink.NumberOfBytesAppended()); + } + return key; +} + +int32_t +RuleBasedCollator::getSortKey(const UnicodeString &s, + uint8_t *dest, int32_t capacity) const { + return getSortKey(s.getBuffer(), s.length(), dest, capacity); +} + +int32_t +RuleBasedCollator::getSortKey(const char16_t *s, int32_t length, + uint8_t *dest, int32_t capacity) const { + if((s == nullptr && length != 0) || capacity < 0 || (dest == nullptr && capacity > 0)) { + return 0; + } + uint8_t noDest[1] = { 0 }; + if(dest == nullptr) { + // Distinguish pure preflighting from an allocation error. + dest = noDest; + capacity = 0; + } + FixedSortKeyByteSink sink(reinterpret_cast(dest), capacity); + UErrorCode errorCode = U_ZERO_ERROR; + writeSortKey(s, length, sink, errorCode); + return U_SUCCESS(errorCode) ? sink.NumberOfBytesAppended() : 0; +} + +void +RuleBasedCollator::writeSortKey(const char16_t *s, int32_t length, + SortKeyByteSink &sink, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return; } + const char16_t *limit = (length >= 0) ? s + length : nullptr; + UBool numeric = settings->isNumeric(); + CollationKeys::LevelCallback callback; + if(settings->dontCheckFCD()) { + UTF16CollationIterator iter(data, numeric, s, s, limit); + CollationKeys::writeSortKeyUpToQuaternary(iter, data->compressibleBytes, *settings, + sink, Collation::PRIMARY_LEVEL, + callback, true, errorCode); + } else { + FCDUTF16CollationIterator iter(data, numeric, s, s, limit); + CollationKeys::writeSortKeyUpToQuaternary(iter, data->compressibleBytes, *settings, + sink, Collation::PRIMARY_LEVEL, + callback, true, errorCode); + } + if(settings->getStrength() == UCOL_IDENTICAL) { + writeIdenticalLevel(s, limit, sink, errorCode); + } + static const char terminator = 0; // TERMINATOR_BYTE + sink.Append(&terminator, 1); +} + +void +RuleBasedCollator::writeIdenticalLevel(const char16_t *s, const char16_t *limit, + SortKeyByteSink &sink, UErrorCode &errorCode) const { + // NFD quick check + const char16_t *nfdQCYesLimit = data->nfcImpl.decompose(s, limit, nullptr, errorCode); + if(U_FAILURE(errorCode)) { return; } + sink.Append(Collation::LEVEL_SEPARATOR_BYTE); + UChar32 prev = 0; + if(nfdQCYesLimit != s) { + prev = u_writeIdenticalLevelRun(prev, s, (int32_t)(nfdQCYesLimit - s), sink); + } + // Is there non-NFD text? + int32_t destLengthEstimate; + if(limit != nullptr) { + if(nfdQCYesLimit == limit) { return; } + destLengthEstimate = (int32_t)(limit - nfdQCYesLimit); + } else { + // s is NUL-terminated + if(*nfdQCYesLimit == 0) { return; } + destLengthEstimate = -1; + } + UnicodeString nfd; + data->nfcImpl.decompose(nfdQCYesLimit, limit, nfd, destLengthEstimate, errorCode); + u_writeIdenticalLevelRun(prev, nfd.getBuffer(), nfd.length(), sink); +} + +namespace { + +/** + * internalNextSortKeyPart() calls CollationKeys::writeSortKeyUpToQuaternary() + * with an instance of this callback class. + * When another level is about to be written, the callback + * records the level and the number of bytes that will be written until + * the sink (which is actually a FixedSortKeyByteSink) fills up. + * + * When internalNextSortKeyPart() is called again, it restarts with the last level + * and ignores as many bytes as were written previously for that level. + */ +class PartLevelCallback : public CollationKeys::LevelCallback { +public: + PartLevelCallback(const SortKeyByteSink &s) + : sink(s), level(Collation::PRIMARY_LEVEL) { + levelCapacity = sink.GetRemainingCapacity(); + } + virtual ~PartLevelCallback() {} + virtual UBool needToWrite(Collation::Level l) override { + if(!sink.Overflowed()) { + // Remember a level that will be at least partially written. + level = l; + levelCapacity = sink.GetRemainingCapacity(); + return true; + } else { + return false; + } + } + Collation::Level getLevel() const { return level; } + int32_t getLevelCapacity() const { return levelCapacity; } + +private: + const SortKeyByteSink &sink; + Collation::Level level; + int32_t levelCapacity; +}; + +} // namespace + +int32_t +RuleBasedCollator::internalNextSortKeyPart(UCharIterator *iter, uint32_t state[2], + uint8_t *dest, int32_t count, UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + if(iter == nullptr || state == nullptr || count < 0 || (count > 0 && dest == nullptr)) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(count == 0) { return 0; } + + FixedSortKeyByteSink sink(reinterpret_cast(dest), count); + sink.IgnoreBytes((int32_t)state[1]); + iter->move(iter, 0, UITER_START); + + Collation::Level level = (Collation::Level)state[0]; + if(level <= Collation::QUATERNARY_LEVEL) { + UBool numeric = settings->isNumeric(); + PartLevelCallback callback(sink); + if(settings->dontCheckFCD()) { + UIterCollationIterator ci(data, numeric, *iter); + CollationKeys::writeSortKeyUpToQuaternary(ci, data->compressibleBytes, *settings, + sink, level, callback, false, errorCode); + } else { + FCDUIterCollationIterator ci(data, numeric, *iter, 0); + CollationKeys::writeSortKeyUpToQuaternary(ci, data->compressibleBytes, *settings, + sink, level, callback, false, errorCode); + } + if(U_FAILURE(errorCode)) { return 0; } + if(sink.NumberOfBytesAppended() > count) { + state[0] = (uint32_t)callback.getLevel(); + state[1] = (uint32_t)callback.getLevelCapacity(); + return count; + } + // All of the normal levels are done. + if(settings->getStrength() == UCOL_IDENTICAL) { + level = Collation::IDENTICAL_LEVEL; + iter->move(iter, 0, UITER_START); + } + // else fall through to setting ZERO_LEVEL + } + + if(level == Collation::IDENTICAL_LEVEL) { + int32_t levelCapacity = sink.GetRemainingCapacity(); + UnicodeString s; + for(;;) { + UChar32 c = iter->next(iter); + if(c < 0) { break; } + s.append((char16_t)c); + } + const char16_t *sArray = s.getBuffer(); + writeIdenticalLevel(sArray, sArray + s.length(), sink, errorCode); + if(U_FAILURE(errorCode)) { return 0; } + if(sink.NumberOfBytesAppended() > count) { + state[0] = (uint32_t)level; + state[1] = (uint32_t)levelCapacity; + return count; + } + } + + // ZERO_LEVEL: Fill the remainder of dest with 00 bytes. + state[0] = (uint32_t)Collation::ZERO_LEVEL; + state[1] = 0; + int32_t length = sink.NumberOfBytesAppended(); + int32_t i = length; + while(i < count) { dest[i++] = 0; } + return length; +} + +void +RuleBasedCollator::internalGetCEs(const UnicodeString &str, UVector64 &ces, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return; } + const char16_t *s = str.getBuffer(); + const char16_t *limit = s + str.length(); + UBool numeric = settings->isNumeric(); + if(settings->dontCheckFCD()) { + UTF16CollationIterator iter(data, numeric, s, s, limit); + int64_t ce; + while((ce = iter.nextCE(errorCode)) != Collation::NO_CE) { + ces.addElement(ce, errorCode); + } + } else { + FCDUTF16CollationIterator iter(data, numeric, s, s, limit); + int64_t ce; + while((ce = iter.nextCE(errorCode)) != Collation::NO_CE) { + ces.addElement(ce, errorCode); + } + } +} + +namespace { + +void appendSubtag(CharString &s, char letter, const char *subtag, int32_t length, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || length == 0) { return; } + if(!s.isEmpty()) { + s.append('_', errorCode); + } + s.append(letter, errorCode); + for(int32_t i = 0; i < length; ++i) { + s.append(uprv_toupper(subtag[i]), errorCode); + } +} + +void appendAttribute(CharString &s, char letter, UColAttributeValue value, + UErrorCode &errorCode) { + if(U_FAILURE(errorCode)) { return; } + if(!s.isEmpty()) { + s.append('_', errorCode); + } + static const char *valueChars = "1234...........IXO..SN..LU......"; + s.append(letter, errorCode); + s.append(valueChars[value], errorCode); +} + +} // namespace + +int32_t +RuleBasedCollator::internalGetShortDefinitionString(const char *locale, + char *buffer, int32_t capacity, + UErrorCode &errorCode) const { + if(U_FAILURE(errorCode)) { return 0; } + if(buffer == nullptr ? capacity != 0 : capacity < 0) { + errorCode = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(locale == nullptr) { + locale = internalGetLocaleID(ULOC_VALID_LOCALE, errorCode); + } + + char resultLocale[ULOC_FULLNAME_CAPACITY + 1]; + int32_t length = ucol_getFunctionalEquivalent(resultLocale, ULOC_FULLNAME_CAPACITY, + "collation", locale, + nullptr, &errorCode); + if(U_FAILURE(errorCode)) { return 0; } + resultLocale[length] = 0; + + // Append items in alphabetic order of their short definition letters. + CharString result; + char subtag[ULOC_KEYWORD_AND_VALUES_CAPACITY]; + + if(attributeHasBeenSetExplicitly(UCOL_ALTERNATE_HANDLING)) { + appendAttribute(result, 'A', getAttribute(UCOL_ALTERNATE_HANDLING, errorCode), errorCode); + } + // ATTR_VARIABLE_TOP not supported because 'B' was broken. + // See ICU tickets #10372 and #10386. + if(attributeHasBeenSetExplicitly(UCOL_CASE_FIRST)) { + appendAttribute(result, 'C', getAttribute(UCOL_CASE_FIRST, errorCode), errorCode); + } + if(attributeHasBeenSetExplicitly(UCOL_NUMERIC_COLLATION)) { + appendAttribute(result, 'D', getAttribute(UCOL_NUMERIC_COLLATION, errorCode), errorCode); + } + if(attributeHasBeenSetExplicitly(UCOL_CASE_LEVEL)) { + appendAttribute(result, 'E', getAttribute(UCOL_CASE_LEVEL, errorCode), errorCode); + } + if(attributeHasBeenSetExplicitly(UCOL_FRENCH_COLLATION)) { + appendAttribute(result, 'F', getAttribute(UCOL_FRENCH_COLLATION, errorCode), errorCode); + } + // Note: UCOL_HIRAGANA_QUATERNARY_MODE is deprecated and never changes away from default. + length = uloc_getKeywordValue(resultLocale, "collation", subtag, UPRV_LENGTHOF(subtag), &errorCode); + appendSubtag(result, 'K', subtag, length, errorCode); + length = uloc_getLanguage(resultLocale, subtag, UPRV_LENGTHOF(subtag), &errorCode); + if (length == 0) { + appendSubtag(result, 'L', "root", 4, errorCode); + } else { + appendSubtag(result, 'L', subtag, length, errorCode); + } + if(attributeHasBeenSetExplicitly(UCOL_NORMALIZATION_MODE)) { + appendAttribute(result, 'N', getAttribute(UCOL_NORMALIZATION_MODE, errorCode), errorCode); + } + length = uloc_getCountry(resultLocale, subtag, UPRV_LENGTHOF(subtag), &errorCode); + appendSubtag(result, 'R', subtag, length, errorCode); + if(attributeHasBeenSetExplicitly(UCOL_STRENGTH)) { + appendAttribute(result, 'S', getAttribute(UCOL_STRENGTH, errorCode), errorCode); + } + length = uloc_getVariant(resultLocale, subtag, UPRV_LENGTHOF(subtag), &errorCode); + appendSubtag(result, 'V', subtag, length, errorCode); + length = uloc_getScript(resultLocale, subtag, UPRV_LENGTHOF(subtag), &errorCode); + appendSubtag(result, 'Z', subtag, length, errorCode); + + if(U_FAILURE(errorCode)) { return 0; } + return result.extract(buffer, capacity, errorCode); +} + +UBool +RuleBasedCollator::isUnsafe(UChar32 c) const { + return data->isUnsafeBackward(c, settings->isNumeric()); +} + +void U_CALLCONV +RuleBasedCollator::computeMaxExpansions(const CollationTailoring *t, UErrorCode &errorCode) { + t->maxExpansions = CollationElementIterator::computeMaxExpansions(t->data, errorCode); +} + +UBool +RuleBasedCollator::initMaxExpansions(UErrorCode &errorCode) const { + umtx_initOnce(tailoring->maxExpansionsInitOnce, computeMaxExpansions, tailoring, errorCode); + return U_SUCCESS(errorCode); +} + +CollationElementIterator * +RuleBasedCollator::createCollationElementIterator(const UnicodeString& source) const { + UErrorCode errorCode = U_ZERO_ERROR; + if(!initMaxExpansions(errorCode)) { return nullptr; } + CollationElementIterator *cei = new CollationElementIterator(source, this, errorCode); + if(U_FAILURE(errorCode)) { + delete cei; + return nullptr; + } + return cei; +} + +CollationElementIterator * +RuleBasedCollator::createCollationElementIterator(const CharacterIterator& source) const { + UErrorCode errorCode = U_ZERO_ERROR; + if(!initMaxExpansions(errorCode)) { return nullptr; } + CollationElementIterator *cei = new CollationElementIterator(source, this, errorCode); + if(U_FAILURE(errorCode)) { + delete cei; + return nullptr; + } + return cei; +} + +int32_t +RuleBasedCollator::getMaxExpansion(int32_t order) const { + UErrorCode errorCode = U_ZERO_ERROR; + (void)initMaxExpansions(errorCode); + return CollationElementIterator::getMaxExpansion(tailoring->maxExpansions, order); +} + +U_NAMESPACE_END + +#endif // !UCONFIG_NO_COLLATION diff --git a/intl/icu/source/i18n/scientificnumberformatter.cpp b/intl/icu/source/i18n/scientificnumberformatter.cpp new file mode 100644 index 0000000000..8f9c19c384 --- /dev/null +++ b/intl/icu/source/i18n/scientificnumberformatter.cpp @@ -0,0 +1,305 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (c) 2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ +#include "unicode/utypes.h" + +#if !UCONFIG_NO_FORMATTING + +#include "unicode/scientificnumberformatter.h" +#include "unicode/dcfmtsym.h" +#include "unicode/fpositer.h" +#include "unicode/utf16.h" +#include "unicode/uniset.h" +#include "unicode/decimfmt.h" +#include "static_unicode_sets.h" + +U_NAMESPACE_BEGIN + +static const char16_t kSuperscriptDigits[] = { + 0x2070, + 0xB9, + 0xB2, + 0xB3, + 0x2074, + 0x2075, + 0x2076, + 0x2077, + 0x2078, + 0x2079}; + +static const char16_t kSuperscriptPlusSign = 0x207A; +static const char16_t kSuperscriptMinusSign = 0x207B; + +static UBool copyAsSuperscript( + const UnicodeString &s, + int32_t beginIndex, + int32_t endIndex, + UnicodeString &result, + UErrorCode &status) { + if (U_FAILURE(status)) { + return false; + } + for (int32_t i = beginIndex; i < endIndex;) { + UChar32 c = s.char32At(i); + int32_t digit = u_charDigitValue(c); + if (digit < 0) { + status = U_INVALID_CHAR_FOUND; + return false; + } + result.append(kSuperscriptDigits[digit]); + i += U16_LENGTH(c); + } + return true; +} + +ScientificNumberFormatter *ScientificNumberFormatter::createSuperscriptInstance( + DecimalFormat *fmtToAdopt, UErrorCode &status) { + return createInstance(fmtToAdopt, new SuperscriptStyle(), status); +} + +ScientificNumberFormatter *ScientificNumberFormatter::createSuperscriptInstance( + const Locale &locale, UErrorCode &status) { + return createInstance( + static_cast( + DecimalFormat::createScientificInstance(locale, status)), + new SuperscriptStyle(), + status); +} + +ScientificNumberFormatter *ScientificNumberFormatter::createMarkupInstance( + DecimalFormat *fmtToAdopt, + const UnicodeString &beginMarkup, + const UnicodeString &endMarkup, + UErrorCode &status) { + return createInstance( + fmtToAdopt, + new MarkupStyle(beginMarkup, endMarkup), + status); +} + +ScientificNumberFormatter *ScientificNumberFormatter::createMarkupInstance( + const Locale &locale, + const UnicodeString &beginMarkup, + const UnicodeString &endMarkup, + UErrorCode &status) { + return createInstance( + static_cast( + DecimalFormat::createScientificInstance(locale, status)), + new MarkupStyle(beginMarkup, endMarkup), + status); +} + +ScientificNumberFormatter *ScientificNumberFormatter::createInstance( + DecimalFormat *fmtToAdopt, + Style *styleToAdopt, + UErrorCode &status) { + LocalPointer fmt(fmtToAdopt); + LocalPointer