diff options
Diffstat (limited to '')
30 files changed, 9303 insertions, 0 deletions
diff --git a/contrib/snowball/algorithms/arabic.sbl b/contrib/snowball/algorithms/arabic.sbl new file mode 100644 index 0000000..d827ee7 --- /dev/null +++ b/contrib/snowball/algorithms/arabic.sbl @@ -0,0 +1,561 @@ +/* + * Authors: + * - Assem Chelli, < assem [dot] ch [at] gmail > + * - Abdelkrim Aries <ab [underscore] aries [at] esi [dot] dz> + * +*/ + +stringescapes { } + +/* the Arabic letters in Unicode */ +// Hamza +stringdef o '{U+0621}' // Hamza +stringdef ao '{U+0623}' // Hamza above Alef +stringdef ao_ '{U+0625}' // Hamza below Alef +stringdef a~ '{U+0622}' // Alef madda +stringdef wo '{U+0624}' // Hamza above waw +stringdef yo '{U+0626}' // Hamza above yeh + +// Letters +stringdef a '{U+0627}' // Alef +stringdef a_ '{U+0649}' // Alef Maksura +stringdef b '{U+0628}' // Beh +stringdef t_ '{U+0629}' // Teh_Marbuta +stringdef t '{U+062A}' // Teh +stringdef th '{U+062B}' // Theh +stringdef j '{U+062C}' // Jeem +stringdef h '{U+062D}' // Hah +stringdef x '{U+062E}' // Khah +stringdef d '{U+062F}' // Dal +stringdef dz '{U+0630}' // Thal +stringdef r '{U+0631}' // Reh +stringdef z '{U+0632}' // Zain +stringdef s '{U+0633}' // Seen +stringdef sh '{U+0634}' // Sheen +stringdef c '{U+0635}' // Sad +stringdef dh '{U+0636}' // Dad +stringdef tt '{U+0637}' // Tah +stringdef zh '{U+0638}' // Zah +stringdef i '{U+0639}' // Ain +stringdef gh '{U+063A}' // Ghain +stringdef f '{U+0641}' // Feh +stringdef q '{U+0642}' // Qaf +stringdef k '{U+0643}' // Kaf +stringdef l '{U+0644}' // Lam +stringdef m '{U+0645}' // Meem +stringdef n '{U+0646}' // Noon +stringdef e '{U+0647}' // Heh +stringdef w '{U+0648}' // Waw +stringdef y '{U+064A}' // Yeh + +// Diacritics +stringdef aan '{U+064B}' // FatHatan +stringdef uun '{U+064C}' // Dammatan +stringdef iin '{U+064D}' // Kasratan +stringdef aa '{U+064E}' // FatHa +stringdef uu '{U+064F}' // Damma +stringdef ii '{U+0650}' // Kasra +stringdef oo '{U+0652}' // Sukun +stringdef ~ '{U+0651}' // Shadda + +// Hindu–Arabic numerals +stringdef 0 '{U+0660}' +stringdef 1 '{U+0661}' +stringdef 2 '{U+0662}' +stringdef 3 '{U+0663}' +stringdef 4 '{U+0664}' +stringdef 5 '{U+0665}' +stringdef 6 '{U+0666}' +stringdef 7 '{U+0667}' +stringdef 8 '{U+0668}' +stringdef 9 '{U+0669}' + + +// Kasheeda +stringdef _ '{U+0640}' // Kasheeda, Tatweel + +// Shaped forms +stringdef o1 '{U+FE80}' // HAMZA +stringdef ao1 '{U+FE83}' // ALEF_HAMZA_ABOVE +stringdef ao2 '{U+FE84}' // ALEF_HAMZA_ABOVE +stringdef ao_1 '{U+FE87}' // ALEF_HAMZA_BELOW +stringdef ao_2 '{U+FE88}' // ALEF_HAMZA_BELOW +stringdef yo1 '{U+FE8B}' // YEH_HAMZA +stringdef yo2 '{U+FE8C}' // YEH_HAMZA +stringdef yo3 '{U+FE89}' // YEH_HAMZA +stringdef yo4 '{U+FE8A}' // YEH_HAMZA +stringdef a~1 '{U+FE81}' // ALEF_MADDA +stringdef a~2 '{U+FE82}' // ALEF_MADDA +stringdef wo1 '{U+FE85}' // WAW_HAMZA +stringdef wo2 '{U+FE86}' // WAW_HAMZA +stringdef a1 '{U+FE8D}' // ALEF +stringdef a2 '{U+FE8E}' // ALEF +stringdef b1 '{U+FE8F}' // BEH +stringdef b2 '{U+FE90}' // BEH +stringdef b3 '{U+FE91}' // BEH +stringdef b4 '{U+FE92}' // BEH +stringdef t_1 '{U+FE93}' // TEH_MARBUTA +stringdef t_2 '{U+FE94}' // TEH_MARBUTA +stringdef t1 '{U+FE97}' // TEH +stringdef t2 '{U+FE98}' // TEH +stringdef t3 '{U+FE95}' // TEH +stringdef t4 '{U+FE96}' // TEH +stringdef th1 '{U+FE9B}' // THEH +stringdef th2 '{U+FE9C}' // THEH +stringdef th3 '{U+FE9A}' // THEH +stringdef th4 '{U+FE99}' // THEH +stringdef j1 '{U+FE9F}' // JEEM +stringdef j2 '{U+FEA0}' // JEEM +stringdef j3 '{U+FE9D}' // JEEM +stringdef j4 '{U+FE9E}' // JEEM +stringdef h1 '{U+FEA3}' // HAH +stringdef h2 '{U+FEA4}' // HAH +stringdef h3 '{U+FEA1}' // HAH +stringdef h4 '{U+FEA2}' // HAH +stringdef x1 '{U+FEA7}' // KHAH +stringdef x2 '{U+FEA8}' // KHAH +stringdef x3 '{U+FEA5}' // KHAH +stringdef x4 '{U+FEA6}' // KHAH +stringdef d1 '{U+FEA9}' // DAL +stringdef d2 '{U+FEAA}' // DAL +stringdef dz1 '{U+FEAB}' // THAL +stringdef dz2 '{U+FEAC}' // THAL +stringdef r1 '{U+FEAD}' // REH +stringdef r2 '{U+FEAE}' // REH +stringdef z1 '{U+FEAF}' // ZAIN +stringdef z2 '{U+FEB0}' // ZAIN +stringdef s1 '{U+FEB3}' // SEEN +stringdef s2 '{U+FEB4}' // SEEN +stringdef s3 '{U+FEB1}' // SEEN +stringdef s4 '{U+FEB2}' // SEEN +stringdef sh1 '{U+FEB7}' // SHEEN +stringdef sh2 '{U+FEB8}' // SHEEN +stringdef sh3 '{U+FEB5}' // SHEEN +stringdef sh4 '{U+FEB6}' // SHEEN +stringdef c1 '{U+FEBB}' // SAD +stringdef c2 '{U+FEBC}' // SAD +stringdef c3 '{U+FEB9}' // SAD +stringdef c4 '{U+FEBA}' // SAD +stringdef dh1 '{U+FEBF}' // DAD +stringdef dh2 '{U+FEC0}' // DAD +stringdef dh3 '{U+FEBD}' // DAD +stringdef dh4 '{U+FEBE}' // DAD +stringdef tt1 '{U+FEC3}' // TAH +stringdef tt2 '{U+FEC4}' // TAH +stringdef tt3 '{U+FEC1}' // TAH +stringdef tt4 '{U+FEC2}' // TAH +stringdef zh1 '{U+FEC7}' // ZAH +stringdef zh2 '{U+FEC8}' // ZAH +stringdef zh3 '{U+FEC5}' // ZAH +stringdef zh4 '{U+FEC6}' // ZAH +stringdef i1 '{U+FECB}' // AIN +stringdef i2 '{U+FECC}' // AIN +stringdef i3 '{U+FEC9}' // AIN +stringdef i4 '{U+FECA}' // AIN +stringdef gh1 '{U+FECF}' // GHAIN +stringdef gh2 '{U+FED0}' // GHAIN +stringdef gh3 '{U+FECD}' // GHAIN +stringdef gh4 '{U+FECE}' // GHAIN +stringdef f1 '{U+FED3}' // FEH +stringdef f2 '{U+FED4}' // FEH +stringdef f3 '{U+FED1}' // FEH +stringdef f4 '{U+FED2}' // FEH +stringdef q1 '{U+FED7}' // QAF +stringdef q2 '{U+FED8}' // QAF +stringdef q3 '{U+FED5}' // QAF +stringdef q4 '{U+FED6}' // QAF +stringdef k1 '{U+FEDB}' // KAF +stringdef k2 '{U+FEDC}' // KAF +stringdef k3 '{U+FED9}' // KAF +stringdef k4 '{U+FEDA}' // KAF +stringdef l1 '{U+FEDF}' // LAM +stringdef l2 '{U+FEE0}' // LAM +stringdef l3 '{U+FEDD}' // LAM +stringdef l4 '{U+FEDE}' // LAM +stringdef m1 '{U+FEE3}' // MEEM +stringdef m2 '{U+FEE4}' // MEEM +stringdef m3 '{U+FEE1}' // MEEM +stringdef m4 '{U+FEE2}' // MEEM +stringdef n1 '{U+FEE7}' // NOON +stringdef n2 '{U+FEE8}' // NOON +stringdef n3 '{U+FEE5}' // NOON +stringdef n4 '{U+FEE6}' // NOON +stringdef e1 '{U+FEEB}' // HEH +stringdef e2 '{U+FEEC}' // HEH +stringdef e3 '{U+FEE9}' // HEH +stringdef e4 '{U+FEEA}' // HEH +stringdef w1 '{U+FEED}' // WAW +stringdef w2 '{U+FEEE}' // WAW +stringdef a_1 '{U+FEEF}' // ALEF_MAKSURA +stringdef a_2 '{U+FEF0}' // ALEF_MAKSURA +stringdef y1 '{U+FEF3}' // YEH +stringdef y2 '{U+FEF4}' // YEH +stringdef y3 '{U+FEF1}' // YEH +stringdef y4 '{U+FEF2}' // YEH + +// Ligatures Lam-Alef +stringdef la '{U+FEFB}' // LAM_ALEF +stringdef la2 '{U+FEFC}' // LAM_ALEF +stringdef lao '{U+FEF7}' // LAM_ALEF_HAMZA_ABOVE +stringdef lao2 '{U+FEF8}' // LAM_ALEF_HAMZA_ABOVE +stringdef lao_ '{U+FEF9}' // LAM_ALEF_HAMZA_BELOW +stringdef lao_2 '{U+FEFA}' // LAM_ALEF_HAMZA_BELOW +stringdef la~ '{U+FEF5}' // LAM_ALEF_MADDA_ABOVE +stringdef la~2 '{U+FEF6}' // LAM_ALEF_MADDA_ABOVE + + +booleans ( + is_noun + is_verb + is_defined + ) + +routines ( + Prefix_Step1 + Prefix_Step2 + Prefix_Step3a_Noun + Prefix_Step3b_Noun + Prefix_Step3_Verb + Prefix_Step4_Verb + + Suffix_All_alef_maqsura + Suffix_Noun_Step1a + Suffix_Noun_Step1b + Suffix_Noun_Step2a + Suffix_Noun_Step2b + Suffix_Noun_Step2c1 + Suffix_Noun_Step2c2 + Suffix_Noun_Step3 + Suffix_Verb_Step1 + Suffix_Verb_Step2a + Suffix_Verb_Step2b + Suffix_Verb_Step2c + + Normalize_post + Normalize_pre + + Checks1 +) + +externals ( stem ) + +groupings ( ) + + +// Normalizations +define Normalize_pre as ( + do repeat ( + ( + [substring] among ( + '{aan}' '{uun}' '{iin}' '{aa}' '{uu}' '{ii}' '{oo}' '{~}'( delete ) // strip vocalization + '{_}' ( delete ) // strip kasheeda + + // Hindu–Arabic numerals + '{0}' ( <- '0') + '{1}' ( <- '1') + '{2}' ( <- '2') + '{3}' ( <- '3') + '{4}' ( <- '4') + '{5}' ( <- '5') + '{6}' ( <- '6') + '{7}' ( <- '7') + '{8}' ( <- '8') + '{9}' ( <- '9') + + // Shaped forms + '{o1}' ( <- '{o}' ) // HAMZA + '{ao1}' '{ao2}' ( <- '{ao}' ) // ALEF_HAMZA_ABOVE + '{ao_1}' '{ao_2}' ( <- '{ao_}' ) // ALEF_HAMZA_BELOW + '{yo1}' '{yo2}' '{yo3}' '{yo4}' ( <- '{yo}' ) // YEH_HAMZA + '{a~1}' '{a~2}'( <- '{a~}' ) // ALEF_MADDA + '{wo1}' '{wo2}'( <- '{wo}' ) // WAW_HAMZA + '{a1}' '{a2}' ( <- '{a}' ) // ALEF + '{b1}' '{b2}' '{b3}' '{b4}' ( <- '{b}' ) // BEH + '{t_1}' '{t_2}' ( <- '{t_}' ) // TEH_MARBUTA + '{t1}' '{t2}' '{t3}' '{t4}' ( <- '{t}' ) // TEH + '{th1}' '{th2}' '{th3}' '{th4}' ( <- '{th}' ) // THEH + '{j1}' '{j2}' '{j3}' '{j4}'( <- '{j}' ) // JEEM + '{h1}' '{h2}' '{h3}' '{h4}' ( <- '{h}' ) // HAH + '{x1}' '{x2}' '{x3}' '{x4}'( <- '{x}' ) // KHAH + '{d1}' '{d2}' ( <- '{d}' ) // DAL + '{dz1}''{dz2}' ( <- '{dz}' ) // THAL + '{r1}' '{r2}'( <- '{r}' ) // REH + '{z1}' '{z2}' ( <- '{z}' ) // ZAIN + '{s1}' '{s2}' '{s3}' '{s4}'( <- '{s}' ) // SEEN + '{sh1}' '{sh2}' '{sh3}' '{sh4}' ( <- '{sh}' ) // SHEEN + '{c1}' '{c2}' '{c3}' '{c4}'( <- '{c}' ) // SAD + '{dh1}' '{dh2}' '{dh3}' '{dh4}'( <- '{dh}' ) // DAD + '{tt1}' '{tt2}' '{tt3}' '{tt4}' ( <- '{tt}' ) // TAH + '{zh1}' '{zh2}' '{zh3}' '{zh4}'( <- '{zh}' ) // ZAH + '{i1}' '{i2}' '{i3}' '{i4}'( <- '{i}' ) // AIN + '{gh1}' '{gh2}' '{gh3}' '{gh4}'( <- '{gh}' ) // GHAIN + '{f1}' '{f2}' '{f3}' '{f4}' ( <- '{f}' ) // FEH + '{q1}' '{q2}' '{q3}' '{q4}' ( <- '{q}' ) // QAF + '{k1}' '{k2}' '{k3}' '{k4}'( <- '{k}' ) // KAF + '{l1}' '{l2}' '{l3}' '{l4}'( <- '{l}' ) // LAM + '{m1}' '{m2}' '{m3}' '{m4}' ( <- '{m}' ) // MEEM + '{n1}' '{n2}' '{n3}' '{n4}'( <- '{n}' ) // NOON + '{e1}' '{e2}' '{e3}' '{e4}' ( <- '{e}' ) // HEH + '{w1}' '{w2}' ( <- '{w}' ) // WAW + '{a_1}' '{a_2}' ( <- '{a_}' ) // ALEF_MAKSURA + '{y1}' '{y2}' '{y3}' '{y4}' ( <- '{y}' ) // YEH + + // Ligatures Lam-Alef + '{la}' '{la2}' (<- '{l}{a}') + '{lao}' '{lao2}' (<- '{l}{ao}') + '{lao_}' '{lao_2}' (<- '{l}{ao_}') + '{la~}' '{la~2}' (<- '{l}{a~}') + + ) + ) + or + next + ) +) + +define Normalize_post as ( + + do ( + // normalize last hamza + backwards ( + [substring] among ( + '{ao}''{ao_}' '{a~}' ( <- '{o}') + '{wo}' ( <- '{o}') + '{yo}' ( <- '{o}') + ) + ) + ) + + do repeat ( + ( + // normalize other hamza's + [substring] among ( + '{ao}''{ao_}' '{a~}' ( <- '{a}') + '{wo}' ( <- '{w}') + '{yo}' ( <- '{y}') + ) + ) + or + next + ) +) + +// Checks +define Checks1 as ( + [substring] among ( + '{b}{a}{l}' '{k}{a}{l}' ($(len > 4) set is_noun unset is_verb set is_defined) + '{l}{l}' '{a}{l}' ($(len > 3) set is_noun unset is_verb set is_defined) + ) +) + + +//prefixes +define Prefix_Step1 as ( + [substring] among ( + '{ao}{ao}' ($(len > 3) <- '{ao}' ) + '{ao}{a~}' ($(len > 3) <- '{a~}' ) + '{ao}{wo}' ($(len > 3) <- '{ao}' ) + '{ao}{a}' ($(len > 3) <- '{a}' ) + '{ao}{ao_}' ($(len > 3) <- '{ao_}' ) + // '{ao}' ($(len > 3) delete) //rare case + ) +) + +define Prefix_Step2 as ( + not '{f}{a}' + not '{w}{a}' + [substring] among ( + '{f}' ($(len > 3) delete) + '{w}' ($(len > 3) delete) + ) +) + +define Prefix_Step3a_Noun as ( // it is noun and defined + [substring] among ( + '{b}{a}{l}' '{k}{a}{l}' ($(len > 5) delete) + '{l}{l}' '{a}{l}' ($(len > 4) delete) + ) +) + +define Prefix_Step3b_Noun as ( // probably noun and defined + not '{b}{a}' // exception + [substring] among ( + '{b}' ($(len > 3) delete) + // '{k}' '{l}' ($(len > 3) delete) // BUG: cause confusion + '{b}{b}' ($(len > 3) <- '{b}' ) + '{k}{k}' ($(len > 3) <- '{k}' ) + ) + +) + +define Prefix_Step3_Verb as ( + [substring] among ( + //'{s}' ($(len > 4) delete)// BUG: cause confusion + '{s}{y}' ($(len > 4) <- '{y}' ) + '{s}{t}' ($(len > 4) <- '{t}') + '{s}{n}' ($(len > 4) <- '{n}') + '{s}{ao}' ($(len > 4) <- '{ao}') + ) +) + +define Prefix_Step4_Verb as ( + [substring] among ( + '{y}{s}{t}' '{n}{s}{t}' '{t}{s}{t}' ($(len > 4) set is_verb unset is_noun <- '{a}{s}{t}' ) + ) +) + +// suffixes +backwardmode ( + + define Suffix_Noun_Step1a as ( + [substring] among ( + '{y}' '{k}' '{e}' ($(len >= 4) delete) + '{n}{a}' '{k}{m}' '{e}{a}' '{e}{n}' '{e}{m}' ($(len >= 5) delete) + '{k}{m}{a}' '{e}{m}{a}' ($(len >= 6) delete) + ) + ) + define Suffix_Noun_Step1b as ( + [substring] among ( + '{n}' ($(len > 5) delete) + ) + ) + + define Suffix_Noun_Step2a as ( + [substring] among ( + '{a}' '{y}' '{w}' ($(len > 4) delete) + ) + ) + + define Suffix_Noun_Step2b as ( + [substring] among ( + '{a}{t}' ($(len >= 5) delete) + ) + ) + + define Suffix_Noun_Step2c1 as ( + [substring] among ( + '{t}' ($(len >= 4) delete) + ) + ) + define Suffix_Noun_Step2c2 as ( // feminine t_ + [substring] among ( + '{t_}' ($(len >= 4) delete) + ) + ) + define Suffix_Noun_Step3 as ( // ya' nisbiya + [substring] among ( + '{y}' ($(len >= 3) delete) + ) + ) + + define Suffix_Verb_Step1 as ( + [substring] among ( + '{e}' '{k}' ($(len >= 4) delete) + '{n}{y}' '{n}{a}' '{e}{a}' '{e}{m}' '{e}{n}' '{k}{m}' '{k}{n}' ($(len >= 5) delete) + '{e}{m}{a}' '{k}{m}{a}' '{k}{m}{w}'($(len >= 6) delete) + ) + ) + define Suffix_Verb_Step2a as ( + [substring] among ( + '{t}' ($(len >= 4) delete) + '{a}' '{n}' '{y}' ($(len >= 4) delete) + '{n}{a}' '{t}{a}' '{t}{n}' ($(len >= 5) delete)// past + '{a}{n}' '{w}{n}' '{y}{n}' ($(len > 5) delete) // present + '{t}{m}{a}' ($(len >= 6) delete) + ) + ) + + define Suffix_Verb_Step2b as ( + [substring] among ( + '{w}{a}' '{t}{m}' ($(len >= 5) delete) + ) + ) + + + define Suffix_Verb_Step2c as ( + [substring] among ( + '{w}' ($(len >= 4) delete) + '{t}{m}{w}' ($(len >= 6) delete) + ) + ) + + define Suffix_All_alef_maqsura as ( + [substring] among ( + '{a_}' ( <- '{y}' ) // spell error + // '{a_}' ( delete ) // if noun > 3 + // '{a_}' ( <- '{a}') // if verb + ) + ) +) + +define stem as ( + // set initial values + set is_noun + set is_verb + unset is_defined + + // guess type and properties + do Checks1 + + // normalization pre-stemming + do Normalize_pre + + + backwards ( + + do ( + //Suffixes for verbs + ( + is_verb + ( + ( + (atleast 1 Suffix_Verb_Step1) + ( Suffix_Verb_Step2a or Suffix_Verb_Step2c or next) + ) + or Suffix_Verb_Step2b + or Suffix_Verb_Step2a + ) + ) + //Suffixes for nouns + or ( + is_noun + ( + + try ( + Suffix_Noun_Step2c2 + or (not is_defined Suffix_Noun_Step1a ( + Suffix_Noun_Step2a + or Suffix_Noun_Step2b + or Suffix_Noun_Step2c1 + or next)) + or (Suffix_Noun_Step1b ( + Suffix_Noun_Step2a + or Suffix_Noun_Step2b + or Suffix_Noun_Step2c1)) + or (not is_defined Suffix_Noun_Step2a) + or (Suffix_Noun_Step2b) + ) + Suffix_Noun_Step3 + ) + + ) + + // Suffixes for alef maqsura + or Suffix_All_alef_maqsura + ) + ) + + //Prefixes + do ( + try Prefix_Step1 + try Prefix_Step2 + ( Prefix_Step3a_Noun + or (is_noun Prefix_Step3b_Noun) + or (is_verb try Prefix_Step3_Verb Prefix_Step4_Verb) + ) + ) + + // normalization post-stemming + do Normalize_post + +) diff --git a/contrib/snowball/algorithms/basque.sbl b/contrib/snowball/algorithms/basque.sbl new file mode 100644 index 0000000..267abc7 --- /dev/null +++ b/contrib/snowball/algorithms/basque.sbl @@ -0,0 +1,149 @@ +routines ( + aditzak + izenak + adjetiboak + mark_regions + RV R2 R1 +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v ) + +stringescapes {} + +/* special characters */ + +stringdef n~ '{U+00F1}' + +define v 'aeiou' + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v (non-v gopast v) or (v gopast non-v) ) + or + ( non-v (non-v gopast v) or (v next) ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +backwardmode ( + + define RV as $pV <= cursor + define R2 as $p2 <= cursor + define R1 as $p1 <= cursor + + define aditzak as ( + [substring] among( + 'le' 'la' 'tzaile' 'aldatu' 'atu' 'tzailea' 'taile' 'tailea' 'pera' 'gale' 'galea' + 'gura' 'kura' 'kor' 'korra' 'or' 'orra' 'tun' 'tuna' 'gaitz' 'gaitza' + 'kaitz' 'kaitza' 'ezin' 'ezina' 'tezin' 'tezina' 'errez' 'erreza' + 'karri' 'karria' 'tzaga' 'tzaka' 'tzake' 'tzeke' 'ez' 'eza' 'tzez' + 'keta' 'eta' 'etan' 'pen' 'pena' 'tze' 'atze' 'kuntza' 'kunde' 'kundea' + 'kune' 'kunea' 'kuna' 'kera' 'era' 'kizun' 'kizuna' 'dura' 'tura' 'men' 'mena' + 'go' 'ago' 'tio' 'taldi' 'taldia' 'aldi' 'aldia' 'gune' 'gunea' 'bide' 'bidea' + 'pide' 'pidea' 'gai' 'gaia' 'ki' 'kin' 'rekin' 'kina' 'kari' 'karia' 'ari' 'tari' 'etari' + 'gailu' 'gailua' 'kide' 'kidea' 'ide' 'idea' 'du' 'ka' 'kan' 'an' 'ean' 'tu' 'lari' 'tatu' + 'rean' 'tarazi' 'arazi' 'tzat' 'bera' 'dako' + ( RV delete ) + 'garri' 'garria' 'tza' + (R2 delete) + 'atseden' + (<- 'atseden') + 'arabera' + (<- 'arabera') + 'baditu' + (<- 'baditu') + + ) + ) + + define izenak as ( + [substring] among( + 'ari' 'aria' 'bizia' 'kari' 'karia' 'lari' 'laria' 'tari' 'taria' 'zain' 'zaina' + 'tzain' 'tzaina' 'zale' 'zalea' 'tzale' 'tzalea' 'aizun' 'orde' 'ordea' + 'burua' 'ohi' 'ohia' 'kintza' 'gintzo' 'gintzu' 'tzu' 'tzua' + 'tzo' 'tzoa' 'kuntza' 'talde' 'taldea' 'eria' 'keria' 'teria' 'di' + 'za' 'ada' 'tara' 'etara' 'tra' 'ta' 'tegi' 'tegia' 'keta' 'z' 'zko' 'zkoa' + 'ti' 'tia' 'tsu' 'tsua' 'zu' 'zua' 'bera' 'pera' 'zto' 'ztoa' 'asi' 'asia' + 'gile' 'gilea' 'estu' 'estua' 'larri' 'larria' 'nahi' 'nahia' + 'koi' 'koia' 'oi' 'oia' 'goi' 'min' 'mina' 'dun' 'duna' 'duru' 'durua' + 'duri' 'duria' 'os' 'osa' 'oso' 'osoa' 'ar' 'ara' 'tar' 'dar' 'dara' + 'tiar' 'tiara' 'liar' 'liara' 'gabe' 'gabea' 'kabe' 'kabea' 'ga' 'ge' + 'kada' 'tasun' 'tasuna' 'asun' 'asuna' 'go' 'mendu' 'mendua' 'mentu' 'mentua' + 'mendi' 'mendia' 'zio' 'zioa' 'zino' 'zinoa' 'zione' 'zionea' 'ezia' + 'degi' 'degia' 'egi' 'egia' 'toki' 'tokia' 'leku' 'lekua' 'gintza' 'alde' + 'aldea' 'kalde' 'kaldea' 'gune' 'gunea' 'une' 'unea' 'una' 'pe' 'pea' + 'gibel' 'gibela' 'ondo' 'ondoa' 'arte' 'artea' 'aurre' 'aurrea' + 'etxe' 'etxea' 'ola' 'ontzi' 'ontzia' 'gela' 'denda' 'taldi' 'taldia' + 'aldi' 'aldia' 'te' 'tea' 'zaro' 'zaroa' 'taro' 'taroa' 'oro' 'oroa' + 'aro' 'aroa' 'ero' 'eroa' 'eroz' 'eroza' 'ka' 'kan' 'kana' 'tako' 'etako' 'takoa' + 'kote' 'kotea' 'tzar' 'tzarra' 'handi' 'handia' 'kondo' 'kondoa' 'skila' + 'no' 'noa' '{n~}o' '{n~}oa' 'ska' 'xka' 'zka' 'tila' 'to' 'toa' 'tto' 'ttoa' + 'txo' 'txoa' 'txu' 'txua' 'anda' 'anga' 'urren' 'urrena' 'gai' 'gaia' + 'gei' 'geia' 'eme' 'emea' 'kume' 'kumea' 'sa' 'ko' 'eko' 'koa' 'ena' + 'enea' 'ne' 'nea' 'kor' 'korra' 'ez' 'eza' 'eta' 'etan' + 'ki' 'kia' 'kin' 'kina' 'tu' 'tua' 'du' 'dua' 'ek' + 'tarik' 'tariko' 'tan' 'ordu' 'ordua' 'oste' 'ostea' 'tzara' + 'ra' 'antza' 'behar' 'ro' 'giro' 'ak' 'zp' 'ket' + 'kail' 'kaila' 'ail' 'kirri' 'kirria' 'ngo' 'ngoa' '{n~}i' 'sko' + 'sta' 'koitz' 'koitza' 'na' 'garren' 'garrena' 'kera' + 'gerren' 'gerrena' 'garna' 'kide' 'tz' 'tuko' + ( RV delete ) + 'ora' 'garri' 'garria' 'or' 'buru' 'ren' 'tza' + ( R2 delete ) + 'joka' + (<- 'jok') + 'tzen' 'ten' 'en' 'tatu' + (R1 delete) + 'trako' + (<- 'tra') + 'minutuko' + (<- 'minutu') + 'zehar' + (<- 'zehar') + 'geldi' + (<- 'geldi') + 'igaro' + (<- 'igaro') + 'aurka' + (<- 'aurka') + ) + ) + + define adjetiboak as ( + [substring] among( + 'era' 'ero' 'go' 'tate' 'tade' 'date' 'dade' 'keria' + 'ki' 'to' 'ro' 'la' 'gi' 'larik' 'lanik' 'ik' 'ztik' 'rik' + ( RV delete ) + 'zlea' + (<- 'z') + ) + ) + +) + +define stem as ( + do mark_regions + backwards ( + repeat aditzak + repeat izenak + do adjetiboak + ) + +) + +/* + Note 1: additions of 21 Jul 2010 +*/ diff --git a/contrib/snowball/algorithms/catalan.sbl b/contrib/snowball/algorithms/catalan.sbl new file mode 100644 index 0000000..0a1e3a5 --- /dev/null +++ b/contrib/snowball/algorithms/catalan.sbl @@ -0,0 +1,202 @@ +routines ( + cleaning mark_regions + R1 R2 + attached_pronoun + standard_suffix + verb_suffix + residual_suffix +) + +externals ( stem ) + +integers ( p1 p2 ) + +groupings ( v ) + +stringescapes {} + +/* special characters */ + +stringdef a' '{U+00E1}' // a-acute +stringdef a` '{U+00E0}' // a-grave +stringdef c, '{U+00E7}' // c-cedilla +stringdef e' '{U+00E9}' // e-acute +stringdef e` '{U+00E8}' // e-grave +stringdef i' '{U+00ED}' // i-acute +stringdef i` '{U+00EC}' // i-grave +stringdef i" '{U+00EF}' // i-diaeresis +stringdef o' '{U+00F3}' // o-acute +stringdef o` '{U+00F2}' // o-grave +stringdef u' '{U+00FA}' // u-acute +stringdef u" '{U+00FC}' // u-diaeresis +stringdef . '{U+00B7}' // - per l aggeminades + +define v 'aeiou{a'}{a`}{e'}{e`}{i'}{i"}{o'}{o`}{u'}{u"}' + +define mark_regions as ( + + $p1 = limit + $p2 = limit // defaults + + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define cleaning as repeat ( + [substring] among( + '{a'}' (<- 'a') + '{a`}' (<- 'a') + '{e'}' (<- 'e') + '{e`}' (<- 'e') + '{i'}' (<- 'i') + '{i`}' (<- 'i') + '{o'}' (<- 'o') + '{o`}' (<- 'o') + '{u'}' (<- 'u') + '{u"}' (<- 'u') + '{i"}' (<- 'i') + '{.}' (<- '.') + '' (next) + ) +) + +backwardmode ( + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define attached_pronoun as ( + [substring] among ( + '{'}s' '{'}hi' '{'}ho' '{'}l' '{'}ls' + '-ls' '-la' '-les' '-li' + 'vos' 'se' 'nos' '-nos' '-us' 'us' + '{'}n' '{'}ns' '-n' '-ns' + '{'}m' '-me' '-m' + '-te' '{'}t' + 'li' 'lo' 'los' + 'me' 'sela' 'selo' 'selas' 'selos' 'le' + 'la' 'las' 'les' 'ens' 'ho' 'hi' + (R1 delete) + ) + ) + + define standard_suffix as ( + [substring] among( + 'ar' 'atge' 'formes' 'icte' 'ictes' + 'ell' 'ells' 'ella' '{e'}s' '{e`}s' 'esc' 'essa' 'et' 'ets' 'eta' + 'eres' 'eries' 'ers' 'ina' 'ines' 'able' 'ls' + 'i{o'}' 'itat' 'itats' 'itzar' 'iva' 'ives' 'ivisme' 'ius' + 'fer' 'ment' 'amen' 'ament' 'aments' 'ments' 'ot' 'sfera' 'al' 'als' 'era' 'ana' 'iste' + 'aire' 'eria' 'esa' 'eses' 'esos' 'or' '{i'}cia' '{i'}cies' 'icis' 'ici' '{i'}ci' '{i'}cis' + '{a`}ria' '{a`}ries' 'alla' 'ci{o'}' 'cions' 'n{c,}a' 'nces' '{o'}' 'dor' 'all' + 'il' '{i'}stic' 'enc' 'enca' '{i'}s' 'issa' 'issos' '{i'}ssem' '{i'}ssiu' 'issem' 'isseu' '{i'}sseu' + '{o'}s' 'osa' 'dora' 'dores' 'dors' 'adura' 'ble' 'bles' '{i'}vol' '{i'}vola' 'd{i'}s' 'egar' 'ejar' 'ificar' + 'itar' 'ables' 'adors' 'idores' 'idors' + 'adora' 'aci{o'}' 'doras' 'dur' 'dures' 'alleng{u"}es' + 'ant' 'ants' 'ancia' 'ancies' 'at{o`}ria' 'at{o`}ries' 'tori' 'toris' + 'ats' 'ions' 'ota' 'isam' 'ors' 'ora' 'ores' 'isament' + 'bilitat' 'bilitats' 'ivitat' 'ivitats' 'ari' 'aris' 'ionisme' 'ionista' 'ionistes' + 'ialista' 'ialistes' 'ialisme' 'ialismes' 'ud' 'uts' 'uds' 'encia' 'encies' '{e`}ncia' '{e`}ncies' + '{i"}tat' '{i"}tats' 'atiu' 'atius' 'atives' 'ativa' 'ativitat' 'ativitats' 'ible' 'ibles' + 'assa' 'asses' 'assos' + 'ent' 'ents' + '{i'}ssim' '{i'}ssima' '{i'}ssims' '{i'}ssimes' '{i`}ssem' '{i`}sseu' '{i`}ssin' + 'ims' 'ima' 'imes' + 'isme' 'ista' 'ismes' 'istes' + 'inia' 'inies' '{i'}inia' '{i'}nies' 'ita' 'ites' 'triu' 'trius' + 'oses' 'osos' 'ient' 'otes' 'ots' + (R1 delete) + 'acions' 'ada' 'ades' + (R2 delete) + 'log{i'}a' 'log{i'}es''logia' 'logies' 'logi' 'logis' 'l{o'}gica' 'l{o'}gics' 'l{o'}giques' + (R2 <- 'log') + 'ic' 'ica' 'ics' 'iques' + (R2 <- 'ic') + 'qu{i'}ssim' 'qu{i'}ssims' 'qu{i'}ssimes' 'qu{i'}ssima' + (R1 <- 'c') + ) + ) + + define verb_suffix as ( + [substring] among( + 'ador' 'adora' 'adors' 'adores' 're' 'ie' + 'ent' 'ents' 'udes' 'ar{a`}' 'eren' + 'ar{a'}' 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais' + 'aria' 'arian' 'arien' 'aries' 'ar{a`}s' + 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ara' + 'ar{e'}' 'ar{e'}s' + 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais' + 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}' + 'er{e'}' 'er' 'erau' 'erass' + 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais' + 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}' + 'ir{e'}' '{i'}rem' '{i'}reu' '{i'}eu' + 'ia' 'ies' '{i'}em' '{i`}eu' 'ien' + 'at' 'ut' 'uda' 'ava' 'aves' 'avem' '{a'}vem' '{a`}vem' '{a`}veu' '{a'}veu' 'aven' 'au' 'ats' + 'asseu' 'esseu' 'eresseu' '{a`}sseu' '{a`}ssem' '{a`}ssim' '{a`}ssiu' + 'essen' 'esses' 'assen' 'asses' 'assim' 'assiu' + '{e'}ssen' '{e'}sseu' '{e'}ssim' '{e'}ssiu' '{e'}ssem' + '{i'}' 'ares' '{a`}rem' '{a`}reu' '{a`}ren' + 'ar{i'}em' 'ar{i'}eu' + 'areu' 'aren' 'ant' '{i"}m' '{i"}u' + '{e'}s' '{i"}en' 'en' 'es' 'em' 'am' 'ams' '{i"}a' '{i"}es' + 'dre' 'eix' 'eixer' 'tzar' 'eixes' 'ides' '{i"}des' 'it' '{i"}t' '{i"}da' + 'aba' 'ada' 'ades' 'ida' '{i'}a' 'iera' 'ad' 'ed' 'its' + 'id' 'ids' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an' + 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado' + 'ido' 'iendo' 'i{o'}' 'ar' 'ir' 'as' + 'ieu' 'ii' 'io' 'i{a`}' + 'ess' 'essin' 'essis' 'ass' 'assin' 'assis' 'essim' '{e`}ssim' '{e`}ssiu' + 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases' + 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais' + 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados' + 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos' 'ques' + '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos' + 'ira' 'iran' 'irem' 'iren' 'ires' 'ireu' 'iria' 'irien' + 'iries' 'ir{a`}' 'ir{a`}s' 'ir{e`}' 'ir{i`}em' 'ir{i`}eu' + 'isquen' 'iguem' 'igueu' 'esqui' 'esquin' 'esquis' 'eixi' 'eixin' 'eixis' + 'eixen' 'eixo' 'isin' 'isis' 'esques' 'sis' 'sin' + 'int' 'ir{i'}em' 'ir{i'}eu' 'isc' 'atges' 'esca' 'esquen' + 'issen' 'isses' 'issin' 'issis' 'isca' 'issiu' 'issim' + '{i"}sc' '{i"}sca' '{i"}ssin' '{i'}ssiu' '{i'}ssim' '{i"}ssis' '{i"}guem' '{i"}gueu' + '{i"}ra' '{i"}ren' '{i"}res' + '{i"}squen' '{i"}sques' '{i"}ssen' '{i"}sses' '{i"}xo' '{i"}xen' '{i"}xes' '{i"}x' + 'ixo' 'ixen' 'ixes' 'ix' 'ixa' 'inin' 'inis' 'ini' 'ineu' 'itza' 'itzi' 'itzeu' 'itzis' + 'itzo' 'itz' 'itz{a`}' 'arem' 'in' '{a`}s' 'i{i"}' 'i{i"}n' 'i{i"}s' + (R1 delete) + 'ando' + (R2 delete) + ) + ) + + define residual_suffix as ( + [substring] among( + 'os' 'a' 'o' '{a'}' '{a`}' '{i'}' '{o'}' 'e' '{e'}' 'eu' 'iu' + 'is' 'i' 'ir' 's' '{i`}' 'itz' '{i"}' '{i"}n' '{i"}s' 'it' + (R1 delete) + 'iqu' + (R1 <- 'ic') + ) + ) +) + +define stem as ( + do mark_regions + backwards ( + do attached_pronoun + do ( standard_suffix or + verb_suffix + ) + do residual_suffix + ) + do cleaning +) + +/* + First works 2010/07/19 + First Grammatical Reviews: https://ca.wikipedia.org/wiki/Gram%C3%A0tica_del_catal%C3%A0 + Suffix list: https://ca.wikipedia.org/wiki/Llista_de_sufixos + Irregular Verbs: https://ca.wikipedia.org/wiki/Flexi%C3%B3_verbal_del_catal%C3%A0 +*/ diff --git a/contrib/snowball/algorithms/danish.sbl b/contrib/snowball/algorithms/danish.sbl new file mode 100644 index 0000000..761270f --- /dev/null +++ b/contrib/snowball/algorithms/danish.sbl @@ -0,0 +1,93 @@ +routines ( + mark_regions + main_suffix + consonant_pair + other_suffix + undouble +) + +externals ( stem ) + +strings ( ch ) + +integers ( p1 x ) + +groupings ( c v s_ending ) + +stringescapes {} + +/* special characters */ + +stringdef ae '{U+00E6}' +stringdef ao '{U+00E5}' +stringdef o/ '{U+00F8}' + +define c 'bcdfghjklmnpqrstvwxz' + +define v 'aeiouy{ae}{ao}{o/}' + +define s_ending 'abcdfghjklmnoprtvyz{ao}' + +define mark_regions as ( + + $p1 = limit + + test ( hop 3 setmark x ) + goto v gopast non-v setmark p1 + try ( $p1 < x $p1 = x ) +) + +backwardmode ( + + define main_suffix as ( + setlimit tomark p1 for ([substring]) + among( + + 'hed' 'ethed' 'ered' 'e' 'erede' 'ende' 'erende' 'ene' 'erne' 'ere' + 'en' 'heden' 'eren' 'er' 'heder' 'erer' 'heds' 'es' 'endes' + 'erendes' 'enes' 'ernes' 'eres' 'ens' 'hedens' 'erens' 'ers' 'ets' + 'erets' 'et' 'eret' + (delete) + 's' + (s_ending delete) + ) + ) + + define consonant_pair as ( + test ( + setlimit tomark p1 for ([substring]) + among( + 'gd' // significant in the call from other_suffix + 'dt' 'gt' 'kt' + ) + ) + next] delete + ) + + define other_suffix as ( + do ( ['st'] 'ig' delete ) + setlimit tomark p1 for ([substring]) + among( + 'ig' 'lig' 'elig' 'els' + (delete do consonant_pair) + 'l{o/}st' + (<-'l{o/}s') + ) + ) + define undouble as ( + setlimit tomark p1 for ([c] ->ch) + ch + delete + ) +) + +define stem as ( + + do mark_regions + backwards ( + do main_suffix + do consonant_pair + do other_suffix + do undouble + ) +) diff --git a/contrib/snowball/algorithms/dutch.sbl b/contrib/snowball/algorithms/dutch.sbl new file mode 100644 index 0000000..f24c82d --- /dev/null +++ b/contrib/snowball/algorithms/dutch.sbl @@ -0,0 +1,164 @@ +routines ( + prelude postlude + e_ending + en_ending + mark_regions + R1 R2 + undouble + standard_suffix +) + +externals ( stem ) + +booleans ( e_found ) + +integers ( p1 p2 ) + +groupings ( v v_I v_j ) + +stringescapes {} + +/* special characters */ + +stringdef a" '{U+00E4}' +stringdef e" '{U+00EB}' +stringdef i" '{U+00EF}' +stringdef o" '{U+00F6}' +stringdef u" '{U+00FC}' + +stringdef a' '{U+00E1}' +stringdef e' '{U+00E9}' +stringdef i' '{U+00ED}' +stringdef o' '{U+00F3}' +stringdef u' '{U+00FA}' + +stringdef e` '{U+00E8}' + +define v 'aeiouy{e`}' +define v_I v + 'I' +define v_j v + 'j' + +define prelude as ( + test repeat ( + [substring] among( + '{a"}' '{a'}' + (<- 'a') + '{e"}' '{e'}' + (<- 'e') + '{i"}' '{i'}' + (<- 'i') + '{o"}' '{o'}' + (<- 'o') + '{u"}' '{u'}' + (<- 'u') + '' (next) + ) //or next + ) + try(['y'] <- 'Y') + repeat goto ( + v [('i'] v <- 'I') or + ('y'] <- 'Y') + ) +) + +define mark_regions as ( + + $p1 = limit + $p2 = limit + + gopast v gopast non-v setmark p1 + try($p1 < 3 $p1 = 3) // at least 3 + gopast v gopast non-v setmark p2 + +) + +define postlude as repeat ( + + [substring] among( + 'Y' (<- 'y') + 'I' (<- 'i') + '' (next) + ) //or next + +) + +backwardmode ( + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define undouble as ( + test among('kk' 'dd' 'tt') [next] delete + ) + + define e_ending as ( + unset e_found + ['e'] R1 test non-v delete + set e_found + undouble + ) + + define en_ending as ( + R1 non-v and not 'gem' delete + undouble + ) + + define standard_suffix as ( + do ( + [substring] among( + 'heden' + ( R1 <- 'heid' + ) + 'en' 'ene' + ( en_ending + ) + 's' 'se' + ( R1 non-v_j delete + ) + ) + ) + do e_ending + + do ( ['heid'] R2 not 'c' delete + ['en'] en_ending + ) + + do ( + [substring] among( + 'end' 'ing' + ( R2 delete + (['ig'] R2 not 'e' delete) or undouble + ) + 'ig' + ( R2 not 'e' delete + ) + 'lijk' + ( R2 delete e_ending + ) + 'baar' + ( R2 delete + ) + 'bar' + ( R2 e_found delete + ) + ) + ) + do ( + non-v_I + test ( + among ('aa' 'ee' 'oo' 'uu') + non-v + ) + [next] delete + ) + ) +) + +define stem as ( + + do prelude + do mark_regions + backwards + do standard_suffix + do postlude +) diff --git a/contrib/snowball/algorithms/english.sbl b/contrib/snowball/algorithms/english.sbl new file mode 100644 index 0000000..fe18d7a --- /dev/null +++ b/contrib/snowball/algorithms/english.sbl @@ -0,0 +1,229 @@ +integers ( p1 p2 ) +booleans ( Y_found ) + +routines ( + prelude postlude + mark_regions + shortv + R1 R2 + Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5 + exception1 + exception2 +) + +externals ( stem ) + +groupings ( v v_WXY valid_LI ) + +stringescapes {} + +define v 'aeiouy' +define v_WXY v + 'wxY' + +define valid_LI 'cdeghkmnrt' + +define prelude as ( + unset Y_found + do ( ['{'}'] delete) + do ( ['y'] <-'Y' set Y_found) + do repeat(goto (v ['y']) <-'Y' set Y_found) +) + +define mark_regions as ( + $p1 = limit + $p2 = limit + do( + among ( + 'gener' + 'commun' // added May 2005 + 'arsen' // added Nov 2006 (arsenic/arsenal) + // ... extensions possible here ... + ) or (gopast v gopast non-v) + setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +backwardmode ( + + define shortv as ( + ( non-v_WXY v non-v ) + or + ( non-v v atlimit ) + ) + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define Step_1a as ( + try ( + [substring] among ( + '{'}' '{'}s' '{'}s{'}' + (delete) + ) + ) + [substring] among ( + 'sses' (<-'ss') + 'ied' 'ies' + ((hop 2 <-'i') or <-'ie') + 's' (next gopast v delete) + 'us' 'ss' + ) + ) + + define Step_1b as ( + [substring] among ( + 'eed' 'eedly' + (R1 <-'ee') + 'ed' 'edly' 'ing' 'ingly' + ( + test gopast v delete + test substring among( + 'at' 'bl' 'iz' + (<+ 'e') + 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt' + // ignoring double c, h, j, k, q, v, w, and x + ([next] delete) + '' (atmark p1 test shortv <+ 'e') + ) + ) + ) + ) + + define Step_1c as ( + ['y' or 'Y'] + non-v not atlimit + <-'i' + ) + + define Step_2 as ( + [substring] R1 among ( + 'tional' (<-'tion') + 'enci' (<-'ence') + 'anci' (<-'ance') + 'abli' (<-'able') + 'entli' (<-'ent') + 'izer' 'ization' + (<-'ize') + 'ational' 'ation' 'ator' + (<-'ate') + 'alism' 'aliti' 'alli' + (<-'al') + 'fulness' (<-'ful') + 'ousli' 'ousness' + (<-'ous') + 'iveness' 'iviti' + (<-'ive') + 'biliti' 'bli' + (<-'ble') + 'ogi' ('l' <-'og') + 'fulli' (<-'ful') + 'lessli' (<-'less') + 'li' (valid_LI delete) + ) + ) + + define Step_3 as ( + [substring] R1 among ( + 'tional' (<- 'tion') + 'ational' (<- 'ate') + 'alize' (<-'al') + 'icate' 'iciti' 'ical' + (<-'ic') + 'ful' 'ness' + (delete) + 'ative' + (R2 delete) // 'R2' added Dec 2001 + ) + ) + + define Step_4 as ( + [substring] R2 among ( + 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement' + 'ment' 'ent' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize' + (delete) + 'ion' ('s' or 't' delete) + ) + ) + + define Step_5 as ( + [substring] among ( + 'e' (R2 or (R1 not shortv) delete) + 'l' (R2 'l' delete) + ) + ) + + define exception2 as ( + + [substring] atlimit among( + 'inning' 'outing' 'canning' 'herring' 'earring' + 'proceed' 'exceed' 'succeed' + + // ... extensions possible here ... + + ) + ) +) + +define exception1 as ( + + [substring] atlimit among( + + /* special changes: */ + + 'skis' (<-'ski') + 'skies' (<-'sky') + 'dying' (<-'die') + 'lying' (<-'lie') + 'tying' (<-'tie') + + /* special -LY cases */ + + 'idly' (<-'idl') + 'gently' (<-'gentl') + 'ugly' (<-'ugli') + 'early' (<-'earli') + 'only' (<-'onli') + 'singly' (<-'singl') + + // ... extensions possible here ... + + /* invariant forms: */ + + 'sky' + 'news' + 'howe' + + 'atlas' 'cosmos' 'bias' 'andes' // not plural forms + + // ... extensions possible here ... + ) +) + +define postlude as (Y_found repeat(goto (['Y']) <-'y')) + +define stem as ( + + exception1 or + not hop 3 or ( + do prelude + do mark_regions + backwards ( + + do Step_1a + + exception2 or ( + + do Step_1b + do Step_1c + + do Step_2 + do Step_3 + do Step_4 + + do Step_5 + ) + ) + do postlude + ) +) diff --git a/contrib/snowball/algorithms/finnish.sbl b/contrib/snowball/algorithms/finnish.sbl new file mode 100644 index 0000000..3891d22 --- /dev/null +++ b/contrib/snowball/algorithms/finnish.sbl @@ -0,0 +1,197 @@ + +/* Finnish stemmer. + + Numbers in square brackets refer to the sections in + Fred Karlsson, Finnish: An Essential Grammar. Routledge, 1999 + ISBN 0-415-20705-3 + +*/ + +routines ( + mark_regions + R2 + particle_etc possessive + LONG VI + case_ending + i_plural + t_plural + other_endings + tidy +) + +externals ( stem ) + +integers ( p1 p2 ) +strings ( x ) +booleans ( ending_removed ) +groupings ( AEI C V1 V2 particle_end ) + +stringescapes {} + +/* special characters */ + +stringdef a" '{U+00E4}' +stringdef o" '{U+00F6}' + +define AEI 'a{a"}ei' +define C 'bcdfghjklmnpqrstvwxz' +define V1 'aeiouy{a"}{o"}' +define V2 'aeiou{a"}{o"}' +define particle_end V1 + 'nt' + +define mark_regions as ( + + $p1 = limit + $p2 = limit + + goto V1 gopast non-V1 setmark p1 + goto V1 gopast non-V1 setmark p2 +) + +backwardmode ( + + define R2 as $p2 <= cursor + + define particle_etc as ( + setlimit tomark p1 for ([substring]) + among( + 'kin' + 'kaan' 'k{a"}{a"}n' + 'ko' 'k{o"}' + 'han' 'h{a"}n' + 'pa' 'p{a"}' // Particles [91] + (particle_end) + 'sti' // Adverb [87] + (R2) + ) + delete + ) + define possessive as ( // [36] + setlimit tomark p1 for ([substring]) + among( + 'si' + (not 'k' delete) // take 'ksi' as the Comitative case + 'ni' + (delete ['kse'] <- 'ksi') // kseni = ksi + ni + 'nsa' 'ns{a"}' + 'mme' + 'nne' + (delete) + /* Now for Vn possessives after case endings: [36] */ + 'an' + (among('ta' 'ssa' 'sta' 'lla' 'lta' 'na') delete) + '{a"}n' + (among('t{a"}' 'ss{a"}' 'st{a"}' + 'll{a"}' 'lt{a"}' 'n{a"}') delete) + 'en' + (among('lle' 'ine') delete) + ) + ) + + define LONG as + among('aa' 'ee' 'ii' 'oo' 'uu' '{a"}{a"}' '{o"}{o"}') + + define VI as ('i' V2) + + define case_ending as ( + setlimit tomark p1 for ([substring]) + among( + 'han' ('a') //-. + 'hen' ('e') // | + 'hin' ('i') // | + 'hon' ('o') // | + 'h{a"}n' ('{a"}') // Illative [43] + 'h{o"}n' ('{o"}') // | + 'siin' VI // | + 'seen' LONG //-' + + 'den' VI + 'tten' VI // Genitive plurals [34] + () + 'n' // Genitive or Illative + ( try ( LONG // Illative + or 'ie' // Genitive + and next ] + ) + /* otherwise Genitive */ + ) + + 'a' '{a"}' //-. + (V1 C) // | + 'tta' 'tt{a"}' // Partitive [32] + ('e') // | + 'ta' 't{a"}' //-' + + 'ssa' 'ss{a"}' // Inessive [41] + 'sta' 'st{a"}' // Elative [42] + + 'lla' 'll{a"}' // Adessive [44] + 'lta' 'lt{a"}' // Ablative [51] + 'lle' // Allative [46] + 'na' 'n{a"}' // Essive [49] + 'ksi' // Translative[50] + 'ine' // Comitative [51] + + /* Abessive and Instructive are too rare for + inclusion [51] */ + + ) + delete + set ending_removed + ) + define other_endings as ( + setlimit tomark p2 for ([substring]) + among( + 'mpi' 'mpa' 'mp{a"}' + 'mmi' 'mma' 'mm{a"}' // Comparative forms [85] + (not 'po') //-improves things + 'impi' 'impa' 'imp{a"}' + 'immi' 'imma' 'imm{a"}' // Superlative forms [86] + 'eja' 'ej{a"}' // indicates agent [93.1B] + ) + delete + ) + define i_plural as ( // [26] + setlimit tomark p1 for ([substring]) + among( + 'i' 'j' + ) + delete + ) + define t_plural as ( // [26] + setlimit tomark p1 for ( + ['t'] test V1 + delete + ) + setlimit tomark p2 for ([substring]) + among( + 'mma' (not 'po') //-mmat endings + 'imma' //-immat endings + ) + delete + ) + define tidy as ( + setlimit tomark p1 for ( + do ( LONG and ([next] delete ) ) // undouble vowel + do ( [AEI] C delete ) // remove trailing a, a", e, i + do ( ['j'] 'o' or 'u' delete ) + do ( ['o'] 'j' delete ) + ) + goto non-V1 [C] -> x x delete // undouble consonant + ) +) + +define stem as ( + + do mark_regions + unset ending_removed + backwards ( + do particle_etc + do possessive + do case_ending + do other_endings + (ending_removed do i_plural) or do t_plural + do tidy + ) +) + diff --git a/contrib/snowball/algorithms/french.sbl b/contrib/snowball/algorithms/french.sbl new file mode 100644 index 0000000..5c4f32d --- /dev/null +++ b/contrib/snowball/algorithms/french.sbl @@ -0,0 +1,254 @@ +routines ( + prelude postlude mark_regions + RV R1 R2 + standard_suffix + i_verb_suffix + verb_suffix + residual_suffix + un_double + un_accent +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v keep_with_s ) + +stringescapes {} + +/* special characters */ + +stringdef a^ '{U+00E2}' // a-circumflex +stringdef a` '{U+00E0}' // a-grave +stringdef c, '{U+00E7}' // c-cedilla + +stringdef e" '{U+00EB}' // e-diaeresis (rare) +stringdef e' '{U+00E9}' // e-acute +stringdef e^ '{U+00EA}' // e-circumflex +stringdef e` '{U+00E8}' // e-grave +stringdef i" '{U+00EF}' // i-diaeresis +stringdef i^ '{U+00EE}' // i-circumflex +stringdef o^ '{U+00F4}' // o-circumflex +stringdef u^ '{U+00FB}' // u-circumflex +stringdef u` '{U+00F9}' // u-grave + +define v 'aeiouy{a^}{a`}{e"}{e'}{e^}{e`}{i"}{i^}{o^}{u^}{u`}' + +define prelude as repeat goto ( + + ( v [ ('u' ] v <- 'U') or + ('i' ] v <- 'I') or + ('y' ] <- 'Y') + ) + or + ( [ '{e"}' ] <- 'He' ) + or + ( [ '{i"}' ] <- 'Hi' ) + or + ( ['y'] v <- 'Y' ) + or + ( 'q' ['u'] <- 'U' ) +) + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v v next ) + or + among ( // this exception list begun Nov 2006 + 'par' // paris, parie, pari + 'col' // colis + 'tap' // tapis + // extensions possible here + ) + or + ( next gopast v ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define postlude as repeat ( + + [substring] among( + 'I' (<- 'i') + 'U' (<- 'u') + 'Y' (<- 'y') + 'He' (<- '{e"}') + 'Hi' (<- '{i"}') + 'H' (delete) + '' (next) + ) +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define standard_suffix as ( + [substring] among( + + 'ance' 'iqUe' 'isme' 'able' 'iste' 'eux' + 'ances' 'iqUes' 'ismes' 'ables' 'istes' + ( R2 delete ) + 'atrice' 'ateur' 'ation' + 'atrices' 'ateurs' 'ations' + ( R2 delete + try ( ['ic'] (R2 delete) or <-'iqU' ) + ) + 'logie' + 'logies' + ( R2 <- 'log' ) + 'usion' 'ution' + 'usions' 'utions' + ( R2 <- 'u' ) + 'ence' + 'ences' + ( R2 <- 'ent' ) + 'ement' + 'ements' + ( + RV delete + try ( + [substring] among( + 'iv' (R2 delete ['at'] R2 delete) + 'eus' ((R2 delete) or (R1<-'eux')) + 'abl' 'iqU' + (R2 delete) + 'i{e`}r' 'I{e`}r' //) + (RV <-'i') //)--new 2 Sept 02 + ) + ) + ) + 'it{e'}' + 'it{e'}s' + ( + R2 delete + try ( + [substring] among( + 'abil' ((R2 delete) or <-'abl') + 'ic' ((R2 delete) or <-'iqU') + 'iv' (R2 delete) + ) + ) + ) + 'if' 'ive' + 'ifs' 'ives' + ( + R2 delete + try ( ['at'] R2 delete ['ic'] (R2 delete) or <-'iqU' ) + ) + 'eaux' (<- 'eau') + 'aux' (R1 <- 'al') + 'euse' + 'euses'((R2 delete) or (R1<-'eux')) + + 'issement' + 'issements'(R1 non-v delete) // verbal + + // fail(...) below forces entry to verb_suffix. -ment typically + // follows the p.p., e.g 'confus{e'}ment'. + + 'amment' (RV fail(<- 'ant')) + 'emment' (RV fail(<- 'ent')) + 'ment' + 'ments' (test(v RV) fail(delete)) + // v is e,i,u,{e'},I or U + ) + ) + + define i_verb_suffix as setlimit tomark pV for ( + [substring] among ( + '{i^}mes' '{i^}t' '{i^}tes' 'i' 'ie' 'ies' 'ir' 'ira' 'irai' + 'iraIent' 'irais' 'irait' 'iras' 'irent' 'irez' 'iriez' + 'irions' 'irons' 'iront' 'is' 'issaIent' 'issais' 'issait' + 'issant' 'issante' 'issantes' 'issants' 'isse' 'issent' 'isses' + 'issez' 'issiez' 'issions' 'issons' 'it' + (not 'H' non-v delete) + ) + ) + + define verb_suffix as setlimit tomark pV for ( + [substring] among ( + 'ions' + (R2 delete) + + '{e'}' '{e'}e' '{e'}es' '{e'}s' '{e`}rent' 'er' 'era' 'erai' + 'eraIent' 'erais' 'erait' 'eras' 'erez' 'eriez' 'erions' + 'erons' 'eront' 'ez' 'iez' + + // 'ons' //-best omitted + + (delete) + + '{a^}mes' '{a^}t' '{a^}tes' 'a' 'ai' 'aIent' 'ais' 'ait' 'ant' + 'ante' 'antes' 'ants' 'as' 'asse' 'assent' 'asses' 'assiez' + 'assions' + (delete + try(['e'] delete) + ) + ) + ) + + define keep_with_s 'aiou{e`}s' + + define residual_suffix as ( + try(['s'] test ('Hi' or non-keep_with_s) delete) + setlimit tomark pV for ( + [substring] among( + 'ion' (R2 's' or 't' delete) + 'ier' 'i{e`}re' + 'Ier' 'I{e`}re' (<-'i') + 'e' (delete) + ) + ) + ) + + define un_double as ( + test among('enn' 'onn' 'ett' 'ell' 'eill') [next] delete + ) + + define un_accent as ( + atleast 1 non-v + [ '{e'}' or '{e`}' ] <-'e' + ) +) + +define stem as ( + + do prelude + do mark_regions + backwards ( + + do ( + ( + ( standard_suffix or + i_verb_suffix or + verb_suffix + ) + and + try( [ ('Y' ] <- 'i' ) or + ('{c,}'] <- 'c' ) + ) + ) or + residual_suffix + ) + + // try(['ent'] RV delete) // is best omitted + + do un_double + do un_accent + ) + do postlude +) + diff --git a/contrib/snowball/algorithms/german.sbl b/contrib/snowball/algorithms/german.sbl new file mode 100644 index 0000000..61f24ef --- /dev/null +++ b/contrib/snowball/algorithms/german.sbl @@ -0,0 +1,139 @@ + +/* + Extra rule for -nisse ending added 11 Dec 2009 +*/ + +routines ( + prelude postlude + mark_regions + R1 R2 + standard_suffix +) + +externals ( stem ) + +integers ( p1 p2 x ) + +groupings ( v s_ending st_ending ) + +stringescapes {} + +/* special characters */ + +stringdef a" '{U+00E4}' +stringdef o" '{U+00F6}' +stringdef u" '{U+00FC}' +stringdef ss '{U+00DF}' + +define v 'aeiouy{a"}{o"}{u"}' + +define s_ending 'bdfghklmnrt' +define st_ending s_ending - 'r' + +define prelude as ( + + test repeat ( + ( + ['{ss}'] <- 'ss' + ) or next + ) + + repeat goto ( + v [('u'] v <- 'U') or + ('y'] v <- 'Y') + ) +) + +define mark_regions as ( + + $p1 = limit + $p2 = limit + + test(hop 3 setmark x) + + gopast v gopast non-v setmark p1 + try($p1 < x $p1 = x) // at least 3 + gopast v gopast non-v setmark p2 + +) + +define postlude as repeat ( + + [substring] among( + 'Y' (<- 'y') + 'U' (<- 'u') + '{a"}' (<- 'a') + '{o"}' (<- 'o') + '{u"}' (<- 'u') + '' (next) + ) + +) + +backwardmode ( + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define standard_suffix as ( + do ( + [substring] R1 among( + 'em' 'ern' 'er' + ( delete + ) + 'e' 'en' 'es' + ( delete + try (['s'] 'nis' delete) + ) + 's' + ( s_ending delete + ) + ) + ) + do ( + [substring] R1 among( + 'en' 'er' 'est' + ( delete + ) + 'st' + ( st_ending hop 3 delete + ) + ) + ) + do ( + [substring] R2 among( + 'end' 'ung' + ( delete + try (['ig'] not 'e' R2 delete) + ) + 'ig' 'ik' 'isch' + ( not 'e' delete + ) + 'lich' 'heit' + ( delete + try ( + ['er' or 'en'] R1 delete + ) + ) + 'keit' + ( delete + try ( + [substring] R2 among( + 'lich' 'ig' + ( delete + ) + ) + ) + ) + ) + ) + ) +) + +define stem as ( + do prelude + do mark_regions + backwards + do standard_suffix + do postlude +) diff --git a/contrib/snowball/algorithms/german2.sbl b/contrib/snowball/algorithms/german2.sbl new file mode 100644 index 0000000..47ff61e --- /dev/null +++ b/contrib/snowball/algorithms/german2.sbl @@ -0,0 +1,145 @@ + +/* + Extra rule for -nisse ending added 11 Dec 2009 +*/ + +routines ( + prelude postlude + mark_regions + R1 R2 + standard_suffix +) + +externals ( stem ) + +integers ( p1 p2 x ) + +groupings ( v s_ending st_ending ) + +stringescapes {} + +/* special characters */ + +stringdef a" '{U+00E4}' +stringdef o" '{U+00F6}' +stringdef u" '{U+00FC}' +stringdef ss '{U+00DF}' + +define v 'aeiouy{a"}{o"}{u"}' + +define s_ending 'bdfghklmnrt' +define st_ending s_ending - 'r' + +define prelude as ( + + test repeat goto ( + v [('u'] v <- 'U') or + ('y'] v <- 'Y') + ) + + repeat ( + [substring] among( + '{ss}' (<- 'ss') + 'ae' (<- '{a"}') + 'oe' (<- '{o"}') + 'ue' (<- '{u"}') + 'qu' (hop 2) + '' (next) + ) + ) + +) + +define mark_regions as ( + + $p1 = limit + $p2 = limit + + test(hop 3 setmark x) + + gopast v gopast non-v setmark p1 + try($p1 < x $p1 = x) // at least 3 + gopast v gopast non-v setmark p2 + +) + +define postlude as repeat ( + + [substring] among( + 'Y' (<- 'y') + 'U' (<- 'u') + '{a"}' (<- 'a') + '{o"}' (<- 'o') + '{u"}' (<- 'u') + '' (next) + ) + +) + +backwardmode ( + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define standard_suffix as ( + do ( + [substring] R1 among( + 'em' 'ern' 'er' + ( delete + ) + 'e' 'en' 'es' + ( delete + try (['s'] 'nis' delete) + ) + 's' + ( s_ending delete + ) + ) + ) + do ( + [substring] R1 among( + 'en' 'er' 'est' + ( delete + ) + 'st' + ( st_ending hop 3 delete + ) + ) + ) + do ( + [substring] R2 among( + 'end' 'ung' + ( delete + try (['ig'] not 'e' R2 delete) + ) + 'ig' 'ik' 'isch' + ( not 'e' delete + ) + 'lich' 'heit' + ( delete + try ( + ['er' or 'en'] R1 delete + ) + ) + 'keit' + ( delete + try ( + [substring] R2 among( + 'lich' 'ig' + ( delete + ) + ) + ) + ) + ) + ) + ) +) + +define stem as ( + do prelude + do mark_regions + backwards + do standard_suffix + do postlude +) diff --git a/contrib/snowball/algorithms/greek.sbl b/contrib/snowball/algorithms/greek.sbl new file mode 100644 index 0000000..02df6c3 --- /dev/null +++ b/contrib/snowball/algorithms/greek.sbl @@ -0,0 +1,706 @@ +// A stemmer for Modern Greek language, based on: +// +// Ntais, Georgios. Development of a Stemmer for the Greek +// Language. Diss. Royal Institute of Technology, 2006. +// https://sais.se/mthprize/2007/ntais2007.pdf +// +// Saroukos, Spyridon. Enhancing a Greek language stemmer. +// University of Tampere, 2008. +// https://tampub.uta.fi/bitstream/handle/10024/80480/gradu03463.pdf + +stringescapes {} + +stringdef a '{U+03B1}' // alpha +stringdef v '{U+03B2}' // beta +stringdef g '{U+03B3}' // gamma +stringdef d '{U+03B4}' // delta +stringdef e '{U+03B5}' // epsilon +stringdef z '{U+03B6}' // zeta +stringdef i '{U+03B7}' // eta +stringdef th '{U+03B8}' // theta +stringdef y '{U+03B9}' // iota +stringdef k '{U+03BA}' // kappa +stringdef l '{U+03BB}' // lamda +stringdef m '{U+03BC}' // mu +stringdef n '{U+03BD}' // nu +stringdef x '{U+03BE}' // xi +stringdef o '{U+03BF}' // omicron +stringdef p '{U+03C0}' // pi +stringdef r '{U+03C1}' // rho +stringdef ss '{U+03C2}' // sigma final +stringdef s '{U+03C3}' // sigma +stringdef t '{U+03C4}' // tau +stringdef u '{U+03C5}' // upsilon +stringdef f '{U+03C6}' // phi +stringdef ch '{U+03C7}' // chi +stringdef ps '{U+03C8}' // psi +stringdef oo '{U+03C9}' // omega + +stringdef A '{U+0391}' // Alpha +stringdef V '{U+0392}' // Beta +stringdef G '{U+0393}' // Gamma +stringdef D '{U+0394}' // Delta +stringdef E '{U+0395}' // Epsilon +stringdef Z '{U+0396}' // Zeta +stringdef I '{U+0397}' // Eta +stringdef Th '{U+0398}' // Theta +stringdef Y '{U+0399}' // Iota +stringdef K '{U+039A}' // Kappa +stringdef L '{U+039B}' // Lamda +stringdef M '{U+039C}' // Mu +stringdef N '{U+039D}' // Nu +stringdef X '{U+039E}' // Xi +stringdef O '{U+039F}' // Omicron +stringdef P '{U+03A0}' // Pi +stringdef R '{U+03A1}' // Rho +stringdef S '{U+03A3}' // Sigma +stringdef T '{U+03A4}' // Tau +stringdef U '{U+03A5}' // Upsilon +stringdef F '{U+03A6}' // Phi +stringdef Ch '{U+03A7}' // Chi +stringdef Ps '{U+03A8}' // Psi +stringdef Oo '{U+03A9}' // Omega + +stringdef Y: '{U+03AA}' // Iota with dialytika +stringdef U: '{U+03AB}' // Upsilon with dialytika + +stringdef a' '{U+03AC}' // alpha with tonos +stringdef e' '{U+03AD}' // epsilon with tonos +stringdef i' '{U+03AE}' // eta with tonos +stringdef y' '{U+03AF}' // iota with tonos +stringdef o' '{U+03CC}' // omicron with tonos +stringdef u' '{U+03CD}' // upsilon with tonos +stringdef oo' '{U+03CE}' // omega with tonos + +stringdef i:' '{U+0390}' // iota with dialytika and tonos +stringdef u:' '{U+03B0}' // upsilon with dialytika and tonos + +stringdef i: '{U+03CA}' // iota with dialytika +stringdef u: '{U+03CB}' // upsilon with dialytika + +stringdef A' '{U+0386}' // Alpha with tonos +stringdef E' '{U+0388}' // Epsilon with tonos +stringdef I' '{U+0389}' // Eta with tonos +stringdef Y' '{U+038A}' // Iota with tonos +stringdef O' '{U+038C}' // Omicron with tonos +stringdef U' '{U+038E}' // Upsilon with tonos +stringdef OO' '{U+038F}' // Omega with tonos + +externals ( stem ) + +booleans ( test1 ) + +groupings ( v v2 ) + +routines ( tolower has_min_length + steps1 steps2 steps3 steps4 steps5 steps6 steps7 + steps8 steps9 steps10 + step1 step2a step2b step2c step2d step3 step4 + step5a step5b step5c step5d step5e step5f + step5g step5h step5i + step5j step5k step5l step5m + step6 step7 ) + +define v '{a}{e}{i}{y}{o}{u}{oo}' +define v2 '{a}{e}{i}{y}{o}{oo}' + +backwardmode ( + define has_min_length as ( + $(len >= 3) + ) + + define tolower as ( + repeat ( + [substring] among ( + '{A}' (<- '{a}') + '{V}' (<- '{v}') + '{G}' (<- '{g}') + '{D}' (<- '{d}') + '{E}' (<- '{e}') + '{Z}' (<- '{z}') + '{I}' (<- '{i}') + '{Th}' (<- '{th}') + '{Y}' (<- '{y}') + '{K}' (<- '{k}') + '{L}' (<- '{l}') + '{M}' (<- '{m}') + '{N}' (<- '{n}') + '{X}' (<- '{x}') + '{O}' (<- '{o}') + '{P}' (<- '{p}') + '{R}' (<- '{r}') + '{S}' (<- '{s}') + '{T}' (<- '{t}') + '{U}' (<- '{u}') + '{F}' (<- '{f}') + '{Ch}' (<- '{ch}') + '{Ps}' (<- '{ps}') + '{Oo}' (<- '{oo}') + '{Y:}' (<- '{y}') + '{U:}' (<- '{u}') + '{a'}' (<- '{a}') + '{e'}' (<- '{e}') + '{i'}' (<- '{i}') + '{y'}' (<- '{y}') + '{o'}' (<- '{o}') + '{u'}' (<- '{u}') + '{oo'}' (<- '{oo}') + '{i:'}' (<- '{i}') + '{u:'}' (<- '{u}') + '{i:}' (<- '{i}') + '{u:}' (<- '{u}') + '{A'}' (<- '{a}') + '{E'}' (<- '{e}') + '{I'}' (<- '{i}') + '{Y'}' (<- '{y}') + '{O'}' (<- '{o}') + '{U'}' (<- '{u}') + '{OO'}' (<- '{oo}') + '{ss}' (<- '{s}') + '' (next) + ) + ) + ) + + define step1 as ( + [substring] among ( + '{f}{a}{g}{y}{a}' '{f}{a}{g}{y}{o}{u}' '{f}{a}{g}{y}{oo}{n}' (<- '{f}{a}') + '{s}{k}{a}{g}{y}{a}' '{s}{k}{a}{g}{y}{o}{u}' '{s}{k}{a}{g}{y}{oo}{n}' (<- '{s}{k}{a}') + '{o}{l}{o}{g}{y}{o}{u}' '{o}{l}{o}{g}{y}{a}' '{o}{l}{o}{g}{y}{oo}{n}' (<- '{o}{l}{o}') + '{s}{o}{g}{y}{o}{u}' '{s}{o}{g}{y}{a}' '{s}{o}{g}{y}{oo}{n}' (<- '{s}{o}') + '{t}{a}{t}{o}{g}{y}{a}' '{t}{a}{t}{o}{g}{y}{o}{u}' '{t}{a}{t}{o}{g}{y}{oo}{n}' (<- '{t}{a}{t}{o}') + '{k}{r}{e}{a}{s}' '{k}{r}{e}{a}{t}{o}{s}' '{k}{r}{e}{a}{t}{a}' '{k}{r}{e}{a}{t}{oo}{n}' (<- '{k}{r}{e}') + '{p}{e}{r}{a}{s}' '{p}{e}{r}{a}{t}{o}{s}' '{p}{e}{r}{a}{t}{i}' '{p}{e}{r}{a}{t}{a}' '{p}{e}{r}{a}{t}{oo}{n}' (<- '{p}{e}{r}') + '{t}{e}{r}{a}{s}' '{t}{e}{r}{a}{t}{o}{s}' '{t}{e}{r}{a}{t}{a}' '{t}{e}{r}{a}{t}{oo}{n}' (<- '{t}{e}{r}') + '{f}{oo}{s}' '{f}{oo}{t}{o}{s}' '{f}{oo}{t}{a}' '{f}{oo}{t}{oo}{n}' (<- '{f}{oo}') + '{k}{a}{th}{e}{s}{t}{oo}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{o}{s}' '{k}{a}{th}{e}{s}{t}{oo}{t}{a}' '{k}{a}{th}{e}{s}{t}{oo}{t}{oo}{n}' (<- '{k}{a}{th}{e}{s}{t}') + '{g}{e}{g}{o}{n}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{o}{s}' '{g}{e}{g}{o}{n}{o}{t}{a}' '{g}{e}{g}{o}{n}{o}{t}{oo}{n}' (<- '{g}{e}{g}{o}{n}') + ) + unset test1 + ) + + define steps1 as ( + [substring] among ( + '{y}{z}{a}' '{y}{z}{e}{s}' '{y}{z}{e}' '{y}{z}{a}{m}{e}' '{y}{z}{a}{t}{e}' '{y}{z}{a}{n}' '{y}{z}{a}{n}{e}' '{y}{z}{oo}' '{y}{z}{e}{y}{s}' '{y}{z}{e}{y}' + '{y}{z}{o}{u}{m}{e}' '{y}{z}{e}{t}{e}' '{y}{z}{o}{u}{n}' '{y}{z}{o}{u}{n}{e}' ( + delete + unset test1 + ([] substring atlimit among ( + '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{p}{a}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}' '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' + (<- '{y}') + )) or + ([] substring atlimit among ( + '{m}{a}{r}{k}' '{k}{o}{r}{n}' '{a}{m}{p}{a}{r}' '{a}{r}{r}' '{v}{a}{th}{u}{r}{y}' '{v}{a}{r}{k}' '{v}' '{v}{o}{l}{v}{o}{r}' '{g}{k}{r}' + '{g}{l}{u}{k}{o}{r}' '{g}{l}{u}{k}{u}{r}' '{y}{m}{p}' '{l}' '{l}{o}{u}' '{m}{a}{r}' '{m}' '{p}{r}' '{m}{p}{r}' '{p}{o}{l}{u}{r}' '{p}' + '{r}' '{p}{y}{p}{e}{r}{o}{r}' + (<- '{y}{z}') + )) + ) + ) + ) + + define steps2 as ( + [substring] among ( + '{oo}{th}{i}{k}{a}' '{oo}{th}{i}{k}{e}{s}' '{oo}{th}{i}{k}{e}' '{oo}{th}{i}{k}{a}{m}{e}' '{oo}{th}{i}{k}{a}{t}{e}' '{oo}{th}{i}{k}{a}{n}' '{oo}{th}{i}{k}{a}{n}{e}' ( + delete + unset test1 + [] substring atlimit among ( + '{a}{l}' '{v}{y}' '{e}{n}' '{u}{ps}' '{l}{y}' '{z}{oo}' '{s}' '{ch}' (<- '{oo}{n}') + ) + ) + ) + ) + + define steps3 as ( + [substring] among ( + '{y}{s}{a}' '{y}{s}{e}{s}' '{y}{s}{e}' '{y}{s}{a}{m}{e}' '{y}{s}{a}{t}{e}' '{y}{s}{a}{n}' '{y}{s}{a}{n}{e}' ( + delete + unset test1 + ('{y}{s}{a}' atlimit <- '{y}{s}') or + ([] substring atlimit among ( + '{a}{n}{a}{m}{p}{a}' '{a}{th}{r}{o}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}' + '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}' + '{a}{p}{e}{k}{l}{e}' '{e}{k}{l}{e}' '{p}{e}' + (<- '{y}') + )) or + ([] substring atlimit among ( + '{a}{n}' '{a}{f}' '{g}{e}' '{g}{y}{g}{a}{n}{t}{o}{a}{f}' '{g}{k}{e}' '{d}{i}{m}{o}{k}{r}{a}{t}' '{k}{o}{m}' '{g}{k}' '{m}' '{p}' + '{p}{o}{u}{k}{a}{m}' '{o}{l}{o}' '{l}{a}{r}' + (<- '{y}{s}') + )) + ) + ) + ) + + define steps4 as ( + [substring] among ( + '{y}{s}{oo}' '{y}{s}{e}{y}{s}' '{y}{s}{e}{y}' '{y}{s}{o}{u}{m}{e}' '{y}{s}{e}{t}{e}' '{y}{s}{o}{u}{n}' '{y}{s}{o}{u}{n}{e}' ( + delete + unset test1 + [] substring atlimit among ( + '{a}{n}{a}{m}{p}{a}' '{e}{m}{p}{a}' '{e}{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{e}{p}{a}' '{x}{a}{n}{a}{p}{a}' '{e}{p}{e}' '{p}{e}{r}{y}{p}{a}' '{a}{th}{r}{o}' + '{s}{u}{n}{a}{th}{r}{o}' '{d}{a}{n}{e}' '{k}{l}{e}' '{ch}{a}{r}{t}{o}{p}{a}' '{e}{x}{a}{r}{ch}{a}' '{m}{e}{t}{e}{p}{e}' '{a}{p}{o}{k}{l}{e}' '{a}{p}{e}{k}{l}{e}' + '{e}{k}{l}{e}' '{p}{e}' + (<- '{y}') + ) + ) + ) + ) + + define steps5 as ( + [substring] among ( + '{y}{s}{t}{o}{s}' '{y}{s}{t}{o}{u}' '{y}{s}{t}{o}' '{y}{s}{t}{e}' '{y}{s}{t}{o}{y}' '{y}{s}{t}{oo}{n}' '{y}{s}{t}{o}{u}{s}' '{y}{s}{t}{i}' '{y}{s}{t}{i}{s}' + '{y}{s}{t}{a}' '{y}{s}{t}{e}{s}' ( + delete + unset test1 + ([] substring atlimit among ( + '{d}{a}{n}{e}' '{s}{u}{n}{a}{th}{r}{o}' '{k}{l}{e}' '{s}{e}' '{e}{s}{oo}{k}{l}{e}' '{a}{s}{e}' '{p}{l}{e}' + (<- '{y}') + )) or + ([] substring atlimit among ( + '{m}' '{p}' '{a}{p}' '{a}{r}' '{i}{d}' '{k}{t}' '{s}{k}' '{s}{ch}' '{u}{ps}' '{f}{a}' '{ch}{r}' '{ch}{t}' '{a}{k}{t}' + '{a}{o}{r}' '{a}{s}{ch}' '{a}{t}{a}' '{a}{ch}{n}' '{a}{ch}{t}' '{g}{e}{m}' '{g}{u}{r}' '{e}{m}{p}' '{e}{u}{p}' '{e}{ch}{th}' '{i}{f}{a}' + '{k}{a}{th}' '{k}{a}{k}' '{k}{u}{l}' '{l}{u}{g}' '{m}{a}{k}' '{m}{e}{g}' '{t}{a}{ch}' '{f}{y}{l}' '{ch}{oo}{r}' + (<- '{y}{s}{t}') + )) + ) + ) + ) + + define steps6 as ( + [substring] among ( + '{y}{s}{m}{o}' '{y}{s}{m}{o}{y}' '{y}{s}{m}{o}{s}' '{y}{s}{m}{o}{u}' '{y}{s}{m}{o}{u}{s}' '{y}{s}{m}{oo}{n}' ( + delete + unset test1 + ([] substring atlimit among ( + '{s}{e}' '{m}{e}{t}{a}{s}{e}' '{m}{y}{k}{r}{o}{s}{e}' '{e}{g}{k}{l}{e}' '{a}{p}{o}{k}{l}{e}' + (<- '{y}{s}{m}') + )) or + ([] substring atlimit among ( + '{d}{a}{n}{e}' '{a}{n}{t}{y}{d}{a}{n}{e}' + (<- '{y}') + )) or + ([substring] among ( + '{a}{g}{n}{oo}{s}{t}{y}{k}' (<- '{a}{g}{n}{oo}{s}{t}') + '{a}{t}{o}{m}{y}{k}' (<- '{a}{t}{o}{m}') + '{g}{n}{oo}{s}{t}{y}{k}' (<- '{g}{n}{oo}{s}{t}') + '{e}{th}{n}{y}{k}' (<- '{e}{th}{n}') + '{e}{k}{l}{e}{k}{t}{y}{k}' (<- '{e}{k}{l}{e}{k}{t}') + '{s}{k}{e}{p}{t}{y}{k}' (<- '{s}{k}{e}{p}{t}') + '{t}{o}{p}{y}{k}' (<- '{t}{o}{p}') + '{a}{l}{e}{x}{a}{n}{d}{r}{y}{n}' (<- '{a}{l}{e}{x}{a}{n}{d}{r}') + '{v}{u}{z}{a}{n}{t}{y}{n}' (<- '{v}{u}{z}{a}{n}{t}') + '{th}{e}{a}{t}{r}{y}{n}' (<- '{th}{e}{a}{t}{r}') + )) + ) + ) + ) + + define steps7 as ( + [substring] among ( + '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' '{o}{u}{d}{a}{k}{y}' '{o}{u}{d}{a}{k}{y}{a}' ( + delete + unset test1 + [] substring atlimit among ( + '{s}' '{ch}' + (<- '{a}{r}{a}{k}') + ) + ) + ) + ) + + define steps8 as ( + [substring] among ( + '{a}{k}{y}' '{a}{k}{y}{a}' '{y}{t}{s}{a}' '{y}{t}{s}{a}{s}' '{y}{t}{s}{e}{s}' '{y}{t}{s}{oo}{n}' '{a}{r}{a}{k}{y}' '{a}{r}{a}{k}{y}{a}' ( + delete + unset test1 + ([] substring atlimit among ( + '{v}{a}{m}{v}' '{v}{r}' '{k}{a}{y}{m}' '{k}{o}{n}' '{k}{o}{r}' '{l}{a}{v}{r}' '{l}{o}{u}{l}' '{m}{e}{r}' '{m}{o}{u}{s}{t}' + '{n}{a}{g}{k}{a}{s}' '{p}{l}' '{r}' '{r}{u}' '{s}' '{s}{k}' '{s}{o}{k}' '{s}{p}{a}{n}' '{t}{z}' '{f}{a}{r}{m}' '{ch}' '{k}{a}{p}{a}{k}' + '{a}{l}{y}{s}{f}' '{a}{m}{v}{r}' '{a}{n}{th}{r}' '{k}' '{f}{u}{l}' '{k}{a}{t}{r}{a}{p}' '{k}{l}{y}{m}' '{m}{a}{l}' '{s}{l}{o}{v}' '{f}' + '{s}{f}' '{t}{s}{e}{ch}{o}{s}{l}{o}{v}' + (<- '{a}{k}') + )) or + ([] substring atlimit among ( + '{v}' '{v}{a}{l}' '{g}{y}{a}{n}' '{g}{l}' '{z}' '{i}{g}{o}{u}{m}{e}{n}' '{k}{a}{r}{d}' '{k}{o}{n}' '{m}{a}{k}{r}{u}{n}' '{n}{u}{f}' + '{p}{a}{t}{e}{r}' '{p}' '{s}{k}' '{t}{o}{s}' '{t}{r}{y}{p}{o}{l}' + (<- '{y}{t}{s}') + )) or + ([] '{k}{o}{r}' <- '{y}{t}{s}') + ) + ) + ) + + define steps9 as ( + [substring] among ( + '{y}{d}{y}{o}' '{y}{d}{y}{a}' '{y}{d}{y}{oo}{n}' ( + delete + unset test1 + ([] substring atlimit among ( + '{a}{y}{f}{n}' '{y}{r}' '{o}{l}{o}' '{ps}{a}{l}' (<- '{y}{d}') + )) or + ([] substring among ( + '{e}' '{p}{a}{y}{ch}{n}' (<- '{y}{d}') + )) + ) + ) + ) + + define steps10 as ( + [substring] among ( + '{y}{s}{k}{o}{s}' '{y}{s}{k}{o}{u}' '{y}{s}{k}{o}' '{y}{s}{k}{e}' ( + delete + unset test1 + [] substring atlimit among ( + '{d}' '{y}{v}' '{m}{i}{n}' '{r}' '{f}{r}{a}{g}{k}' '{l}{u}{k}' '{o}{v}{e}{l}' + (<- '{y}{s}{k}') + ) + ) + ) + ) + + define step2a as ( + [substring] among ( + '{a}{d}{e}{s}' '{a}{d}{oo}{n}' (delete) + ) + not ([substring] among ( + '{o}{k}' '{m}{a}{m}' '{m}{a}{n}' '{m}{p}{a}{m}{p}' '{p}{a}{t}{e}{r}' '{g}{y}{a}{g}{y}' '{n}{t}{a}{n}{t}' '{k}{u}{r}' '{th}{e}{y}' '{p}{e}{th}{e}{r}' + )) + insert '{a}{d}' + ) + + define step2b as ( + [substring] among ( + '{e}{d}{e}{s}' '{e}{d}{oo}{n}' (delete) + ) + [] substring among ( + '{o}{p}' '{y}{p}' '{e}{m}{p}' '{u}{p}' '{g}{i}{p}' '{d}{a}{p}' '{k}{r}{a}{s}{p}' '{m}{y}{l}' (<- '{e}{d}') + ) + ) + + define step2c as ( + [substring] among ( + '{o}{u}{d}{e}{s}' '{o}{u}{d}{oo}{n}' (delete) + ) + [] substring among ( + '{a}{r}{k}' '{k}{a}{l}{y}{a}{k}' '{p}{e}{t}{a}{l}' '{l}{y}{ch}' '{p}{l}{e}{x}' '{s}{k}' '{s}' '{f}{l}' '{f}{r}' '{v}{e}{l}' '{l}{o}{u}{l}' '{ch}{n}' + '{s}{p}' '{t}{r}{a}{g}' '{f}{e}' (<- '{o}{u}{d}') + ) + ) + + define step2d as ( + [substring] among ( + '{e}{oo}{s}' '{e}{oo}{n}' (delete unset test1) + ) + [] substring atlimit among ( + '{th}' '{d}' '{e}{l}' '{g}{a}{l}' '{n}' '{p}' '{y}{d}' '{p}{a}{r}' (<- '{e}') + ) + ) + + define step3 as ( + [substring] among ( + '{y}{a}' '{y}{o}{u}' '{y}{oo}{n}' (delete unset test1) + ) + ([] v <- '{y}') + ) + + define step4 as ( + [substring] among ( + '{y}{k}{a}' '{y}{k}{o}' '{y}{k}{o}{u}' '{y}{k}{oo}{n}' (delete unset test1) + ) + ([] v <- '{y}{k}') or + [] substring atlimit among ( + '{a}{l}' '{a}{d}' '{e}{n}{d}' '{a}{m}{a}{n}' '{a}{m}{m}{o}{ch}{a}{l}' '{i}{th}' '{a}{n}{i}{th}' '{a}{n}{t}{y}{d}' '{f}{u}{s}' '{v}{r}{oo}{m}' '{g}{e}{r}' + '{e}{x}{oo}{d}' '{k}{a}{l}{p}' '{k}{a}{l}{l}{y}{n}' '{k}{a}{t}{a}{d}' '{m}{o}{u}{l}' '{m}{p}{a}{n}' '{m}{p}{a}{g}{y}{a}{t}' '{m}{p}{o}{l}' '{m}{p}{o}{s}' + '{n}{y}{t}' '{x}{y}{k}' '{s}{u}{n}{o}{m}{i}{l}' '{p}{e}{t}{s}' '{p}{y}{t}{s}' '{p}{y}{k}{a}{n}{t}' '{p}{l}{y}{a}{t}{s}' '{p}{o}{s}{t}{e}{l}{n}' '{p}{r}{oo}{t}{o}{d}' + '{s}{e}{r}{t}' '{s}{u}{n}{a}{d}' '{t}{s}{a}{m}' '{u}{p}{o}{d}' '{f}{y}{l}{o}{n}' '{f}{u}{l}{o}{d}' '{ch}{a}{s}' + (<- '{y}{k}') + ) + ) + + define step5a as ( + do ('{a}{g}{a}{m}{e}' atlimit <- '{a}{g}{a}{m}') + do ( + [substring] among ( + '{a}{g}{a}{m}{e}' '{i}{s}{a}{m}{e}' '{o}{u}{s}{a}{m}{e}' '{i}{k}{a}{m}{e}' '{i}{th}{i}{k}{a}{m}{e}' (delete unset test1) + ) + ) + ['{a}{m}{e}'] + delete + unset test1 + [] substring atlimit among ( + '{a}{n}{a}{p}' '{a}{p}{o}{th}' '{a}{p}{o}{k}' '{a}{p}{o}{s}{t}' '{v}{o}{u}{v}' '{x}{e}{th}' '{o}{u}{l}' '{p}{e}{th}' '{p}{y}{k}{r}' '{p}{o}{t}' '{s}{y}{ch}' '{ch}' + (<- '{a}{m}') + ) + ) + + define step5b as ( + do ( + [substring] among ( + '{a}{g}{a}{n}{e}' '{i}{s}{a}{n}{e}' '{o}{u}{s}{a}{n}{e}' '{y}{o}{n}{t}{a}{n}{e}' '{y}{o}{t}{a}{n}{e}' '{y}{o}{u}{n}{t}{a}{n}{e}' '{o}{n}{t}{a}{n}{e}' '{o}{t}{a}{n}{e}' + '{o}{u}{n}{t}{a}{n}{e}' '{i}{k}{a}{n}{e}' '{i}{th}{i}{k}{a}{n}{e}' ( + delete + unset test1 + [] substring atlimit among ( + '{t}{r}' '{t}{s}' (<- '{a}{g}{a}{n}') + ) + ) + ) + ) + ['{a}{n}{e}'] + delete + unset test1 + ([] v2 <- '{a}{n}') or + [] substring atlimit among ( + '{v}{e}{t}{e}{r}' '{v}{o}{u}{l}{k}' '{v}{r}{a}{ch}{m}' '{g}' '{d}{r}{a}{d}{o}{u}{m}' + '{th}' '{k}{a}{l}{p}{o}{u}{z}' '{k}{a}{s}{t}{e}{l}' '{k}{o}{r}{m}{o}{r}' '{l}{a}{o}{p}{l}' '{m}{oo}{a}{m}{e}{th}' + '{m}' '{m}{o}{u}{s}{o}{u}{l}{m}' '{n}' '{o}{u}{l}' '{p}' '{p}{e}{l}{e}{k}' '{p}{l}' '{p}{o}{l}{y}{s}' + '{p}{o}{r}{t}{o}{l}' '{s}{a}{r}{a}{k}{a}{t}{s}' '{s}{o}{u}{l}{t}' '{t}{s}{a}{r}{l}{a}{t}' '{o}{r}{f}' + '{t}{s}{y}{g}{g}' '{t}{s}{o}{p}' '{f}{oo}{t}{o}{s}{t}{e}{f}' '{ch}' '{ps}{u}{ch}{o}{p}{l}' '{a}{g}' + '{g}{a}{l}' '{g}{e}{r}' '{d}{e}{k}' '{d}{y}{p}{l}' '{a}{m}{e}{r}{y}{k}{a}{n}' '{o}{u}{r}' '{p}{y}{th}' + '{p}{o}{u}{r}{y}{t}' '{s}' '{z}{oo}{n}{t}' '{y}{k}' '{k}{a}{s}{t}' '{k}{o}{p}' '{l}{y}{ch}' + '{l}{o}{u}{th}{i}{r}' '{m}{a}{y}{n}{t}' '{m}{e}{l}' '{s}{y}{g}' '{s}{p}' '{s}{t}{e}{g}' '{t}{r}{a}{g}' + '{t}{s}{a}{g}' '{f}' '{e}{r}' '{a}{d}{a}{p}' '{a}{th}{y}{g}{g}' '{a}{m}{i}{ch}' '{a}{n}{y}{k}' + '{a}{n}{o}{r}{g}' '{a}{p}{i}{g}' '{a}{p}{y}{th}' '{a}{t}{s}{y}{g}{g}' '{v}{a}{s}' '{v}{a}{s}{k}' + '{v}{a}{th}{u}{g}{a}{l}' '{v}{y}{o}{m}{i}{ch}' '{v}{r}{a}{ch}{u}{k}' '{d}{y}{a}{t}' '{d}{y}{a}{f}' '{e}{n}{o}{r}{g}' + '{th}{u}{s}' '{k}{a}{p}{n}{o}{v}{y}{o}{m}{i}{ch}' '{k}{a}{t}{a}{g}{a}{l}' '{k}{l}{y}{v}' '{k}{o}{y}{l}{a}{r}{f}' + '{l}{y}{v}' '{m}{e}{g}{l}{o}{v}{y}{o}{m}{i}{ch}' '{m}{y}{k}{r}{o}{v}{y}{o}{m}{i}{ch}' '{n}{t}{a}{v}' + '{x}{i}{r}{o}{k}{l}{y}{v}' '{o}{l}{y}{g}{o}{d}{a}{m}' '{o}{l}{o}{g}{a}{l}' '{p}{e}{n}{t}{a}{r}{f}' '{p}{e}{r}{i}{f}' + '{p}{e}{r}{y}{t}{r}' '{p}{l}{a}{t}' '{p}{o}{l}{u}{d}{a}{p}' '{p}{o}{l}{u}{m}{i}{ch}' '{s}{t}{e}{f}' '{t}{a}{v}' + '{t}{e}{t}' '{u}{p}{e}{r}{i}{f}' '{u}{p}{o}{k}{o}{p}' '{ch}{a}{m}{i}{l}{o}{d}{a}{p}' '{ps}{i}{l}{o}{t}{a}{v}' + (<- '{a}{n}') + ) + ) + + define step5c as ( + do ( + [substring] among ( + '{i}{s}{e}{t}{e}' (delete unset test1) + ) + ) + ['{e}{t}{e}'] + delete + unset test1 + ([] v2 <- '{e}{t}') or + ([] substring among ( + '{o}{d}' '{a}{y}{r}' '{f}{o}{r}' '{t}{a}{th}' '{d}{y}{a}{th}' '{s}{ch}' '{e}{n}{d}' '{e}{u}{r}' '{t}{y}{th}' '{u}{p}{e}{r}{th}' + '{r}{a}{th}' '{e}{n}{th}' '{r}{o}{th}' '{s}{th}' '{p}{u}{r}' '{a}{y}{n}' '{s}{u}{n}{d}' '{s}{u}{n}' '{s}{u}{n}{th}' '{ch}{oo}{r}' + '{p}{o}{n}' '{v}{r}' '{k}{a}{th}' '{e}{u}{th}' '{e}{k}{th}' '{n}{e}{t}' '{r}{o}{n}' '{a}{r}{k}' '{v}{a}{r}' '{v}{o}{l}' '{oo}{f}{e}{l}' + (<- '{e}{t}') + )) or + [] substring atlimit among ( + '{a}{v}{a}{r}' '{v}{e}{n}' '{e}{n}{a}{r}' '{a}{v}{r}' '{a}{d}' '{a}{th}' '{a}{n}' '{a}{p}{l}' '{v}{a}{r}{o}{n}' '{n}{t}{r}' '{s}{k}' '{k}{o}{p}' + '{m}{p}{o}{r}' '{n}{y}{f}' '{p}{a}{g}' '{p}{a}{r}{a}{k}{a}{l}' '{s}{e}{r}{p}' '{s}{k}{e}{l}' '{s}{u}{r}{f}' '{t}{o}{k}' '{u}' '{d}' '{e}{m}' + '{th}{a}{r}{r}' '{th}' + (<- '{e}{t}') + ) + ) + + define step5d as ( + [substring] among ( + '{o}{n}{t}{a}{s}' '{oo}{n}{t}{a}{s}' ( + delete + unset test1 + ([] '{a}{r}{ch}' atlimit <- '{o}{n}{t}') or + ([] '{k}{r}{e}' <- '{oo}{n}{t}') + ) + ) + ) + + define step5e as ( + [substring] among ( + '{o}{m}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{e}' ( + delete + unset test1 + ([] '{o}{n}' atlimit <- '{o}{m}{a}{s}{t}') + ) + ) + ) + + define step5f as ( + do ( + ['{y}{e}{s}{t}{e}'] + delete + unset test1 + [] substring atlimit among ( + '{p}' '{a}{p}' '{s}{u}{m}{p}' '{a}{s}{u}{m}{p}' '{a}{k}{a}{t}{a}{p}' '{a}{m}{e}{t}{a}{m}{f}' (<- '{y}{e}{s}{t}') + ) + ) + ['{e}{s}{t}{e}'] + delete + unset test1 + [] substring atlimit among ( + '{a}{l}' '{a}{r}' '{e}{k}{t}{e}{l}' '{z}' '{m}' '{x}' '{p}{a}{r}{a}{k}{a}{l}' '{p}{r}{o}' '{n}{y}{s}' + (<- '{y}{e}{s}{t}') + ) + ) + + define step5g as ( + do ( + [substring] among ( + '{i}{th}{i}{k}{a}' '{i}{th}{i}{k}{e}{s}' '{i}{th}{i}{k}{e}' (delete unset test1) + ) + ) + [substring] among ( + '{i}{k}{a}' '{i}{k}{e}{s}' '{i}{k}{e}' ( + delete + unset test1 + ([] substring among ( + '{s}{k}{oo}{l}' '{s}{k}{o}{u}{l}' '{n}{a}{r}{th}' '{s}{f}' '{o}{th}' '{p}{y}{th}' (<- '{i}{k}') + )) or + ([] substring atlimit among ( + '{d}{y}{a}{th}' '{th}' '{p}{a}{r}{a}{k}{a}{t}{a}{th}' '{p}{r}{o}{s}{th}' '{s}{u}{n}{th}' (<- '{i}{k}') + )) + ) + ) + ) + + define step5h as ( + [substring] among ( + '{o}{u}{s}{a}' '{o}{u}{s}{e}{s}' '{o}{u}{s}{e}' ( + delete + unset test1 + ([] substring among ( + '{p}{o}{d}{a}{r}' '{v}{l}{e}{p}' '{p}{a}{n}{t}{a}{ch}' '{f}{r}{u}{d}' '{m}{a}{n}{t}{y}{l}' '{m}{a}{l}{l}' '{k}{u}{m}{a}{t}' '{l}{a}{ch}' '{l}{i}{g}' + '{f}{a}{g}' '{o}{m}' '{p}{r}{oo}{t}' (<- '{o}{u}{s}') + + )) or + ([] substring atlimit among ( + '{f}{a}{r}{m}{a}{k}' '{ch}{a}{d}' '{a}{g}{k}' '{a}{n}{a}{r}{r}' '{v}{r}{o}{m}' '{e}{k}{l}{y}{p}' '{l}{a}{m}{p}{y}{d}' '{l}{e}{ch}' '{m}' '{p}{a}{t}' + '{r}' '{l}' '{m}{e}{d}' '{m}{e}{s}{a}{z}' '{u}{p}{o}{t}{e}{y}{n}' '{a}{m}' '{a}{y}{th}' '{a}{n}{i}{k}' '{d}{e}{s}{p}{o}{z}' + '{e}{n}{d}{y}{a}{f}{e}{r}' '{d}{e}' '{d}{e}{u}{t}{e}{r}{e}{u}' '{k}{a}{th}{a}{r}{e}{u}' '{p}{l}{e}' '{t}{s}{a}' + (<- '{o}{u}{s}') + )) + ) + ) + ) + + define step5i as ( + [substring] among ( + '{a}{g}{a}' '{a}{g}{e}{s}' '{a}{g}{e}' ( + delete + unset test1 + ([] '{k}{o}{l}{l}' <- '{a}{g}') or ( + not ([substring] among ('{ps}{o}{f}' '{n}{a}{u}{l}{o}{ch}')) + ([] substring among ( + '{o}{f}' '{p}{e}{l}' '{ch}{o}{r}{t}' '{l}{l}' '{s}{f}' '{r}{p}' '{f}{r}' '{p}{r}' '{l}{o}{ch}' '{s}{m}{i}{n}' + (<- '{a}{g}') + )) or + ([] substring atlimit among ( + '{a}{v}{a}{s}{t}' '{p}{o}{l}{u}{f}' '{a}{d}{i}{f}' '{p}{a}{m}{f}' '{r}' '{a}{s}{p}' '{a}{f}' '{a}{m}{a}{l}' '{a}{m}{a}{l}{l}{y}' + '{a}{n}{u}{s}{t}' '{a}{p}{e}{r}' '{a}{s}{p}{a}{r}' '{a}{ch}{a}{r}' '{d}{e}{r}{v}{e}{n}' '{d}{r}{o}{s}{o}{p}' '{x}{e}{f}' '{n}{e}{o}{p}' + '{n}{o}{m}{o}{t}' '{o}{l}{o}{p}' '{o}{m}{o}{t}' '{p}{r}{o}{s}{t}' '{p}{r}{o}{s}{oo}{p}{o}{p}' '{s}{u}{m}{p}' '{s}{u}{n}{t}' '{t}' '{u}{p}{o}{t}' + '{ch}{a}{r}' '{a}{e}{y}{p}' '{a}{y}{m}{o}{s}{t}' '{a}{n}{u}{p}' '{a}{p}{o}{t}' '{a}{r}{t}{y}{p}' '{d}{y}{a}{t}' '{e}{n}' '{e}{p}{y}{t}' + '{k}{r}{o}{k}{a}{l}{o}{p}' '{s}{y}{d}{i}{r}{o}{p}' '{l}' '{n}{a}{u}' '{o}{u}{l}{a}{m}' '{o}{u}{r}' '{p}' '{t}{r}' '{m}' + (<- '{a}{g}') + )) + ) + ) + ) + ) + + define step5j as ( + [substring] among ( + '{i}{s}{e}' '{i}{s}{o}{u}' '{i}{s}{a}' (delete unset test1) + ) + [] substring atlimit among ( + '{n}' '{ch}{e}{r}{s}{o}{n}' '{d}{oo}{d}{e}{k}{a}{n}' '{e}{r}{i}{m}{o}{n}' '{m}{e}{g}{a}{l}{o}{n}' '{e}{p}{t}{a}{n}' (<- '{i}{s}') + ) + ) + + define step5k as ( + [substring] among ( + '{i}{s}{t}{e}' (delete unset test1) + ) + [] substring atlimit among ( + '{a}{s}{v}' '{s}{v}' '{a}{ch}{r}' '{ch}{r}' '{a}{p}{l}' '{a}{e}{y}{m}{n}' '{d}{u}{s}{ch}{r}' '{e}{u}{ch}{r}' '{k}{o}{y}{n}{o}{ch}{r}' '{p}{a}{l}{y}{m}{ps}' + (<- '{i}{s}{t}') + ) + ) + + define step5l as ( + [substring] among ( + '{o}{u}{n}{e}' '{i}{s}{o}{u}{n}{e}' '{i}{th}{o}{u}{n}{e}' (delete unset test1) + ) + [] substring atlimit among ( + '{n}' '{r}' '{s}{p}{y}' '{s}{t}{r}{a}{v}{o}{m}{o}{u}{t}{s}' '{k}{a}{k}{o}{m}{o}{u}{t}{s}' '{e}{x}{oo}{n}' (<- '{o}{u}{n}') + ) + ) + + define step5m as ( + [substring] among ( + '{o}{u}{m}{e}' '{i}{s}{o}{u}{m}{e}' '{i}{th}{o}{u}{m}{e}' (delete unset test1) + ) + [] substring atlimit among ( + '{p}{a}{r}{a}{s}{o}{u}{s}' '{f}' '{ch}' '{oo}{r}{y}{o}{p}{l}' '{a}{z}' '{a}{l}{l}{o}{s}{o}{u}{s}' '{a}{s}{o}{u}{s}' + (<- '{o}{u}{m}') + ) + ) + + define step6 as ( + do ( + [substring] among ( + '{m}{a}{t}{a}' '{m}{a}{t}{oo}{n}' '{m}{a}{t}{o}{s}' (<- '{m}{a}') + ) + ) + test1 + [substring] among ( + '{a}' '{a}{g}{a}{t}{e}' '{a}{g}{a}{n}' '{a}{e}{y}' '{a}{m}{a}{y}' '{a}{n}' '{a}{s}' '{a}{s}{a}{y}' '{a}{t}{a}{y}' '{a}{oo}' '{e}' '{e}{y}' + '{e}{y}{s}' '{e}{y}{t}{e}' '{e}{s}{a}{y}' '{e}{s}' '{e}{t}{a}{y}' '{y}' '{y}{e}{m}{a}{y}' '{y}{e}{m}{a}{s}{t}{e}' '{y}{e}{t}{a}{y}' '{y}{e}{s}{a}{y}' + '{y}{e}{s}{a}{s}{t}{e}' '{y}{o}{m}{a}{s}{t}{a}{n}' '{y}{o}{m}{o}{u}{n}' '{y}{o}{m}{o}{u}{n}{a}' '{y}{o}{n}{t}{a}{n}' '{y}{o}{n}{t}{o}{u}{s}{a}{n}' '{y}{o}{s}{a}{s}{t}{a}{n}' + '{y}{o}{s}{a}{s}{t}{e}' '{y}{o}{s}{o}{u}{n}' '{y}{o}{s}{o}{u}{n}{a}' '{y}{o}{t}{a}{n}' '{y}{o}{u}{m}{a}' '{y}{o}{u}{m}{a}{s}{t}{e}' '{y}{o}{u}{n}{t}{a}{y}' + '{y}{o}{u}{n}{t}{a}{n}' '{i}' '{i}{d}{e}{s}' '{i}{d}{oo}{n}' '{i}{th}{e}{y}' '{i}{th}{e}{y}{s}' '{i}{th}{e}{y}{t}{e}' '{i}{th}{i}{k}{a}{t}{e}' '{i}{th}{i}{k}{a}{n}' + '{i}{th}{o}{u}{n}' '{i}{th}{oo}' '{i}{k}{a}{t}{e}' '{i}{k}{a}{n}' '{i}{s}' '{i}{s}{a}{n}' '{i}{s}{a}{t}{e}' '{i}{s}{e}{y}' '{i}{s}{e}{s}' '{i}{s}{o}{u}{n}' + '{i}{s}{oo}' '{o}' '{o}{y}' '{o}{m}{a}{y}' '{o}{m}{a}{s}{t}{a}{n}' '{o}{m}{o}{u}{n}' '{o}{m}{o}{u}{n}{a}' '{o}{n}{t}{a}{y}' '{o}{n}{t}{a}{n}' + '{o}{n}{t}{o}{u}{s}{a}{n}' '{o}{s}' '{o}{s}{a}{s}{t}{a}{n}' '{o}{s}{a}{s}{t}{e}' '{o}{s}{o}{u}{n}' '{o}{s}{o}{u}{n}{a}' '{o}{t}{a}{n}' '{o}{u}' '{o}{u}{m}{a}{y}' + '{o}{u}{m}{a}{s}{t}{e}' '{o}{u}{n}' '{o}{u}{n}{t}{a}{y}' '{o}{u}{n}{t}{a}{n}' '{o}{u}{s}' '{o}{u}{s}{a}{n}' '{o}{u}{s}{a}{t}{e}' '{u}' '{u}{s}' '{oo}' + '{oo}{n}' (delete) + ) + ) + + define step7 as ( + [substring] among ( + '{e}{s}{t}{e}{r}' '{e}{s}{t}{a}{t}' '{o}{t}{e}{r}' '{o}{t}{a}{t}' '{u}{t}{e}{r}' '{u}{t}{a}{t}' '{oo}{t}{e}{r}' '{oo}{t}{a}{t}' (delete) + ) + ) +) + +define stem as ( + backwards ( + do tolower + has_min_length + set test1 + do step1 + do steps1 + do steps2 + do steps3 + do steps4 + do steps5 + do steps6 + do steps7 + do steps8 + do steps9 + do steps10 + do step2a + do step2b + do step2c + do step2d + do step3 + do step4 + do step5a + do step5b + do step5c + do step5d + do step5e + do step5f + do step5g + do step5h + do step5j + do step5i + do step5k + do step5l + do step5m + do step6 + do step7 + ) +) diff --git a/contrib/snowball/algorithms/hindi.sbl b/contrib/snowball/algorithms/hindi.sbl new file mode 100644 index 0000000..bfdfac0 --- /dev/null +++ b/contrib/snowball/algorithms/hindi.sbl @@ -0,0 +1,323 @@ +// An implementation of "A Lightweight Stemmer for Hindi": +// http://www.kbcs.in/downloads/papers/StmmerHindi.pdf + +externals ( stem ) + +stringescapes {} + +// The transliteration scheme used for our stringdefs matches that used in the +// paper, as documented in the appendix. It appears to match the WX notation +// (https://en.wikipedia.org/wiki/WX_notation) except that WX apparently +// uses 'z' for Anunasika whereas the paper uses Mh. +// +// We discriminate dependent vowels by adding a leading "_" to their stringdef +// names (mnemonic: the _ signifies removing the implicit a from the preceding +// character). + +// Vowels and sonorants: +stringdef a '{U+0905}' +stringdef A '{U+0906}' +stringdef i '{U+0907}' +stringdef I '{U+0908}' +stringdef u '{U+0909}' +stringdef U '{U+090A}' +stringdef q '{U+090B}' +stringdef e '{U+090F}' +stringdef E '{U+0910}' +stringdef o '{U+0913}' +stringdef O '{U+0914}' + +// Vowel signs: +stringdef _A '{U+093E}' +stringdef _i '{U+093F}' +stringdef _I '{U+0940}' +stringdef _u '{U+0941}' +stringdef _U '{U+0942}' +stringdef _q '{U+0943}' +stringdef _e '{U+0947}' +stringdef _E '{U+0948}' +stringdef _o '{U+094B}' +stringdef _O '{U+094C}' + +// Diacritics: +stringdef M '{U+0902}' +stringdef H '{U+0903}' +stringdef Mh '{U+0901}' +stringdef Z '{U+093C}' // Nukta +stringdef virama '{U+094D}' + +// Velar consonants: +stringdef k '{U+0915}' +stringdef K '{U+0916}' +stringdef g '{U+0917}' +stringdef G '{U+0918}' +stringdef f '{U+0919}' + +// Palatal consonants: +stringdef c '{U+091A}' +stringdef C '{U+091B}' +stringdef j '{U+091C}' +stringdef J '{U+091D}' +stringdef F '{U+091E}' + +// Retroflex consonants: +stringdef t '{U+091F}' +stringdef T '{U+0920}' +stringdef d '{U+0921}' +stringdef D '{U+0922}' +stringdef N '{U+0923}' + +// Dental consonants: +stringdef w '{U+0924}' +stringdef W '{U+0925}' +stringdef x '{U+0926}' +stringdef X '{U+0927}' +stringdef n '{U+0928}' + +// Labial consonants: +stringdef p '{U+092A}' +stringdef P '{U+092B}' +stringdef b '{U+092C}' +stringdef B '{U+092D}' +stringdef m '{U+092E}' + +// Semi-vowels: +stringdef y '{U+092F}' +stringdef r '{U+0930}' +stringdef l '{U+0932}' +stringdef v '{U+0935}' + +// Fricatives: +stringdef S '{U+0936}' +stringdef R '{U+0937}' +stringdef s '{U+0938}' +stringdef h '{U+0939}' + +stringdef lY '{U+0933}' + +// Precomposed characters - letters + nukta: +stringdef nZ '{U+0929}' // ≡ {n}{Z} +stringdef rZ '{U+0931}' // ≡ {r}{Z} +stringdef lYZ '{U+0934}' // ≡ {lY}{Z} +stringdef kZ '{U+0958}' // ≡ {k}{Z} +stringdef KZ '{U+0959}' // ≡ {K}{Z} +stringdef gZ '{U+095A}' // ≡ {g}{Z} +stringdef jZ '{U+095B}' // ≡ {j}{Z} +stringdef dZ '{U+095C}' // ≡ {d}{Z} +stringdef DZ '{U+095D}' // ≡ {D}{Z} +stringdef PZ '{U+095E}' // ≡ {P}{Z} +stringdef yZ '{U+095F}' // ≡ {y}{Z} + +integers ( p ) + +groupings ( consonant ) + +routines ( CONSONANT ) + +define consonant '{k}{K}{g}{G}{f}' + + '{c}{C}{j}{J}{F}' + + '{t}{T}{d}{D}{N}' + + '{w}{W}{x}{X}{n}' + + '{p}{P}{b}{B}{m}' + + '{y}{r}{l}{v}' + + '{S}{R}{s}{h}' + + '{lY}' + + '{Z}' + // Nukta + // Precomposed characters - letter and nukta: + '{nZ}{rZ}{lYZ}{kZ}{KZ}{gZ}{jZ}{dZ}{DZ}{PZ}{yZ}' + +backwardmode ( define CONSONANT as ( consonant ) ) + +define stem as ( + test ( next setmark p ) + backwards ( + // We assume in this implementation that the whole word doesn't count + // as a valid suffix to remove, so we remove the longest suffix from + // the list which leaves at least one character. This change affects + // 47 words out of the 65,140 in the sample vocabulary from Hindi + // wikipedia. + setlimit tomark p for ([substring]) + among ( + // The list below is derived from figure 3 in the paper. + // + // We perform the stemming on the Devanagari characters rather than + // transliterating to Latin, so we have adapted the list below to + // reflect this by converting suffixes back to Devanagari as + // follows: + // + // * within the suffixes, "a" after a consonant is dropped since + // consonants have an implicit "a". + // + // * within the suffixes, a vowel other than "a" after a consonant + // is a dependent vowel (vowel sign); a vowel (including "a") + // after a non-consonant is an independent vowel. + // + // * to allow the vowel at the start of each suffix being dependent + // or independent, we include each suffix twice. For the + // dependent version, a leading "a" is dropped and we check that + // the suffix is preceded by a consonant (which will have an + // implicit "a"). + // + // * we add '{a}', which is needed for the example given right at + // the end of section 5 to work (conflating BarawIya and + // BarawIyawA), and which 3.1 a.v strongly suggests should be in + // the list: + // + // Thus, the following suffix deletions (longest possible + // match) are required to reduce inflected forms of masculine + // nouns to a common stem: + // a A i [...] + // + // Adding '{a}' only affect 2 words out of the 65,140 in the + // sample vocabulary. + // + // * The transliterations of our stems would end with "a" when our + // stems end in a consonant, so we also include {virama} in the + // list of suffixes to remove (this affects 222 words from the + // sample vocabulary). + // + // We've also assumed that Mh in the suffix list always means {Mh} + // and never {M}{h}{virama}. Only one of the 65,140 words in the + // sample vocabulary stems differently due to this (and that word + // seems to be a typo). + + '{virama}' + + '{a}' + '{A}' + '{i}' + '{I}' + '{u}' + '{U}' + '{e}' + '{o}' + '{e}{M}' + '{o}{M}' + '{A}{M}' + '{u}{A}{M}' + '{u}{e}{M}' + '{u}{o}{M}' + '{A}{e}{M}' + '{A}{o}{M}' + '{i}{y}{_A}{M}' + '{i}{y}{_o}{M}' + '{A}{i}{y}{_A}{M}' + '{A}{i}{y}{_o}{M}' + '{A}{Mh}' + '{i}{y}{_A}{Mh}' + '{A}{i}{y}{_A}{Mh}' + '{a}{w}{_A}{e}{M}' + '{a}{w}{_A}{o}{M}' + '{a}{n}{_A}{e}{M}' + '{a}{n}{_A}{o}{M}' + '{a}{w}{_A}' + '{a}{w}{_I}' + '{I}{M}' + '{a}{w}{_I}{M}' + '{a}{w}{_e}' + '{A}{w}{_A}' + '{A}{w}{_I}' + '{A}{w}{_I}{M}' + '{A}{w}{_e}' + '{a}{n}{_A}' + '{a}{n}{_I}' + '{a}{n}{_e}' + '{A}{n}{_A}' + '{A}{n}{_e}' + '{U}{M}{g}{_A}' + '{U}{M}{g}{_I}' + '{A}{U}{M}{g}{_A}' + '{A}{U}{M}{g}{_I}' + '{e}{M}{g}{_e}' + '{e}{M}{g}{_I}' + '{A}{e}{M}{g}{_e}' + '{A}{e}{M}{g}{_I}' + '{o}{g}{_e}' + '{o}{g}{_I}' + '{A}{o}{g}{_e}' + '{A}{o}{g}{_I}' + '{e}{g}{_A}' + '{e}{g}{_I}' + '{A}{e}{g}{_A}' + '{A}{e}{g}{_I}' + '{A}{y}{_A}' + '{A}{e}' + '{A}{I}' + '{A}{I}{M}' + '{i}{e}' + '{A}{o}' + '{A}{i}{e}' + '{a}{k}{r}' + '{A}{k}{r}' + + '{_A}' + '{_i}' + '{_I}' + '{_u}' + '{_U}' + '{_e}' + '{_o}' + '{_e}{M}' + '{_o}{M}' + '{_A}{M}' + '{_u}{A}{M}' + '{_u}{e}{M}' + '{_u}{o}{M}' + '{_A}{e}{M}' + '{_A}{o}{M}' + '{_i}{y}{_A}{M}' + '{_i}{y}{_o}{M}' + '{_A}{i}{y}{_A}{M}' + '{_A}{i}{y}{_o}{M}' + '{_A}{Mh}' + '{_i}{y}{_A}{Mh}' + '{_A}{i}{y}{_A}{Mh}' + '{_I}{M}' + '{_A}{w}{_A}' + '{_A}{w}{_I}' + '{_A}{w}{_I}{M}' + '{_A}{w}{_e}' + '{_A}{n}{_A}' + '{_A}{n}{_e}' + '{_U}{M}{g}{_A}' + '{_U}{M}{g}{_I}' + '{_A}{U}{M}{g}{_A}' + '{_A}{U}{M}{g}{_I}' + '{_e}{M}{g}{_e}' + '{_e}{M}{g}{_I}' + '{_A}{e}{M}{g}{_e}' + '{_A}{e}{M}{g}{_I}' + '{_o}{g}{_e}' + '{_o}{g}{_I}' + '{_A}{o}{g}{_e}' + '{_A}{o}{g}{_I}' + '{_e}{g}{_A}' + '{_e}{g}{_I}' + '{_A}{e}{g}{_A}' + '{_A}{e}{g}{_I}' + '{_A}{y}{_A}' + '{_A}{e}' + '{_A}{I}' + '{_A}{I}{M}' + '{_i}{e}' + '{_A}{o}' + '{_A}{i}{e}' + '{_A}{k}{r}' + + /* Suffixes with a leading implicit a: */ + '{w}{_A}{e}{M}' CONSONANT + '{w}{_A}{o}{M}' CONSONANT + '{n}{_A}{e}{M}' CONSONANT + '{n}{_A}{o}{M}' CONSONANT + '{w}{_A}' CONSONANT + '{w}{_I}' CONSONANT + '{w}{_I}{M}' CONSONANT + '{w}{_e}' CONSONANT + '{n}{_A}' CONSONANT + '{n}{_I}' CONSONANT + '{n}{_e}' CONSONANT + '{k}{r}' CONSONANT + ) + delete + ) +) diff --git a/contrib/snowball/algorithms/hungarian.sbl b/contrib/snowball/algorithms/hungarian.sbl new file mode 100644 index 0000000..2d7885c --- /dev/null +++ b/contrib/snowball/algorithms/hungarian.sbl @@ -0,0 +1,241 @@ +/* +Hungarian Stemmer +Removes noun inflections +*/ + +routines ( + mark_regions + R1 + v_ending + case + case_special + case_other + plural + owned + sing_owner + plur_owner + instrum + factive + undouble + double +) + +externals ( stem ) + +integers ( p1 ) +groupings ( v ) + +stringescapes {} + +/* special characters */ + +stringdef a' '{U+00E1}' //a-acute +stringdef e' '{U+00E9}' //e-acute +stringdef i' '{U+00ED}' //i-acute +stringdef o' '{U+00F3}' //o-acute +stringdef o" '{U+00F6}' //o-umlaut +stringdef oq '{U+0151}' //o-double acute +stringdef u' '{U+00FA}' //u-acute +stringdef u" '{U+00FC}' //u-umlaut +stringdef uq '{U+0171}' //u-double acute + +define v 'aeiou{a'}{e'}{i'}{o'}{o"}{oq}{u'}{u"}{uq}' + +define mark_regions as ( + + $p1 = limit + + (v goto non-v + among('cs' 'gy' 'ly' 'ny' 'sz' 'ty' 'zs' 'dzs') or next + setmark p1) + or + + (non-v gopast v setmark p1) +) + +backwardmode ( + + define R1 as $p1 <= cursor + + define v_ending as ( + [substring] R1 among( + '{a'}' (<- 'a') + '{e'}' (<- 'e') + ) + ) + + define double as ( + test among('bb' 'cc' 'ccs' 'dd' 'ff' 'gg' 'ggy' 'jj' 'kk' 'll' 'lly' 'mm' + 'nn' 'nny' 'pp' 'rr' 'ss' 'ssz' 'tt' 'tty' 'vv' 'zz' 'zzs') + ) + + define undouble as ( + next [hop 1] delete + ) + + define instrum as( + [substring] R1 among( + 'al' (double) + 'el' (double) + ) + delete + undouble + ) + + + define case as ( + [substring] R1 among( + 'ban' 'ben' + 'ba' 'be' + 'ra' 're' + 'nak' 'nek' + 'val' 'vel' + 't{o'}l' 't{oq}l' + 'r{o'}l' 'r{oq}l' + 'b{o'}l' 'b{oq}l' + 'hoz' 'hez' 'h{o"}z' + 'n{a'}l' 'n{e'}l' + 'ig' + 'at' 'et' 'ot' '{o"}t' + '{e'}rt' + 'k{e'}pp' 'k{e'}ppen' + 'kor' + 'ul' '{u"}l' + 'v{a'}' 'v{e'}' + 'onk{e'}nt' 'enk{e'}nt' 'ank{e'}nt' + 'k{e'}nt' + 'en' 'on' 'an' '{o"}n' + 'n' + 't' + ) + delete + v_ending + ) + + define case_special as( + [substring] R1 among( + '{e'}n' (<- 'e') + '{a'}n' (<- 'a') + '{a'}nk{e'}nt' (<- 'a') + ) + ) + + define case_other as( + [substring] R1 among( + 'astul' 'est{u"}l' (delete) + 'stul' 'st{u"}l' (delete) + '{a'}stul' (<- 'a') + '{e'}st{u"}l' (<- 'e') + ) + ) + + define factive as( + [substring] R1 among( + '{a'}' (double) + '{e'}' (double) + ) + delete + undouble + ) + + define plural as ( + [substring] R1 among( + '{a'}k' (<- 'a') + '{e'}k' (<- 'e') + '{o"}k' (delete) + 'ak' (delete) + 'ok' (delete) + 'ek' (delete) + 'k' (delete) + ) + ) + + define owned as ( + [substring] R1 among ( + 'ok{e'}' '{o"}k{e'}' 'ak{e'}' 'ek{e'}' (delete) + '{e'}k{e'}' (<- 'e') + '{a'}k{e'}' (<- 'a') + 'k{e'}' (delete) + '{e'}{e'}i' (<- 'e') + '{a'}{e'}i' (<- 'a') + '{e'}i' (delete) + '{e'}{e'}' (<- 'e') + '{e'}' (delete) + ) + ) + + define sing_owner as ( + [substring] R1 among( + '{u"}nk' 'unk' (delete) + '{a'}nk' (<- 'a') + '{e'}nk' (<- 'e') + 'nk' (delete) + '{a'}juk' (<- 'a') + '{e'}j{u"}k' (<- 'e') + 'juk' 'j{u"}k' (delete) + 'uk' '{u"}k' (delete) + 'em' 'om' 'am' (delete) + '{a'}m' (<- 'a') + '{e'}m' (<- 'e') + 'm' (delete) + 'od' 'ed' 'ad' '{o"}d' (delete) + '{a'}d' (<- 'a') + '{e'}d' (<- 'e') + 'd' (delete) + 'ja' 'je' (delete) + 'a' 'e' 'o' (delete) + '{a'}' (<- 'a') + '{e'}' (<- 'e') + ) + ) + + define plur_owner as ( + [substring] R1 among( + 'jaim' 'jeim' (delete) + '{a'}im' (<- 'a') + '{e'}im' (<- 'e') + 'aim' 'eim' (delete) + 'im' (delete) + 'jaid' 'jeid' (delete) + '{a'}id' (<- 'a') + '{e'}id' (<- 'e') + 'aid' 'eid' (delete) + 'id' (delete) + 'jai' 'jei' (delete) + '{a'}i' (<- 'a') + '{e'}i' (<- 'e') + 'ai' 'ei' (delete) + 'i' (delete) + 'jaink' 'jeink' (delete) + 'eink' 'aink' (delete) + '{a'}ink' (<- 'a') + '{e'}ink' (<- 'e') + 'ink' + 'jaitok' 'jeitek' (delete) + 'aitok' 'eitek' (delete) + '{a'}itok' (<- 'a') + '{e'}itek' (<- 'e') + 'itek' (delete) + 'jeik' 'jaik' (delete) + 'aik' 'eik' (delete) + '{a'}ik' (<- 'a') + '{e'}ik' (<- 'e') + 'ik' (delete) + ) + ) +) + +define stem as ( + do mark_regions + backwards ( + do instrum + do case + do case_special + do case_other + do factive + do owned + do sing_owner + do plur_owner + do plural + ) +) diff --git a/contrib/snowball/algorithms/indonesian.sbl b/contrib/snowball/algorithms/indonesian.sbl new file mode 100644 index 0000000..ac0ee36 --- /dev/null +++ b/contrib/snowball/algorithms/indonesian.sbl @@ -0,0 +1,192 @@ +// An implementation of the "Porter Stemmer for Bahasa Indonesia" from: +// http://www.illc.uva.nl/Research/Publications/Reports/MoL-2003-02.text.pdf + +integers ( + // The paper defines measure as the number of vowels in the word. We + // count this initially, then adjust the count each time we remove a + // prefix or suffix. + measure + + // Numeric code for the type of prefix removed: + // + // 0 other/none + // 1 'di' or 'meng' or 'ter' + // 2 'per' + // 3 'ke' or 'peng' + // 4 'ber' + // + // Some of these have variant forms, so e.g. "meng" includes "men", "me", + // "meny", "mem". + // + // Note that the value of prefix is only used in remove_suffix (and + // routines it calls) so we don't need to worry about + // remove_second_order_prefix overwriting a value of prefix set by + // remove_first_order_prefix since remove_suffix gets called between + // the two. + prefix +) + +groupings ( vowel ) + +routines ( + remove_particle + remove_possessive_pronoun + remove_first_order_prefix + remove_second_order_prefix + remove_suffix + KER + SUFFIX_KAN_OK + SUFFIX_AN_OK + SUFFIX_I_OK + VOWEL +) + +externals ( stem ) + +stringescapes {} + +backwardmode ( + + define remove_particle as ( + [substring] among ( + 'kah' 'lah' 'pun' (delete $measure-=1) + ) + ) + + define remove_possessive_pronoun as ( + [substring] among ( + 'ku' 'mu' 'nya' (delete $measure-=1) + ) + ) + + // prefix not in {ke, peng, per} + define SUFFIX_KAN_OK as ( + // On page 29, the example "kompas Q.31" says "Both Nazief and Porter + // stemmer converted the word peledakan (blast, explotion) to ledak (to + // blast, to explode)". However, the algorithm as described doesn't + // behave in this way - grammatically the prefix pe- occurs as a + // variation of both the first-order derivational prefix peng- and the + // second-order derivational prefix per-, but table 2.5 doesn't include + // "pe", only table 2.6 does, so "peledakan" is handled (incorrectly) + // as having prefix "per" not "peng", and so we remove derivational + // suffix "kan" rather than "an" to give stem leda. (Porter-style + // stemmers remove the longest suffix they can amongst those available, + // which this paper notes in the last paragraph on page 15). + // + // We resolve this by amending the condition on suffix "kan" to + // "prefix ∉ {ke, peng, per}", which seems to make the stemmer's + // behaviour match all the examples in the paper except for one: + // "perbaikan" is shown in table 3.4 as stemming to "bai", but with + // this change it now stems to "baik". The table notes that "baik" is + // the actual root so this deviation is an improvement. In a sample + // vocabulary derived from the most common words in id.wikipedia.org, + // this change only affects 0.12% of words (76 out of 64,587, including + // "peledakan" and "perbaikan"). + $prefix != 3 and $prefix != 2 + ) + + // prefix not in {di, meng, ter} + define SUFFIX_AN_OK as ( $prefix != 1 ) + + define SUFFIX_I_OK as ( + // prefix not in {ke, peng, ber} + $prefix <= 2 + + // The rest of the condition from the paper is: + // V|K...c₁c₁, c₁ ≠ s, c₂ ≠ i + // + // The meaning of this is unclear in several ways, and none of the + // examples given of the stemmer's behaviour in the paper help to + // resolve these issues. + // + // Notice that c₂ isn't actually used - the most obvious explanation + // seems to be that "c₁c₁" should read "c₁c₂", or maybe "c₂c₁". + // + // Elsewhere the paper defines V... as meaning "the stem starts with + // a vowel" and K... as meaning "the stem starts with a consonant". + // + // In other places where it says X|Y... it seems the | binds more + // tightly, so it's (V|K)...cᵢcⱼ not V|(K...cᵢcⱼ). That seems a bit + // odd as the first letter must be either a vowel or a consonant, so + // that really just means "ends cᵢcⱼ". However, nowhere in the paper + // uses or defines a notation such as ...X, which may explain this + // seemingly redundant way of specifying this. + // + // The conditions elsewhere on prefix removal (e.g. V...) are clearly + // on the stem left after the prefix is removed. None of the other + // rules for suffix removal have conditions on the stem, but for + // consistency with the prefix rules we might expect that the cᵢcⱼ + // test is on what's left *after* removing the "i" suffix. + // + // However, studying Indonesian wordlists and discussion with a native + // speaker leads us to conclude that the purpose of this check is to + // protect words of foreign origin (e.g. "televisi", "organisasi", + // "komunikasi") from stemming, and the common feature of these is + // that the word ends "-si", so we conclude that the condition here + // should be read as "word does not end -si", and this is what we + // have implemented. + not 's' + ) + + define remove_suffix as ( + [substring] among ( + 'kan' SUFFIX_KAN_OK 'an' SUFFIX_AN_OK 'i' SUFFIX_I_OK + (delete $measure-=1) + ) + ) +) + +define vowel 'aeiou' + +define VOWEL as ( vowel ) + +define KER as ( non-vowel 'er' ) + +define remove_first_order_prefix as ( + [substring] among ( + 'di' 'meng' 'men' 'me' 'ter' (delete $prefix=1 $measure-=1) + 'ke' 'peng' 'pen' (delete $prefix=3 $measure-=1) + 'meny' VOWEL ($prefix=1 <-'s' $measure-=1) + 'peny' VOWEL ($prefix=3 <-'s' $measure-=1) + 'mem' ($prefix=1 $measure-=1 vowel and <-'p' or delete) + 'pem' ($prefix=3 $measure-=1 vowel and <-'p' or delete) + ) +) + +define remove_second_order_prefix as ( + // The paper has the condition on removal of prefix "bel" and "pel" as + // just "ajar" not "ajar..." but it seems that the latter must be what + // is intended so that e.g. "pelajaran" stems to "ajar" not "lajar". + // This change only affects a very small number of words (11 out of + // 64,587) and only for the better. + [substring] among ( + 'per' 'pe' (delete $prefix=2 $measure-=1) + 'pelajar' (<-'ajar' $measure-=1) + 'ber' (delete $prefix=4 $measure-=1) + 'belajar' (<-'ajar' $prefix=4 $measure-=1) + 'be' KER (delete $prefix=4 $measure-=1) + ) +) + +define stem as ( + $measure = 0 + do ( repeat ( gopast vowel $measure+=1 ) ) + $measure > 2 + $prefix = 0 + backwards ( + do remove_particle + $measure > 2 + do remove_possessive_pronoun + ) + $measure > 2 + test ( + remove_first_order_prefix + do ( + test ($measure > 2 backwards remove_suffix) + $measure > 2 remove_second_order_prefix + ) + ) or ( + do remove_second_order_prefix + do ($measure > 2 backwards remove_suffix) + ) +) diff --git a/contrib/snowball/algorithms/irish.sbl b/contrib/snowball/algorithms/irish.sbl new file mode 100644 index 0000000..0b1288a --- /dev/null +++ b/contrib/snowball/algorithms/irish.sbl @@ -0,0 +1,151 @@ +routines ( + R1 R2 RV + initial_morph + mark_regions + noun_sfx + deriv + verb_sfx +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v ) + +stringescapes {} + +/* Accented characters */ + +stringdef a' '{U+00E1}' // a-acute +stringdef e' '{U+00E9}' // e-acute +stringdef i' '{U+00ED}' // i-acute +stringdef o' '{U+00F3}' // o-acute +stringdef u' '{U+00FA}' // u-acute + +define v 'aeiou{a'}{e'}{i'}{o'}{u'}' + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + gopast v setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define initial_morph as ( + [substring] among ( + 'h-' 'n-' 't-' //nAthair -> n-athair, but alone are problematic + (delete) + + // verbs + 'd{'}' + (delete) + 'd{'}fh' + (<- 'f') + // other contractions + 'm{'}' 'b{'}' + (delete) + + 'sh' + (<- 's') + + 'mb' + (<- 'b') + 'gc' + (<- 'c') + 'nd' + (<- 'd') + 'bhf' + (<- 'f') + 'ng' + (<- 'g') + 'bp' + (<- 'p') + 'ts' + (<- 's') + 'dt' + (<- 't') + + // Lenition + 'bh' + (<- 'b') + 'ch' + (<- 'c') + 'dh' + (<- 'd') + 'fh' + (<- 'f') + 'gh' + (<- 'g') + 'mh' + (<- 'm') + 'ph' + (<- 'p') + 'th' + (<- 't') + ) +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define noun_sfx as ( + [substring] among ( + 'amh' 'eamh' 'abh' 'eabh' + 'aibh' 'ibh' 'aimh' 'imh' + 'a{i'}ocht' '{i'}ocht' 'a{i'}ochta' '{i'}ochta' + (R1 delete) + 'ire' 'ir{i'}' 'aire' 'air{i'}' + (R2 delete) + ) + ) + define deriv as ( + [substring] among ( + 'acht' 'eacht' 'ach' 'each' 'eacht{u'}il' 'eachta' 'acht{u'}il' 'achta' + (R2 delete) //siopadóireacht -> siopadóir but not poblacht -> pobl + 'arcacht' 'arcachta{i'}' 'arcachta' + (<- 'arc') // monarcacht -> monarc + 'gineach' 'gineas' 'ginis' + (<- 'gin') + 'grafa{i'}och' 'grafa{i'}ocht' 'grafa{i'}ochta' 'grafa{i'}ochta{i'}' + (<- 'graf') + 'paite' 'patach' 'pataigh' 'patacha' + (<- 'paite') + '{o'}ideach' '{o'}ideacha' '{o'}idigh' + (<- '{o'}id') + ) + ) + define verb_sfx as ( + [substring] among ( + 'imid' 'aimid' '{i'}mid' 'a{i'}mid' + 'faidh' 'fidh' + (RV delete) + 'ain' + 'eadh' 'adh' + '{a'}il' + 'tear' 'tar' + (R1 delete) + ) + ) +) + +define stem as ( + do initial_morph + do mark_regions + backwards ( + do noun_sfx + do deriv + do verb_sfx + ) +) diff --git a/contrib/snowball/algorithms/italian.sbl b/contrib/snowball/algorithms/italian.sbl new file mode 100644 index 0000000..bf0c161 --- /dev/null +++ b/contrib/snowball/algorithms/italian.sbl @@ -0,0 +1,195 @@ + +routines ( + prelude postlude mark_regions + RV R1 R2 + attached_pronoun + standard_suffix + verb_suffix + vowel_suffix +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v AEIO CG ) + +stringescapes {} + +/* special characters */ + +stringdef a' '{U+00E1}' +stringdef a` '{U+00E0}' +stringdef e' '{U+00E9}' +stringdef e` '{U+00E8}' +stringdef i' '{U+00ED}' +stringdef i` '{U+00EC}' +stringdef o' '{U+00F3}' +stringdef o` '{U+00F2}' +stringdef u' '{U+00FA}' +stringdef u` '{U+00F9}' + +define v 'aeiou{a`}{e`}{i`}{o`}{u`}' + +define prelude as ( + test repeat ( + [substring] among( + '{a'}' (<- '{a`}') + '{e'}' (<- '{e`}') + '{i'}' (<- '{i`}') + '{o'}' (<- '{o`}') + '{u'}' (<- '{u`}') + 'qu' (<- 'qU') + '' (next) + ) + ) + repeat goto ( + v [ ('u' ] v <- 'U') or + ('i' ] v <- 'I') + ) +) + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v (non-v gopast v) or (v gopast non-v) ) + or + ( non-v (non-v gopast v) or (v next) ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define postlude as repeat ( + + [substring] among( + 'I' (<- 'i') + 'U' (<- 'u') + '' (next) + ) + +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define attached_pronoun as ( + [substring] among( + 'ci' 'gli' 'la' 'le' 'li' 'lo' + 'mi' 'ne' 'si' 'ti' 'vi' + // the compound forms are: + 'sene' 'gliela' 'gliele' 'glieli' 'glielo' 'gliene' + 'mela' 'mele' 'meli' 'melo' 'mene' + 'tela' 'tele' 'teli' 'telo' 'tene' + 'cela' 'cele' 'celi' 'celo' 'cene' + 'vela' 'vele' 'veli' 'velo' 'vene' + ) + among( (RV) + 'ando' 'endo' (delete) + 'ar' 'er' 'ir' (<- 'e') + ) + ) + + define standard_suffix as ( + [substring] among( + + 'anza' 'anze' 'ico' 'ici' 'ica' 'ice' 'iche' 'ichi' 'ismo' + 'ismi' 'abile' 'abili' 'ibile' 'ibili' 'ista' 'iste' 'isti' + 'ist{a`}' 'ist{e`}' 'ist{i`}' 'oso' 'osi' 'osa' 'ose' 'mente' + 'atrice' 'atrici' + 'ante' 'anti' // Note 1 + ( R2 delete ) + 'azione' 'azioni' 'atore' 'atori' + ( R2 delete + try ( ['ic'] R2 delete ) + ) + 'logia' 'logie' + ( R2 <- 'log' ) + 'uzione' 'uzioni' 'usione' 'usioni' + ( R2 <- 'u' ) + 'enza' 'enze' + ( R2 <- 'ente' ) + 'amento' 'amenti' 'imento' 'imenti' + ( RV delete ) + 'amente' ( + R1 delete + try ( + [substring] R2 delete among( + 'iv' ( ['at'] R2 delete ) + 'os' 'ic' 'abil' + ) + ) + ) + 'it{a`}' ( + R2 delete + try ( + [substring] among( + 'abil' 'ic' 'iv' (R2 delete) + ) + ) + ) + 'ivo' 'ivi' 'iva' 'ive' ( + R2 delete + try ( ['at'] R2 delete ['ic'] R2 delete ) + ) + ) + ) + + define verb_suffix as setlimit tomark pV for ( + [substring] among( + 'ammo' 'ando' 'ano' 'are' 'arono' 'asse' 'assero' 'assi' + 'assimo' 'ata' 'ate' 'ati' 'ato' 'ava' 'avamo' 'avano' 'avate' + 'avi' 'avo' 'emmo' 'enda' 'ende' 'endi' 'endo' 'er{a`}' 'erai' + 'eranno' 'ere' 'erebbe' 'erebbero' 'erei' 'eremmo' 'eremo' + 'ereste' 'eresti' 'erete' 'er{o`}' 'erono' 'essero' 'ete' + 'eva' 'evamo' 'evano' 'evate' 'evi' 'evo' 'Yamo' 'iamo' 'immo' + 'ir{a`}' 'irai' 'iranno' 'ire' 'irebbe' 'irebbero' 'irei' + 'iremmo' 'iremo' 'ireste' 'iresti' 'irete' 'ir{o`}' 'irono' + 'isca' 'iscano' 'isce' 'isci' 'isco' 'iscono' 'issero' 'ita' + 'ite' 'iti' 'ito' 'iva' 'ivamo' 'ivano' 'ivate' 'ivi' 'ivo' + 'ono' 'uta' 'ute' 'uti' 'uto' + + 'ar' 'ir' // but 'er' is problematical + (delete) + ) + ) + + define AEIO 'aeio{a`}{e`}{i`}{o`}' + define CG 'cg' + + define vowel_suffix as ( + try ( + [AEIO] RV delete + ['i'] RV delete + ) + try ( + ['h'] CG RV delete + ) + ) +) + +define stem as ( + do prelude + do mark_regions + backwards ( + do attached_pronoun + do (standard_suffix or verb_suffix) + do vowel_suffix + ) + do postlude +) + +/* + Note 1: additions of 15 Jun 2005 +*/ + diff --git a/contrib/snowball/algorithms/kraaij_pohlmann.sbl b/contrib/snowball/algorithms/kraaij_pohlmann.sbl new file mode 100644 index 0000000..409bf0f --- /dev/null +++ b/contrib/snowball/algorithms/kraaij_pohlmann.sbl @@ -0,0 +1,240 @@ +strings ( ch ) +integers ( p1 p2 ) +booleans ( Y_found stemmed GE_removed ) + +routines ( + + R1 R2 + C V VX + lengthen_V + Step_1 Step_2 Step_3 Step_4 Step_7 + Step_6 Step_1c + Lose_prefix + Lose_infix + measure +) + +externals ( stem ) + +groupings ( v v_WX AOU AIOU ) + +stringescapes {} + +define v 'aeiouy' +define v_WX v + 'wx' +define AOU 'aou' +define AIOU 'aiou' + +backwardmode ( + + define R1 as ($p1 <= cursor) + define R2 as ($p2 <= cursor) + + define V as test (v or 'ij') + define VX as test (next v or 'ij') + define C as test (not 'ij' non-v) + + define lengthen_V as do ( + non-v_WX [ (AOU] test (non-v or atlimit)) or + ('e'] test (non-v or atlimit + not AIOU + not (next AIOU non-v))) + ->ch insert ch + ) + + define Step_1 as + ( + [substring] among ( + + '{'}s' (delete) + 's' (R1 not ('t' R1) C delete) + 'ies' (R1 <-'ie') + 'es' + (('ar' R1 C ] delete lengthen_V) or + ('er' R1 C ] delete) or + (R1 C <-'e')) + + 'aus' (R1 V <-'au') + 'en' (('hed' R1 ] <-'heid') or + ('nd' delete) or + ('d' R1 C ] delete) or + ('i' or 'j' V delete) or + (R1 C delete lengthen_V)) + 'nde' (<-'nd') + ) + ) + + define Step_2 as + ( + [substring] among ( + 'je' (('{'}t' ] delete) or + ('et' ] R1 C delete) or + ('rnt' ] <-'rn') or + ('t' ] R1 VX delete) or + ('ink' ] <-'ing') or + ('mp' ] <-'m') or + ('{'}' ] R1 delete) or + (] R1 C delete)) + 'ge' (R1 <-'g') + 'lijke'(R1 <-'lijk') + 'ische'(R1 <-'isch') + 'de' (R1 C delete) + 'te' (R1 <-'t') + 'se' (R1 <-'s') + 're' (R1 <-'r') + 'le' (R1 delete attach 'l' lengthen_V) + 'ene' (R1 C delete attach 'en' lengthen_V) + 'ieve' (R1 C <-'ief') + ) + ) + + define Step_3 as + ( + [substring] among ( + 'atie' (R1 <-'eer') + 'iteit' (R1 delete lengthen_V) + 'heid' + 'sel' + 'ster' (R1 delete) + 'rder' (<-'r') + 'ing' + 'isme' + 'erij' (R1 delete lengthen_V) + 'arij' (R1 C <-'aar') + 'fie' (R2 delete attach 'f' lengthen_V) + 'gie' (R2 delete attach 'g' lengthen_V) + 'tst' (R1 C <-'t') + 'dst' (R1 C <-'d') + ) + ) + + define Step_4 as + ( + ( [substring] among ( + 'ioneel' (R1 <-'ie') + 'atief' (R1 <-'eer') + 'baar' (R1 delete) + 'naar' (R1 V <-'n') + 'laar' (R1 V <-'l') + 'raar' (R1 V <-'r') + 'tant' (R1 <-'teer') + 'lijker' + 'lijkst' (R1 <-'lijk') + 'achtig' + 'achtiger' + 'achtigst'(R1 delete) + 'eriger' + 'erigst' + 'erig' + 'end' (R1 C delete lengthen_V) + ) + ) + or + ( [substring] among ( + 'iger' + 'igst' + 'ig' (R1 C delete lengthen_V) + ) + ) + ) + + define Step_7 as + ( + [substring] among ( + 'kt' (<-'k') + 'ft' (<-'f') + 'pt' (<-'p') + ) + ) + + define Step_6 as + ( + [substring] among ( + 'bb' (<-'b') + 'cc' (<-'c') + 'dd' (<-'d') + 'ff' (<-'f') + 'gg' (<-'g') + 'hh' (<-'h') + 'jj' (<-'j') + 'kk' (<-'k') + 'll' (<-'l') + 'mm' (<-'m') + 'nn' (<-'n') + 'pp' (<-'p') + 'qq' (<-'q') + 'rr' (<-'r') + 'ss' (<-'s') + 'tt' (<-'t') + 'vv' (<-'v') + 'ww' (<-'w') + 'xx' (<-'x') + 'zz' (<-'z') + 'v' (<-'f') + 'z' (<-'s') + ) + ) + + define Step_1c as + ( + [substring] among ( (R1 C) + 'd' (not ('n' R1) delete) + 't' (not ('h' R1) delete) + ) + ) +) + +define Lose_prefix as ( + ['ge'] test hop 3 (goto v goto non-v) + set GE_removed + delete +) + +define Lose_infix as ( + next + gopast (['ge']) test hop 3 (goto v goto non-v) + set GE_removed + delete +) + +define measure as ( + $p1 = limit + $p2 = limit + do( + repeat non-v atleast 1 ('ij' or v) non-v setmark p1 + repeat non-v atleast 1 ('ij' or v) non-v setmark p2 + ) + +) +define stem as ( + + unset Y_found + unset stemmed + do ( ['y'] <-'Y' set Y_found ) + do repeat(goto (v ['y'])<-'Y' set Y_found ) + + measure + + backwards ( + do (Step_1 set stemmed ) + do (Step_2 set stemmed ) + do (Step_3 set stemmed ) + do (Step_4 set stemmed ) + ) + unset GE_removed + do (Lose_prefix and measure) + backwards ( + do (GE_removed Step_1c) + ) + unset GE_removed + do (Lose_infix and measure) + backwards ( + do (GE_removed Step_1c) + ) + backwards ( + do (Step_7 set stemmed ) + do (stemmed or GE_removed Step_6) + ) + do(Y_found repeat(goto (['Y']) <-'y')) +) + diff --git a/contrib/snowball/algorithms/lithuanian.sbl b/contrib/snowball/algorithms/lithuanian.sbl new file mode 100644 index 0000000..ff7fdb9 --- /dev/null +++ b/contrib/snowball/algorithms/lithuanian.sbl @@ -0,0 +1,373 @@ +externals ( stem ) + +// escape symbols for substituting lithuanian characters +stringescapes { } + +/* Special characters in Unicode Latin Extended-A */ +// ' nosine +stringdef a' '{U+0105}' // ą a + ogonek +stringdef e' '{U+0119}' // ę e + ogonek +stringdef i' '{U+012F}' // į i + ogonek +stringdef u' '{U+0173}' // ų u + ogonek + +// . taskas +stringdef e. '{U+0117}' // ė e + dot + +// - ilgoji +stringdef u- '{U+016B}' // ū u + macron + +// * varnele +stringdef c* '{U+010D}' // č c + caron (haček) +stringdef s* '{U+0161}' // š s + caron (haček) +stringdef z* '{U+017E}' // ž z + caron (haček) + +// [C](VC)^m[V|C] +// definitions of variables for +// p1 - position of m = 0 +integers ( p1 ) + +// groupings +// v - lithuanian vowels +groupings ( v ) + +// v - all lithuanian vowels +define v 'aeiyou{a'}{e'}{i'}{u'}{e.}{u-}' + +// all lithuanian stemmer routines: 4 steps +routines ( + step2 R1 step1 fix_chdz fix_gd fix_conflicts +) + +backwardmode ( + + define R1 as $p1 <= cursor + define step1 as ( + setlimit tomark p1 for ([substring]) R1 among ( + // Daiktavardžiai (Nouns) + // I linksniuotė (declension I) + 'as' 'ias' 'is' 'ys' // vyras, kelias, brolis, gaidys + 'o' 'io' // vyro, kelio + 'ui' 'iui' // vyrui, keliui + '{a'}' 'i{a'}' '{i'}' // vyrą, kelią, brolį + 'u' 'iu' // vyru, keliu + 'e' 'yje' // vyre, kelyje + 'y' 'au' 'i' // kely, brolau, broli, + 'an' // nusižengiman + + 'ai' 'iai' // vyrai, keliai + '{u'}' 'i{u'}' // vyrų, kelių + 'ams' 'am' // vyrams, vyram + 'iams' 'iam' // broliams, broliam + 'us' 'ius' // vyrus, brolius + 'ais' 'iais' // vyrais, keliais + 'uose' 'iuose' 'uos' 'iuos' // vyruose, keliuose, vyruos, keliuos + 'uosna' 'iuosna' // vyruosna, keliuosna + 'ysna' // žutysna + + 'asis' 'aisi' // sukimasis, sukimaisi + 'osi' '{u'}si' // sukimosi, sukimųsi + 'uisi' // sukimuisi + '{a'}si' // sukimąsi + 'usi' // sukimusi + 'esi' // sukimesi + + 'uo' // mėnuo + + + // II linksniuote (declension II) + 'a' 'ia' // galva, vysnios + 'os' 'ios' // galvos, vysnios + 'oj' 'oje' 'ioje' // galvoje, vysnioje + 'osna' 'iosna' // galvosna, vyšniosna + 'om' 'oms' 'ioms' // galvoms, vysnioms + 'omis' 'iomis' // galvomis, vysniomis + 'ose' 'iose' // galvose, vysniose + 'on' 'ion' // galvon, vyšnion + + + // III linksniuote (declension III) + '{e.}' // gervė + '{e.}s' // gervės + 'ei' // gervei + '{e'}' // gervę + '{e.}j' '{e.}je' // gervėj, gervėje + '{e.}ms' // gervėms + 'es' // gerves + '{e.}mis' // gervėmis + '{e.}se' // gervėse + '{e.}sna' // gervėsna + '{e.}n' // žydaitėn + + + // IV linksniuote (declension IV) + 'aus' 'iaus' // sūnaus, skaičiaus + 'umi' 'iumi' // sūnumi, skaičiumi + 'uje' 'iuje' // sūnuje, skaičiuje + 'iau' // skaičiau + + '{u-}s' // sūnūs + 'ums' // sūnums + 'umis' // sūnumis + 'un' 'iun' // sūnun, administratoriun + + + // V linksniuote (declension V) + 'ies' 'ens' 'enio' 'ers' // avies, vandens, sesers + 'eniui' 'eriai' // vandeniui, eriai + 'en{i'}' 'er{i'}' // vandenį, seserį + 'imi' 'eniu' 'erimi' 'eria' // avimi, vandeniu, seserimi, seseria + 'enyje' 'eryje' // vandenyje, seseryje + 'ie' 'enie' 'erie' // avie, vandenie, seserie + + 'enys' 'erys' // vandenys, seserys + // 'en{u'}' konfliktas su 'žandenų' 'antenų' + 'er{u'}' // seserų + 'ims' 'enims' 'erims' // avims, vandemins, seserims + 'enis' // vandenis + 'imis' // žebenkštimis + 'enimis' // vandenimis + 'yse' 'enyse' 'eryse' // avyse, vandenyse, seseryse + + + // Būdvardžiai (Adjectives) + // (i)a linksniuotė + 'iem' 'iems' // geriem, geriems + 'ame' 'iame' // naujame, mediniame + + + // Veiksmažodžiai (Verbs) + // Tiesioginė nuosaka (indicative mood) + // esamasis laikas (present tense) + // (i)a asmenuotė (declension (i)a) + 'uosi' 'iuosi' // dirbuosi, traukiuosi + 'iesi' // dirbiesi + 'asi' 'iasi' // dirbasi, traukiasi + 'am{e.}s' 'iam{e.}s' // dirbamės, traukiamės + 'at' 'ate' 'iat' 'iate' // dirbat, dirbate, ariat, traukiate + 'at{e.}s' 'iat{e.}s' // dirbatės, traukiatės + + // i asmenuotė (declension i) + 'isi' // tikisi + 'im' // mylim + // 'ime' konfliktassu daiktavardžiu vietininku, pvz. 'gėrime' + 'im{e.}s' // tikimės + 'it' 'ite' // mylit, mylite, tikitės + // 'it{e.}s' konfliktas su priesaga ir dgs. vardininko galūne -ait-ės pvz. žydaitės + + // o asmenuotė (declension o) + 'ome' // mokome + 'ot' 'ote' // mokot, mokote + + // būtasis laikas + // o asmenuotė (declension o) + '{e.}jo' '{e.}josi' // tikėjo, tikėjosi + 'ot{e.}s' // tikėjotės/bijotės + + // ė asmenuotė (declension ė) + 'eisi' // mokeisi + '{e.}si' // mokėsi + '{e.}m' '{e.}me' // mokėm, mokėme + '{e.}m{e.}s' // mokėmės + '{e.}t' '{e.}te' // mokėt, mokėte + '{e.}t{e.}s' // mokėtės + + // būtasis dažninis laikas (frequentative past tense) + 'ausi' // mokydavausi + 'om{e.}s' // mokydavomės/bijomės + + + // būsimasis laikas (future tense) + 'siu' 'siuosi' // dirbsiu, mokysiuosi + 'si' 'siesi' // dirbsi, dirbsiesi + 's' 'ysis' // dirbs, mokysis + 'sim' 'sime' // dirbsim, dirbsime + 'sit' 'site' // gersit, gersite + + // tariamoji nuosaka (subjunctive mood) + '{c*}iau' '{c*}iausi' // dirbčiau + 'tum' 'tumei' // dirbtum, dirbtumei + 'tumeis' 'tumeisi' // mokytumeis, mokytumeisi + // 't{u'}' nes blogai batutų -> batų + 't{u'}si' // mokytųsi + // 'tume' konfliktas su 'šventume' + 'tum{e.}m' // dirbtumėm + 'tum{e.}me' // dirbtumėme + 'tum{e.}m{e.}s' // mokytumėmės + 'tute' 'tum{e.}t' // dirbtute, dirbtumėt + 'tum{e.}te' // dirbtumėte + 'tum{e.}t{e.}s' // mokytumėtės + + // liepiamoji nuosaka (imperative mood) + 'k' 'ki' // dirbk, dirbki, mokykis + // 'kis' konfliktas viln-išk-is + // 'kime' konfliktas, nes pirkime + 'kim{e.}s' // mokykimės + + // bendratis (infinitive) + 'uoti' 'iuoti' // meluoti, dygsniuoti + 'auti' 'iauti' // draugauti, girtuokliauti + 'oti' 'ioti' // dovanoti, meškerioti + '{e.}ti' // auklėti + 'yti' // akyti + 'inti' // auginti + 'in{e.}ti' // blusinėti + 'enti' // gyventi + 'tel{e.}ti' // bumbtelėti + 'ter{e.}ti' // bumbterėti + + 'ti' // skalbti + // 'tis' konfliktas, nes rytme-tis -> rytme + + // dalyviai (participles) + '{a'}s' 'i{a'}s' '{i'}s' // dirbąs, žaidžiąs, gulįs + 't{u'}s' // suktųs -> suk + 'sim{e.}s' // suksimės + 'sit{e.}s' // suksitės + 'kite' // supkite + ) + + delete + ) + + define step2 as repeat ( + setlimit tomark p1 for ([substring]) among ( + // daiktavardziu priesagos (Noun suffixes) + + // budvardziu priesagos (Adjective suffixes) + // 'in' // konfliktas su 'augintinis' ir 'akiniais' // lauk-in-is + 'ing' // tvark-ing-as + 'i{s*}k' // lenk-išk-as + '{e.}t' // dem-ėt-as + 'ot' // garban-ot-as + 'uot' 'iuot' // lang-uot-as, akin-iuot-as + // 'tin', nes augintinis // dirb-tin-is + // 'ut', nes batutas, degutas etc. // maž-ut-is + 'yt' // maž-yt-is + 'iuk' // maž-iuk-as + 'iul' // maž-ul-is + '{e.}l' // maž-ėl-is + 'yl' // maž-yl-is + 'u{c*}iuk' // maž-učiuk-as + 'uliuk' // maž-uliuk-as + 'ut{e.}ait' // maž-utėlait-is + 'ok' // did-ok-as + 'iok' // višč-iok-as + 'sv' '{s*}v' 'zgan' // sal-sv-as, pilk-šv-as, bal-zgan-as + 'op' 'iop' // dvej-op-as, viener-iop-as + 'ain' // apval-ain-as + 'yk{s*}t' 'yk{s*}{c*}' // ten-ykšt-is, vakar-ykšč-ias + + // laisniai + 'esn' // did-esn-is + 'aus' 'iaus' // nauj-aus-ias, ger-iaus-ias + + // ivardziuotiniai budvardziai (Pronominal adjectives) + // vyriska gimine (Male gender) + 'ias' // žaliasis + 'oj' 'ioj' // gerojo, žaliojo + 'aj' 'iaj' // gerajam, žaliajam + '{a'}j' 'i{a'}j' // garąjį, žaliąjį + 'uoj' 'iuoj' // geruoju, žaliuoju + 'iej' // gerieji + '{u'}j' 'i{u'}j' // gerųjų, žaliųjų + 'ies' // geriesiems + 'uos' 'iuos' // geruosius, žaliuosius + 'ais' 'iais' // geraisiais, žaliaisiais + + // moteriska gimine (Female gender) + 'os' 'ios' // gerosios, žaliosios + '{a'}s' 'i{a'}s' // gerąsios, žaliąsias + + // būtasis dažninis laikas (frequentative past tense) + 'dav' // ei-dav-o + + // dalyvių priesagos (particple suffix) + 'ant' 'iant' + 'int' // tur-int-is + '{e.}j' // tur-ėj-o + '{e'}' // + '{e.}j{e'}' + '{e'}s' // dirb-ęs-is + + 'siant' // dirb-siant + + // pusdalyviai (participle) + 'dam' // bėg-dam-as + + 'auj' // ūkinink-auj-a + 'jam' + 'iau' + 'am' // baiminim-ams-i + ) + + delete + ) + + define fix_conflicts as ( + [substring] among ( + // 'lietuvaite' -> 'lietuvaitė', konfliktas su 'myl-ite' + 'aite' (<-'ait{e.}') + // 'lietuvaitės' -> 'lietuvaitė', konfliktas su 'myl-itės' + 'ait{e.}s' (<-'ait{e.}') + + // ''ūs-uotės' -> 'ūs-uotė', konfliktas 'mokotės' + 'uot{e.}s' (<-'uot{e.}') + // ''ūs-uote' -> 'ūs-uotė', konfliktas 'mokote' + 'uote' (<-'uot{e.}') + + // 'žerėjime' -> 'žėrėjimas', konfliktas su 'žais-ime' + '{e.}jime' (<-'{e.}jimas') + + // 'žvilgesiu' -> 'žvilgesys', konfliktas su 'dirb-siu' + 'esiu' (<-'esys') + // 'duobkasiu' -> 'duobkasys', konfliktas su 'pakasiu' + 'asius' (<-'asys') + + // 'žioravime' -> 'žioravimas', konfliktas su 'myl-ime' + 'avime' (<-'avimas') + 'ojime' (<-'ojimas') + + // 'advokatės' -> 'advokatė', konfliktas su 'dirb-atės' + 'okat{e.}s' (<-'okat{e.}') + // 'advokate' -> 'advokatė', konfliktas su 'dirb-ate' + 'okate' (<-'okat{e.}') + ) + ) + + define fix_chdz as ( + [substring] among ( + '{c*}' (<-'t') + 'd{z*}' (<-'d') + ) + ) + + define fix_gd as ( + [substring] among ( + 'gd' (<-'g') + // '{e.}k' (<-'{e.}g') + ) + ) + +) + +define stem as ( + + $p1 = limit + + do ( + // priešdėlis 'a' ilgeniuose nei 6 raidės žodžiuose, pvz. 'a-liejus'. + try (test 'a' $(len > 6) hop 1) + + gopast v gopast non-v setmark p1 + ) + + backwards ( + do fix_conflicts + do step1 + do fix_chdz + do step2 + do fix_chdz + do fix_gd + ) + +) diff --git a/contrib/snowball/algorithms/lovins.sbl b/contrib/snowball/algorithms/lovins.sbl new file mode 100644 index 0000000..3f69f15 --- /dev/null +++ b/contrib/snowball/algorithms/lovins.sbl @@ -0,0 +1,208 @@ + +stringescapes {} + +routines ( + A B C D E F G H I J K L M N O P Q R S T U V W X Y Z AA BB CC + + endings + + undouble respell +) + +externals ( stem ) + +backwardmode ( + + /* Lovins' conditions A, B ... CC, as given in her Appendix B, where + a test for a two letter prefix ('test hop 2') is implicitly + assumed. Note that 'e' next 'u' corresponds to her u*e because + Snowball is scanning backwards. */ + + define A as ( hop 2 ) + define B as ( hop 3 ) + define C as ( hop 4 ) + define D as ( hop 5 ) + define E as ( test hop 2 not 'e' ) + define F as ( test hop 3 not 'e' ) + define G as ( test hop 3 'f' ) + define H as ( test hop 2 't' or 'll' ) + define I as ( test hop 2 not 'o' not 'e' ) + define J as ( test hop 2 not 'a' not 'e' ) + define K as ( test hop 3 'l' or 'i' or ('e' next 'u') ) + define L as ( test hop 2 not 'u' not 'x' not ('s' not 'o') ) + define M as ( test hop 2 not 'a' not 'c' not 'e' not 'm' ) + define N as ( test hop 3 ( hop 2 not 's' or hop 2 ) ) + define O as ( test hop 2 'l' or 'i' ) + define P as ( test hop 2 not 'c' ) + define Q as ( test hop 2 test hop 3 not 'l' not 'n' ) + define R as ( test hop 2 'n' or 'r' ) + define S as ( test hop 2 'dr' or ('t' not 't') ) + define T as ( test hop 2 's' or ('t' not 'o') ) + define U as ( test hop 2 'l' or 'm' or 'n' or 'r' ) + define V as ( test hop 2 'c' ) + define W as ( test hop 2 not 's' not 'u' ) + define X as ( test hop 2 'l' or 'i' or ('e' next 'u') ) + define Y as ( test hop 2 'in' ) + define Z as ( test hop 2 not 'f' ) + define AA as ( test hop 2 among ( 'd' 'f' 'ph' 'th' 'l' 'er' 'or' + 'es' 't' ) ) + define BB as ( test hop 3 not 'met' not 'ryst' ) + define CC as ( test hop 2 'l' ) + + + /* The system of endings, as given in Appendix A. */ + + define endings as ( + [substring] among( + 'alistically' B 'arizability' A 'izationally' B + + 'antialness' A 'arisations' A 'arizations' A 'entialness' A + + 'allically' C 'antaneous' A 'antiality' A 'arisation' A + 'arization' A 'ationally' B 'ativeness' A 'eableness' E + 'entations' A 'entiality' A 'entialize' A 'entiation' A + 'ionalness' A 'istically' A 'itousness' A 'izability' A + 'izational' A + + 'ableness' A 'arizable' A 'entation' A 'entially' A + 'eousness' A 'ibleness' A 'icalness' A 'ionalism' A + 'ionality' A 'ionalize' A 'iousness' A 'izations' A + 'lessness' A + + 'ability' A 'aically' A 'alistic' B 'alities' A + 'ariness' E 'aristic' A 'arizing' A 'ateness' A + 'atingly' A 'ational' B 'atively' A 'ativism' A + 'elihood' E 'encible' A 'entally' A 'entials' A + 'entiate' A 'entness' A 'fulness' A 'ibility' A + 'icalism' A 'icalist' A 'icality' A 'icalize' A + 'ication' G 'icianry' A 'ination' A 'ingness' A + 'ionally' A 'isation' A 'ishness' A 'istical' A + 'iteness' A 'iveness' A 'ivistic' A 'ivities' A + 'ization' F 'izement' A 'oidally' A 'ousness' A + + 'aceous' A 'acious' B 'action' G 'alness' A + 'ancial' A 'ancies' A 'ancing' B 'ariser' A + 'arized' A 'arizer' A 'atable' A 'ations' B + 'atives' A 'eature' Z 'efully' A 'encies' A + 'encing' A 'ential' A 'enting' C 'entist' A + 'eously' A 'ialist' A 'iality' A 'ialize' A + 'ically' A 'icance' A 'icians' A 'icists' A + 'ifully' A 'ionals' A 'ionate' D 'ioning' A + 'ionist' A 'iously' A 'istics' A 'izable' E + 'lessly' A 'nesses' A 'oidism' A + + 'acies' A 'acity' A 'aging' B 'aical' A + 'alist' A 'alism' B 'ality' A 'alize' A + 'allic'BB 'anced' B 'ances' B 'antic' C + 'arial' A 'aries' A 'arily' A 'arity' B + 'arize' A 'aroid' A 'ately' A 'ating' I + 'ation' B 'ative' A 'ators' A 'atory' A + 'ature' E 'early' Y 'ehood' A 'eless' A + 'elity' A 'ement' A 'enced' A 'ences' A + 'eness' E 'ening' E 'ental' A 'ented' C + 'ently' A 'fully' A 'ially' A 'icant' A + 'ician' A 'icide' A 'icism' A 'icist' A + 'icity' A 'idine' I 'iedly' A 'ihood' A + 'inate' A 'iness' A 'ingly' B 'inism' J + 'inity'CC 'ional' A 'ioned' A 'ished' A + 'istic' A 'ities' A 'itous' A 'ively' A + 'ivity' A 'izers' F 'izing' F 'oidal' A + 'oides' A 'otide' A 'ously' A + + 'able' A 'ably' A 'ages' B 'ally' B + 'ance' B 'ancy' B 'ants' B 'aric' A + 'arly' K 'ated' I 'ates' A 'atic' B + 'ator' A 'ealy' Y 'edly' E 'eful' A + 'eity' A 'ence' A 'ency' A 'ened' E + 'enly' E 'eous' A 'hood' A 'ials' A + 'ians' A 'ible' A 'ibly' A 'ical' A + 'ides' L 'iers' A 'iful' A 'ines' M + 'ings' N 'ions' B 'ious' A 'isms' B + 'ists' A 'itic' H 'ized' F 'izer' F + 'less' A 'lily' A 'ness' A 'ogen' A + 'ward' A 'wise' A 'ying' B 'yish' A + + 'acy' A 'age' B 'aic' A 'als'BB + 'ant' B 'ars' O 'ary' F 'ata' A + 'ate' A 'eal' Y 'ear' Y 'ely' E + 'ene' E 'ent' C 'ery' E 'ese' A + 'ful' A 'ial' A 'ian' A 'ics' A + 'ide' L 'ied' A 'ier' A 'ies' P + 'ily' A 'ine' M 'ing' N 'ion' Q + 'ish' C 'ism' B 'ist' A 'ite'AA + 'ity' A 'ium' A 'ive' A 'ize' F + 'oid' A 'one' R 'ous' A + + 'ae' A 'al'BB 'ar' X 'as' B + 'ed' E 'en' F 'es' E 'ia' A + 'ic' A 'is' A 'ly' B 'on' S + 'or' T 'um' U 'us' V 'yl' R + '{'}s' A 's{'}' A + + 'a' A 'e' A 'i' A 'o' A + 's' W 'y' B + + (delete) + ) + ) + + /* Undoubling is rule 1 of appendix C. */ + + define undouble as ( + test substring among ('bb' 'dd' 'gg' 'll' 'mm' 'nn' 'pp' 'rr' 'ss' + 'tt') + [next] delete + ) + + /* The other appendix C rules can be done together. */ + + define respell as ( + [substring] among ( + 'iev' (<-'ief') + 'uct' (<-'uc') + 'umpt' (<-'um') + 'rpt' (<-'rb') + 'urs' (<-'ur') + 'istr' (<-'ister') + 'metr' (<-'meter') + 'olv' (<-'olut') + 'ul' (not 'a' not 'i' not 'o' <-'l') + 'bex' (<-'bic') + 'dex' (<-'dic') + 'pex' (<-'pic') + 'tex' (<-'tic') + 'ax' (<-'ac') + 'ex' (<-'ec') + 'ix' (<-'ic') + 'lux' (<-'luc') + 'uad' (<-'uas') + 'vad' (<-'vas') + 'cid' (<-'cis') + 'lid' (<-'lis') + 'erid' (<-'eris') + 'pand' (<-'pans') + 'end' (not 's' <-'ens') + 'ond' (<-'ons') + 'lud' (<-'lus') + 'rud' (<-'rus') + 'her' (not 'p' not 't' <-'hes') + 'mit' (<-'mis') + 'ent' (not 'm' <-'ens') + /* 'ent' was 'end' in the 1968 paper - a typo. */ + 'ert' (<-'ers') + 'et' (not 'n' <-'es') + 'yt' (<-'ys') + 'yz' (<-'ys') + ) + ) +) + +define stem as ( + + backwards ( + do endings + do undouble + do respell + ) +) + diff --git a/contrib/snowball/algorithms/nepali.sbl b/contrib/snowball/algorithms/nepali.sbl new file mode 100644 index 0000000..d388748 --- /dev/null +++ b/contrib/snowball/algorithms/nepali.sbl @@ -0,0 +1,92 @@ +/* + * Authors: + * - Ingroj Shrestha <ing.stha@gmail.com>, Nepali NLP Group + * - Oleg Bartunov <obartunov@gmail.com>, Postgres Professional Ltd. + * - Shreeya Singh Dhakal, Nepali NLP Group + */ + +routines ( + remove_category_1 + check_category_2 + remove_category_2 + remove_category_3 +) + +stringescapes {} + +stringdef dsc '{U+0901}' // DEVANAGARI_SIGN_CANDRABINDU +stringdef dsa '{U+0902}' // DEVANAGARI_SIGN_ANUSVARA +stringdef dli '{U+0907}' // DEVANAGARI_LETTER_I +stringdef dlii '{U+0908}' // DEVANAGARI_LETTER_II +stringdef dle '{U+090F}' // DEVANAGARI_LETTER_E +stringdef dlka '{U+0915}' // DEVANAGARI_LETTER_KA +stringdef dlkha '{U+0916}' // DEVANAGARI_LETTER_KHA +stringdef dlg '{U+0917}' // DEVANAGARI_LETTER_GA +stringdef dlc '{U+091B}' // DEVANAGARI_LETTER_CHA +stringdef dlta '{U+0924}' // DEVANAGARI_LETTER_TA +stringdef dltha '{U+0925}' // DEVANAGARI_LETTER_THA +stringdef dld '{U+0926}' // DEVANAGARI_LETTER_DA +stringdef dln '{U+0928}' // DEVANAGARI_LETTER_NA +stringdef dlpa '{U+092A}' // DEVANAGARI_LETTER_PA +stringdef dlpha '{U+092B}' // DEVANAGARI_LETTER_PHA +stringdef dlb '{U+092D}' // DEVANAGARI_LETTER_BHA +stringdef dlm '{U+092E}' // DEVANAGARI_LETTER_MA +stringdef dly '{U+092F}' // DEVANAGARI_LETTER_YA +stringdef dlr '{U+0930}' // DEVANAGARI_LETTER_RA +stringdef dll '{U+0932}' // DEVANAGARI_LETTER_LA +stringdef dlv '{U+0935}' // DEVANAGARI_LETTER_VA +stringdef dls '{U+0938}' // DEVANAGARI_LETTER_SA +stringdef dlh '{U+0939}' // DEVANAGARI_LETTER_HA +stringdef dvsaa '{U+093E}' // DEVANAGARI_VOWEL_SIGN_AA +stringdef dvsi '{U+093F}' // DEVANAGARI_VOWEL_SIGN_I +stringdef dvsii '{U+0940}' // DEVANAGARI_VOWEL_SIGN_II +stringdef dvsu '{U+0941}' // DEVANAGARI_VOWEL_SIGN_U +stringdef dvsuu '{U+0942}' // DEVANAGARI_VOWEL_SIGN_UU +stringdef dvse '{U+0947}' // DEVANAGARI_VOWEL_SIGN_E +stringdef dvsai '{U+0948}' // DEVANAGARI_VOWEL_SIGN_AI +stringdef dvso '{U+094B}' // DEVANAGARI_VOWEL_SIGN_O +stringdef dvsau '{U+094C}' // DEVANAGARI_VOWEL_SIGN_AU +stringdef dsv '{U+094D}' // DEVANAGARI_SIGN_VIRAMA + +externals ( stem ) +backwardmode ( + define remove_category_1 as( + [substring] among ( + '{dlm}{dvsaa}{dlr}{dsv}{dlpha}{dlta}' '{dld}{dsv}{dlv}{dvsaa}{dlr}{dvsaa}' '{dls}{dsc}{dlg}{dvsai}' '{dls}{dsa}{dlg}' + '{dls}{dsc}{dlg}' '{dll}{dvsaa}{dli}' '{dll}{dvsaa}{dlii}' '{dlpa}{dlc}{dvsi}' + '{dll}{dvse}' '{dlr}{dlta}' '{dlm}{dvsai}' '{dlm}{dvsaa}' + (delete) + '{dlka}{dvso}' '{dlka}{dvsaa}' '{dlka}{dvsi}' '{dlka}{dvsii}' '{dlka}{dvsai}'(('{dle}' or '{dvse}' ()) or delete) + ) + ) + + define check_category_2 as( + [substring] among( + '{dsc}' '{dsa}' '{dvsai}' + ) + ) + + define remove_category_2 as ( + [substring] among( + '{dsc}' '{dsa}' ('{dly}{dvsau}' or '{dlc}{dvsau}' or '{dln}{dvsau}' or '{dltha}{dvse}' delete) + '{dvsai}' ('{dlta}{dsv}{dlr}' delete) + ) + ) + + define remove_category_3 as( + [substring] among( + '{dltha}{dvsi}{dli}{dls}{dsv}' '{dlh}{dvsu}{dln}{dvse}{dlc}' '{dlh}{dvsu}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dls}{dsv}' '{dln}{dvse}{dlc}{dln}{dsv}' '{dli}{dle}{dlka}{dvsii}' '{dli}{dle}{dlka}{dvsaa}' '{dli}{dle}{dlka}{dvso}' '{dvsi}{dle}{dlka}{dvsii}' '{dvsi}{dle}{dlka}{dvsaa}' '{dvsi}{dle}{dlka}{dvso}' '{dli}{dlc}{dln}{dsv}' '{dvsi}{dlc}{dln}{dsv}' '{dli}{dlc}{dls}{dsv}' '{dvsi}{dlc}{dls}{dsv}' '{dle}{dlc}{dln}{dsv}' '{dvse}{dlc}{dln}{dsv}' '{dle}{dlc}{dls}{dsv}' '{dvse}{dlc}{dls}{dsv}' '{dlc}{dvsi}{dln}{dsv}' '{dlc}{dvse}{dls}{dsv}' '{dlc}{dsv}{dly}{dvsau}' '{dltha}{dvsi}{dln}{dsv}' '{dltha}{dvsi}{dly}{dvso}' '{dltha}{dvsi}{dly}{dvsau}' '{dltha}{dvsi}{dls}{dsv}' '{dltha}{dsv}{dly}{dvso}' '{dltha}{dsv}{dly}{dvsau}' '{dld}{dvsi}{dly}{dvso}' '{dld}{dvse}{dlkha}{dvsi}' '{dld}{dvse}{dlkha}{dvsii}' '{dll}{dvsaa}{dln}{dsv}' '{dlm}{dvsaa}{dltha}{dvsi}' '{dln}{dvse}{dlka}{dvsai}' '{dln}{dvse}{dlka}{dvsaa}' '{dln}{dvse}{dlka}{dvso}' '{dln}{dvse}{dlc}{dvsau}' '{dlh}{dvso}{dls}{dsv}' '{dli}{dln}{dsv}{dlc}' '{dvsi}{dln}{dsv}{dlc}' '{dln}{dvse}{dlc}{dvsu}' '{dli}{dlc}{dvsau}' '{dvsi}{dlc}{dvsau}' '{dli}{dls}{dsv}' '{dvsi}{dls}{dsv}' '{dvsi}{dly}{dvso}' '{dli}{dly}{dvso}' '{dle}{dlka}{dvsaa}' '{dvse}{dlka}{dvsaa}' '{dle}{dlka}{dvsii}' '{dvse}{dlka}{dvsii}' '{dle}{dlka}{dvsai}' '{dvse}{dlka}{dvsai}' '{dle}{dlka}{dvso}' '{dvse}{dlka}{dvso}' '{dle}{dlc}{dvsu}' '{dvse}{dlc}{dvsu}' '{dle}{dlc}{dvsau}' '{dvse}{dlc}{dvsau}' '{dlc}{dln}{dsv}' '{dlc}{dls}{dsv}' '{dltha}{dvsi}{dle}' '{dlpa}{dlr}{dsv}' '{dlb}{dly}{dvso}' '{dlh}{dlr}{dvsu}' '{dlh}{dlr}{dvsuu}' '{dvsi}{dld}{dvsaa}' '{dli}{dld}{dvsaa}' '{dvsi}{dld}{dvso}' '{dli}{dld}{dvso}' '{dvsi}{dld}{dvsai}' '{dli}{dld}{dvsai}' '{dln}{dvse}{dlc}' '{dli}{dlc}' '{dvsi}{dlc}' '{dle}{dlc}' '{dvse}{dlc}' '{dlc}{dvsu}' '{dlc}{dvse}' '{dlc}{dvsau}' '{dltha}{dvsii}' '{dltha}{dvse}' '{dld}{dvsaa}' '{dld}{dvsii}' '{dld}{dvsai}' '{dld}{dvso}' '{dln}{dvsu}' '{dln}{dvse}' '{dly}{dvso}' '{dly}{dvsau}' '{dlc}' + (delete) + ) + ) + +) + +define stem as ( + backwards ( + do remove_category_1 + do ( + repeat (do (check_category_2 and remove_category_2) remove_category_3) + ) + ) +) diff --git a/contrib/snowball/algorithms/norwegian.sbl b/contrib/snowball/algorithms/norwegian.sbl new file mode 100644 index 0000000..39f4aff --- /dev/null +++ b/contrib/snowball/algorithms/norwegian.sbl @@ -0,0 +1,80 @@ +routines ( + mark_regions + main_suffix + consonant_pair + other_suffix +) + +externals ( stem ) + +integers ( p1 x ) + +groupings ( v s_ending ) + +stringescapes {} + +/* special characters */ + +stringdef ae '{U+00E6}' +stringdef ao '{U+00E5}' +stringdef o/ '{U+00F8}' + +define v 'aeiouy{ae}{ao}{o/}' + +define s_ending 'bcdfghjlmnoprtvyz' + +define mark_regions as ( + + $p1 = limit + + test ( hop 3 setmark x ) + goto v gopast non-v setmark p1 + try ( $p1 < x $p1 = x ) +) + +backwardmode ( + + define main_suffix as ( + setlimit tomark p1 for ([substring]) + among( + + 'a' 'e' 'ede' 'ande' 'ende' 'ane' 'ene' 'hetene' 'en' 'heten' 'ar' + 'er' 'heter' 'as' 'es' 'edes' 'endes' 'enes' 'hetenes' 'ens' + 'hetens' 'ers' 'ets' 'et' 'het' 'ast' + (delete) + 's' + (s_ending or ('k' non-v) delete) + 'erte' 'ert' + (<-'er') + ) + ) + + define consonant_pair as ( + test ( + setlimit tomark p1 for ([substring]) + among( + 'dt' 'vt' + ) + ) + next] delete + ) + + define other_suffix as ( + setlimit tomark p1 for ([substring]) + among( + 'leg' 'eleg' 'ig' 'eig' 'lig' 'elig' 'els' 'lov' 'elov' 'slov' + 'hetslov' + (delete) + ) + ) +) + +define stem as ( + + do mark_regions + backwards ( + do main_suffix + do consonant_pair + do other_suffix + ) +) diff --git a/contrib/snowball/algorithms/porter.sbl b/contrib/snowball/algorithms/porter.sbl new file mode 100644 index 0000000..9533b79 --- /dev/null +++ b/contrib/snowball/algorithms/porter.sbl @@ -0,0 +1,139 @@ +integers ( p1 p2 ) +booleans ( Y_found ) + +routines ( + shortv + R1 R2 + Step_1a Step_1b Step_1c Step_2 Step_3 Step_4 Step_5a Step_5b +) + +externals ( stem ) + +groupings ( v v_WXY ) + +define v 'aeiouy' +define v_WXY v + 'wxY' + +backwardmode ( + + define shortv as ( non-v_WXY v non-v ) + + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define Step_1a as ( + [substring] among ( + 'sses' (<-'ss') + 'ies' (<-'i') + 'ss' () + 's' (delete) + ) + ) + + define Step_1b as ( + [substring] among ( + 'eed' (R1 <-'ee') + 'ed' + 'ing' ( + test gopast v delete + test substring among( + 'at' 'bl' 'iz' + (<+ 'e') + 'bb' 'dd' 'ff' 'gg' 'mm' 'nn' 'pp' 'rr' 'tt' + // ignoring double c, h, j, k, q, v, w, and x + ([next] delete) + '' (atmark p1 test shortv <+ 'e') + ) + ) + ) + ) + + define Step_1c as ( + ['y' or 'Y'] + gopast v + <-'i' + ) + + define Step_2 as ( + [substring] R1 among ( + 'tional' (<-'tion') + 'enci' (<-'ence') + 'anci' (<-'ance') + 'abli' (<-'able') + 'entli' (<-'ent') + 'eli' (<-'e') + 'izer' 'ization' + (<-'ize') + 'ational' 'ation' 'ator' + (<-'ate') + 'alli' (<-'al') + 'alism' 'aliti' + (<-'al') + 'fulness' (<-'ful') + 'ousli' 'ousness' + (<-'ous') + 'iveness' 'iviti' + (<-'ive') + 'biliti' (<-'ble') + ) + ) + + define Step_3 as ( + [substring] R1 among ( + 'alize' (<-'al') + 'icate' 'iciti' 'ical' + (<-'ic') + 'ative' 'ful' 'ness' + (delete) + ) + ) + + define Step_4 as ( + [substring] R2 among ( + 'al' 'ance' 'ence' 'er' 'ic' 'able' 'ible' 'ant' 'ement' + 'ment' 'ent' 'ou' 'ism' 'ate' 'iti' 'ous' 'ive' 'ize' + (delete) + 'ion' ('s' or 't' delete) + ) + ) + + define Step_5a as ( + ['e'] + R2 or (R1 not shortv) + delete + ) + + define Step_5b as ( + ['l'] + R2 'l' + delete + ) +) + +define stem as ( + + unset Y_found + do ( ['y'] <-'Y' set Y_found) + do repeat(goto (v ['y']) <-'Y' set Y_found) + + $p1 = limit + $p2 = limit + do( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) + + backwards ( + do Step_1a + do Step_1b + do Step_1c + do Step_2 + do Step_3 + do Step_4 + do Step_5a + do Step_5b + ) + + do(Y_found repeat(goto (['Y']) <-'y')) + +) diff --git a/contrib/snowball/algorithms/portuguese.sbl b/contrib/snowball/algorithms/portuguese.sbl new file mode 100644 index 0000000..3fb14f1 --- /dev/null +++ b/contrib/snowball/algorithms/portuguese.sbl @@ -0,0 +1,218 @@ +routines ( + prelude postlude mark_regions + RV R1 R2 + standard_suffix + verb_suffix + residual_suffix + residual_form +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v ) + +stringescapes {} + +/* special characters */ + +stringdef a' '{U+00E1}' // a-acute +stringdef a^ '{U+00E2}' // a-circumflex e.g. 'bota^nico +stringdef e' '{U+00E9}' // e-acute +stringdef e^ '{U+00EA}' // e-circumflex +stringdef i' '{U+00ED}' // i-acute +stringdef o^ '{U+00F4}' // o-circumflex +stringdef o' '{U+00F3}' // o-acute +stringdef u' '{U+00FA}' // u-acute +stringdef c, '{U+00E7}' // c-cedilla + +stringdef a~ '{U+00E3}' // a-tilde +stringdef o~ '{U+00F5}' // o-tilde + + +define v 'aeiou{a'}{e'}{i'}{o'}{u'}{a^}{e^}{o^}' + +define prelude as repeat ( + [substring] among( + '{a~}' (<- 'a~') + '{o~}' (<- 'o~') + '' (next) + ) //or next +) + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v (non-v gopast v) or (v gopast non-v) ) + or + ( non-v (non-v gopast v) or (v next) ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define postlude as repeat ( + [substring] among( + 'a~' (<- '{a~}') + 'o~' (<- '{o~}') + '' (next) + ) //or next +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define standard_suffix as ( + [substring] among( + + 'eza' 'ezas' + 'ico' 'ica' 'icos' 'icas' + 'ismo' 'ismos' + '{a'}vel' + '{i'}vel' + 'ista' 'istas' + 'oso' 'osa' 'osos' 'osas' + 'amento' 'amentos' + 'imento' 'imentos' + + 'adora' 'ador' 'a{c,}a~o' + 'adoras' 'adores' 'a{c,}o~es' // no -ic test + 'ante' 'antes' '{a^}ncia' // Note 1 + ( + R2 delete + ) + 'logia' + 'logias' + ( + R2 <- 'log' + ) + 'u{c,}a~o' 'u{c,}o~es' + ( + R2 <- 'u' + ) + '{e^}ncia' '{e^}ncias' + ( + R2 <- 'ente' + ) + 'amente' + ( + R1 delete + try ( + [substring] R2 delete among( + 'iv' (['at'] R2 delete) + 'os' + 'ic' + 'ad' + ) + ) + ) + 'mente' + ( + R2 delete + try ( + [substring] among( + 'ante' // Note 1 + 'avel' + '{i'}vel' (R2 delete) + ) + ) + ) + 'idade' + 'idades' + ( + R2 delete + try ( + [substring] among( + 'abil' + 'ic' + 'iv' (R2 delete) + ) + ) + ) + 'iva' 'ivo' + 'ivas' 'ivos' + ( + R2 delete + try ( + ['at'] R2 delete // but not a further ['ic'] R2 delete + ) + ) + 'ira' 'iras' + ( + RV 'e' // -eira -eiras usually non-verbal + <- 'ir' + ) + ) + ) + + define verb_suffix as setlimit tomark pV for ( + [substring] among( + 'ada' 'ida' 'ia' 'aria' 'eria' 'iria' 'ar{a'}' 'ara' 'er{a'}' + 'era' 'ir{a'}' 'ava' 'asse' 'esse' 'isse' 'aste' 'este' 'iste' + 'ei' 'arei' 'erei' 'irei' 'am' 'iam' 'ariam' 'eriam' 'iriam' + 'aram' 'eram' 'iram' 'avam' 'em' 'arem' 'erem' 'irem' 'assem' + 'essem' 'issem' 'ado' 'ido' 'ando' 'endo' 'indo' 'ara~o' + 'era~o' 'ira~o' 'ar' 'er' 'ir' 'as' 'adas' 'idas' 'ias' + 'arias' 'erias' 'irias' 'ar{a'}s' 'aras' 'er{a'}s' 'eras' + 'ir{a'}s' 'avas' 'es' 'ardes' 'erdes' 'irdes' 'ares' 'eres' + 'ires' 'asses' 'esses' 'isses' 'astes' 'estes' 'istes' 'is' + 'ais' 'eis' '{i'}eis' 'ar{i'}eis' 'er{i'}eis' 'ir{i'}eis' + '{a'}reis' 'areis' '{e'}reis' 'ereis' '{i'}reis' 'ireis' + '{a'}sseis' '{e'}sseis' '{i'}sseis' '{a'}veis' 'ados' 'idos' + '{a'}mos' 'amos' '{i'}amos' 'ar{i'}amos' 'er{i'}amos' + 'ir{i'}amos' '{a'}ramos' '{e'}ramos' '{i'}ramos' '{a'}vamos' + 'emos' 'aremos' 'eremos' 'iremos' '{a'}ssemos' '{e^}ssemos' + '{i'}ssemos' 'imos' 'armos' 'ermos' 'irmos' 'eu' 'iu' 'ou' + + 'ira' 'iras' + (delete) + ) + ) + + define residual_suffix as ( + [substring] among( + 'os' + 'a' 'i' 'o' '{a'}' '{i'}' '{o'}' + ( RV delete ) + ) + ) + + define residual_form as ( + [substring] among( + 'e' '{e'}' '{e^}' + ( RV delete [('u'] test 'g') or + ('i'] test 'c') RV delete ) + '{c,}' (<-'c') + ) + ) +) + +define stem as ( + do prelude + do mark_regions + backwards ( + do ( + ( ( standard_suffix or verb_suffix ) + and do ( ['i'] test 'c' RV delete ) + ) + or residual_suffix + ) + do residual_form + ) + do postlude +) + +/* + Note 1: additions of 15 Jun 2005 +*/ diff --git a/contrib/snowball/algorithms/romanian.sbl b/contrib/snowball/algorithms/romanian.sbl new file mode 100644 index 0000000..7db9e0a --- /dev/null +++ b/contrib/snowball/algorithms/romanian.sbl @@ -0,0 +1,236 @@ + +routines ( + prelude postlude mark_regions + RV R1 R2 + step_0 + standard_suffix combo_suffix + verb_suffix + vowel_suffix +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v ) + +booleans ( standard_suffix_removed ) + +stringescapes {} + +/* special characters */ + +stringdef a^ '{U+00E2}' // a circumflex +stringdef i^ '{U+00EE}' // i circumflex +stringdef a+ '{U+0103}' // a breve +stringdef s, '{U+015F}' // s cedilla +stringdef t, '{U+0163}' // t cedilla + +define v 'aeiou{a^}{i^}{a+}' + +define prelude as ( + repeat goto ( + v [ ('u' ] v <- 'U') or + ('i' ] v <- 'I') + ) +) + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v (non-v gopast v) or (v gopast non-v) ) + or + ( non-v (non-v gopast v) or (v next) ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define postlude as repeat ( + + [substring] among( + 'I' (<- 'i') + 'U' (<- 'u') + '' (next) + ) + +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define step_0 as ( + [substring] R1 among( + 'ul' 'ului' + ( delete ) + 'aua' + ( <-'a' ) + 'ea' 'ele' 'elor' + ( <-'e' ) + 'ii' 'iua' 'iei' 'iile' 'iilor' 'ilor' + ( <-'i') + 'ile' + ( not 'ab' <- 'i' ) + 'atei' + ( <- 'at' ) + 'a{t,}ie' 'a{t,}ia' + ( <- 'a{t,}i' ) + ) + ) + + define combo_suffix as test ( + [substring] R1 ( + among( + /* 'IST'. alternative: include the following + 'alism' 'alisme' + 'alist' 'alista' 'aliste' 'alisti' 'alist{a+}' 'ali{s,}ti' ( + <- 'al' + ) + */ + 'abilitate' 'abilitati' 'abilit{a+}i' 'abilit{a+}{t,}i' ( + <- 'abil' + ) + 'ibilitate' ( + <- 'ibil' + ) + 'ivitate' 'ivitati' 'ivit{a+}i' 'ivit{a+}{t,}i' ( + <- 'iv' + ) + 'icitate' 'icitati' 'icit{a+}i' 'icit{a+}{t,}i' + 'icator' 'icatori' + 'iciv' 'iciva' 'icive' 'icivi' 'iciv{a+}' + 'ical' 'icala' 'icale' 'icali' 'ical{a+}' ( + <- 'ic' + ) + 'ativ' 'ativa' 'ative' 'ativi' 'ativ{a+}' 'a{t,}iune' + 'atoare' 'ator' 'atori' + '{a+}toare' '{a+}tor' '{a+}tori' ( + <- 'at' + ) + 'itiv' 'itiva' 'itive' 'itivi' 'itiv{a+}' 'i{t,}iune' + 'itoare' 'itor' 'itori' ( + <- 'it' + ) + ) + set standard_suffix_removed + ) + ) + + define standard_suffix as ( + unset standard_suffix_removed + repeat combo_suffix + [substring] R2 ( + among( + + // past participle is treated here, rather than + // as a verb ending: + 'at' 'ata' 'at{a+}' 'ati' 'ate' + 'ut' 'uta' 'ut{a+}' 'uti' 'ute' + 'it' 'ita' 'it{a+}' 'iti' 'ite' + + 'ic' 'ica' 'ice' 'ici' 'ic{a+}' + 'abil' 'abila' 'abile' 'abili' 'abil{a+}' + 'ibil' 'ibila' 'ibile' 'ibili' 'ibil{a+}' + 'oasa' 'oas{a+}' 'oase' 'os' 'osi' 'o{s,}i' + 'ant' 'anta' 'ante' 'anti' 'ant{a+}' + 'ator' 'atori' + 'itate' 'itati' 'it{a+}i' 'it{a+}{t,}i' + 'iv' 'iva' 'ive' 'ivi' 'iv{a+}' ( + delete + ) + 'iune' 'iuni' ( + '{t,}'] <- 't' + ) + 'ism' 'isme' + 'ist' 'ista' 'iste' 'isti' 'ist{a+}' 'i{s,}ti' ( + <- 'ist' + /* 'IST'. alternative: remove with <- '' */ + ) + ) + set standard_suffix_removed + ) + ) + + define verb_suffix as setlimit tomark pV for ( + [substring] among( + // 'long' infinitive: + 'are' 'ere' 'ire' '{a^}re' + + // gerund: + 'ind' '{a^}nd' + 'indu' '{a^}ndu' + + 'eze' + 'easc{a+}' + // present: + 'ez' 'ezi' 'eaz{a+}' 'esc' 'e{s,}ti' + 'e{s,}te' + '{a+}sc' '{a+}{s,}ti' + '{a+}{s,}te' + + // imperfect: + 'am' 'ai' 'au' + 'eam' 'eai' 'ea' 'ea{t,}i' 'eau' + 'iam' 'iai' 'ia' 'ia{t,}i' 'iau' + + // past: // (not 'ii') + 'ui' + 'a{s,}i' 'ar{a+}m' 'ar{a+}{t,}i' 'ar{a+}' + 'u{s,}i' 'ur{a+}m' 'ur{a+}{t,}i' 'ur{a+}' + 'i{s,}i' 'ir{a+}m' 'ir{a+}{t,}i' 'ir{a+}' + '{a^}i' '{a^}{s,}i' '{a^}r{a+}m' '{a^}r{a+}{t,}i' '{a^}r{a+}' + + // pluferfect: + 'asem' 'ase{s,}i' 'ase' 'aser{a+}m' 'aser{a+}{t,}i' 'aser{a+}' + 'isem' 'ise{s,}i' 'ise' 'iser{a+}m' 'iser{a+}{t,}i' 'iser{a+}' + '{a^}sem' '{a^}se{s,}i' '{a^}se' '{a^}ser{a+}m' '{a^}ser{a+}{t,}i' + '{a^}ser{a+}' + 'usem' 'use{s,}i' 'use' 'user{a+}m' 'user{a+}{t,}i' 'user{a+}' + + ( non-v or 'u' delete ) + + // present: + '{a+}m' 'a{t,}i' + 'em' 'e{t,}i' + 'im' 'i{t,}i' + '{a^}m' '{a^}{t,}i' + + // past: + 'se{s,}i' 'ser{a+}m' 'ser{a+}{t,}i' 'ser{a+}' + 'sei' 'se' + + // pluperfect: + 'sesem' 'sese{s,}i' 'sese' 'seser{a+}m' 'seser{a+}{t,}i' 'seser{a+}' + (delete) + ) + ) + + define vowel_suffix as ( + [substring] RV among ( + 'a' 'e' 'i' 'ie' '{a+}' ( delete ) + ) + ) +) + +define stem as ( + do prelude + do mark_regions + backwards ( + do step_0 + do standard_suffix + do ( standard_suffix_removed or verb_suffix ) + do vowel_suffix + ) + do postlude +) + diff --git a/contrib/snowball/algorithms/russian.sbl b/contrib/snowball/algorithms/russian.sbl new file mode 100644 index 0000000..20de639 --- /dev/null +++ b/contrib/snowball/algorithms/russian.sbl @@ -0,0 +1,221 @@ +stringescapes {} + +/* the 33 Cyrillic letters represented in ASCII characters following the + * conventions of the standard Library of Congress transliteration: */ + +stringdef a '{U+0430}' +stringdef b '{U+0431}' +stringdef v '{U+0432}' +stringdef g '{U+0433}' +stringdef d '{U+0434}' +stringdef e '{U+0435}' +stringdef e" '{U+0451}' +stringdef zh '{U+0436}' +stringdef z '{U+0437}' +stringdef i '{U+0438}' +stringdef i` '{U+0439}' +stringdef k '{U+043A}' +stringdef l '{U+043B}' +stringdef m '{U+043C}' +stringdef n '{U+043D}' +stringdef o '{U+043E}' +stringdef p '{U+043F}' +stringdef r '{U+0440}' +stringdef s '{U+0441}' +stringdef t '{U+0442}' +stringdef u '{U+0443}' +stringdef f '{U+0444}' +stringdef kh '{U+0445}' +stringdef ts '{U+0446}' +stringdef ch '{U+0447}' +stringdef sh '{U+0448}' +stringdef shch '{U+0449}' +stringdef " '{U+044A}' +stringdef y '{U+044B}' +stringdef ' '{U+044C}' +stringdef e` '{U+044D}' +stringdef iu '{U+044E}' +stringdef ia '{U+044F}' + +routines ( mark_regions R2 + perfective_gerund + adjective + adjectival + reflexive + verb + noun + derivational + tidy_up +) + +externals ( stem ) + +integers ( pV p2 ) + +groupings ( v ) + +define v '{a}{e}{i}{o}{u}{y}{e`}{iu}{ia}' + +define mark_regions as ( + + $pV = limit + $p2 = limit + do ( + gopast v setmark pV gopast non-v + gopast v gopast non-v setmark p2 + ) +) + +backwardmode ( + + define R2 as $p2 <= cursor + + define perfective_gerund as ( + [substring] among ( + '{v}' + '{v}{sh}{i}' + '{v}{sh}{i}{s}{'}' + ('{a}' or '{ia}' delete) + '{i}{v}' + '{i}{v}{sh}{i}' + '{i}{v}{sh}{i}{s}{'}' + '{y}{v}' + '{y}{v}{sh}{i}' + '{y}{v}{sh}{i}{s}{'}' + (delete) + ) + ) + + define adjective as ( + [substring] among ( + '{e}{e}' '{i}{e}' '{y}{e}' '{o}{e}' '{i}{m}{i}' '{y}{m}{i}' + '{e}{i`}' '{i}{i`}' '{y}{i`}' '{o}{i`}' '{e}{m}' '{i}{m}' + '{y}{m}' '{o}{m}' '{e}{g}{o}' '{o}{g}{o}' '{e}{m}{u}' + '{o}{m}{u}' '{i}{kh}' '{y}{kh}' '{u}{iu}' '{iu}{iu}' '{a}{ia}' + '{ia}{ia}' + // and - + '{o}{iu}' // - which is somewhat archaic + '{e}{iu}' // - soft form of {o}{iu} + (delete) + ) + ) + + define adjectival as ( + adjective + + /* of the participle forms, em, vsh, ivsh, yvsh are readily removable. + nn, {iu}shch, shch, u{iu}shch can be removed, with a small proportion of + errors. Removing im, uem, enn creates too many errors. + */ + + try ( + [substring] among ( + '{e}{m}' // present passive participle + '{n}{n}' // adjective from past passive participle + '{v}{sh}' // past active participle + '{iu}{shch}' '{shch}' // present active participle + ('{a}' or '{ia}' delete) + + //but not '{i}{m}' '{u}{e}{m}' // present passive participle + //or '{e}{n}{n}' // adjective from past passive participle + + '{i}{v}{sh}' '{y}{v}{sh}'// past active participle + '{u}{iu}{shch}' // present active participle + (delete) + ) + ) + + ) + + define reflexive as ( + [substring] among ( + '{s}{ia}' + '{s}{'}' + (delete) + ) + ) + + define verb as ( + [substring] among ( + '{l}{a}' '{n}{a}' '{e}{t}{e}' '{i`}{t}{e}' '{l}{i}' '{i`}' + '{l}' '{e}{m}' '{n}' '{l}{o}' '{n}{o}' '{e}{t}' '{iu}{t}' + '{n}{y}' '{t}{'}' '{e}{sh}{'}' + + '{n}{n}{o}' + ('{a}' or '{ia}' delete) + + '{i}{l}{a}' '{y}{l}{a}' '{e}{n}{a}' '{e}{i`}{t}{e}' + '{u}{i`}{t}{e}' '{i}{t}{e}' '{i}{l}{i}' '{y}{l}{i}' '{e}{i`}' + '{u}{i`}' '{i}{l}' '{y}{l}' '{i}{m}' '{y}{m}' '{e}{n}' + '{i}{l}{o}' '{y}{l}{o}' '{e}{n}{o}' '{ia}{t}' '{u}{e}{t}' + '{u}{iu}{t}' '{i}{t}' '{y}{t}' '{e}{n}{y}' '{i}{t}{'}' + '{y}{t}{'}' '{i}{sh}{'}' '{u}{iu}' '{iu}' + (delete) + /* note the short passive participle tests: + '{n}{a}' '{n}' '{n}{o}' '{n}{y}' + '{e}{n}{a}' '{e}{n}' '{e}{n}{o}' '{e}{n}{y}' + */ + ) + ) + + define noun as ( + [substring] among ( + '{a}' '{e}{v}' '{o}{v}' '{i}{e}' '{'}{e}' '{e}' + '{i}{ia}{m}{i}' '{ia}{m}{i}' '{a}{m}{i}' '{e}{i}' '{i}{i}' + '{i}' '{i}{e}{i`}' '{e}{i`}' '{o}{i`}' '{i}{i`}' '{i`}' + '{i}{ia}{m}' '{ia}{m}' '{i}{e}{m}' '{e}{m}' '{a}{m}' '{o}{m}' + '{o}' '{u}' '{a}{kh}' '{i}{ia}{kh}' '{ia}{kh}' '{y}' '{'}' + '{i}{iu}' '{'}{iu}' '{iu}' '{i}{ia}' '{'}{ia}' '{ia}' + (delete) + /* the small class of neuter forms '{e}{n}{i}' '{e}{n}{e}{m}' + '{e}{n}{a}' '{e}{n}' '{e}{n}{a}{m}' '{e}{n}{a}{m}{i}' '{e}{n}{a}{x}' + omitted - they only occur on 12 words. + */ + ) + ) + + define derivational as ( + [substring] R2 among ( + '{o}{s}{t}' + '{o}{s}{t}{'}' + (delete) + ) + ) + + define tidy_up as ( + [substring] among ( + + '{e}{i`}{sh}' + '{e}{i`}{sh}{e}' // superlative forms + (delete + ['{n}'] '{n}' delete + ) + '{n}' + ('{n}' delete) // e.g. -nno endings + '{'}' + (delete) // with some slight false conflations + ) + ) +) + +define stem as ( + + // Normalise {e"} to {e}. The documentation has long suggested the user + // should do this before calling the stemmer - we now do it for them. + do repeat ( goto (['{e"}']) <- '{e}' ) + + do mark_regions + backwards setlimit tomark pV for ( + do ( + perfective_gerund or + ( try reflexive + adjectival or verb or noun + ) + ) + try([ '{i}' ] delete) + // because noun ending -i{iu} is being treated as verb ending -{iu} + + do derivational + do tidy_up + ) +) diff --git a/contrib/snowball/algorithms/serbian.sbl b/contrib/snowball/algorithms/serbian.sbl new file mode 100644 index 0000000..bddf76b --- /dev/null +++ b/contrib/snowball/algorithms/serbian.sbl @@ -0,0 +1,2378 @@ +/* Stemmer for Serbian language, based on: + * + * Ljubesic, Nikola. Pandzic, Ivan. Stemmer for Croatian + * http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/ + * + * authors: Stefan Petkovic and Dragan Ivanovic + * emails: petkovic8 at gmail.com and dragan.ivanovic at uns.ac.rs + * version: 1.0 (20.04.2019) +*/ + +routines ( + cyr_to_lat + prelude + mark_regions + R1 R2 + Step_1 + Step_2 + Step_3 +) + +externals ( stem ) + +integers ( p1 p2 p3 ) + +groupings ( v ca sa rg ) + +stringescapes {} + +/* special characters - Unicode codepoints */ + +/* serbian cyrillic */ + +stringdef cyrA '{U+0430}' +stringdef cyrB '{U+0431}' +stringdef cyrV '{U+0432}' +stringdef cyrG '{U+0433}' +stringdef cyrD '{U+0434}' +stringdef cyrDx '{U+0452}' +stringdef cyrE '{U+0435}' +stringdef cyrZh '{U+0436}' +stringdef cyrZ '{U+0437}' +stringdef cyrI '{U+0438}' +stringdef cyrJ '{U+0458}' +stringdef cyrK '{U+043A}' +stringdef cyrL '{U+043B}' +stringdef cyrLJ '{U+0459}' +stringdef cyrM '{U+043C}' +stringdef cyrN '{U+043D}' +stringdef cyrNJ '{U+045A}' +stringdef cyrO '{U+043E}' +stringdef cyrP '{U+043F}' +stringdef cyrR '{U+0440}' +stringdef cyrS '{U+0441}' +stringdef cyrT '{U+0442}' +stringdef cyrCy '{U+045B}' +stringdef cyrU '{U+0443}' +stringdef cyrF '{U+0444}' +stringdef cyrH '{U+0445}' +stringdef cyrC '{U+0446}' +stringdef cyrCx '{U+0447}' +stringdef cyrDzx '{U+045F}' +stringdef cyrSx '{U+0448}' + +/* serbian latin with diacritics */ + +stringdef cx '{U+010D}' // small c with caron +stringdef cy '{U+0107}' // small c with acute +stringdef zx '{U+017E}' // small z with caron +stringdef sx '{U+0161}' // small s with caron +stringdef dx '{U+0111}' // small d with stroke + +define v 'aeiou' +define sa '{cx}{cy}{zx}{sx}{dx}' +define ca 'bvgdzjklmnprstfhc' + sa +define rg 'r' + + +define cyr_to_lat as ( + + do repeat goto ( + [substring] among ( + '{cyrA}' (<- 'a') + '{cyrB}' (<- 'b') + '{cyrV}' (<- 'v') + '{cyrG}' (<- 'g') + '{cyrD}' (<- 'd') + '{cyrDx}' (<- '{dx}') + '{cyrE}' (<- 'e') + '{cyrZh}' (<- '{zx}') + '{cyrZ}' (<- 'z') + '{cyrI}' (<- 'i') + '{cyrJ}' (<- 'j') + '{cyrK}' (<- 'k') + '{cyrL}' (<- 'l') + '{cyrLJ}' (<- 'lj') + '{cyrM}' (<- 'm') + '{cyrN}' (<- 'n') + '{cyrNJ}' (<- 'nj') + '{cyrO}' (<- 'o') + '{cyrP}' (<- 'p') + '{cyrR}' (<- 'r') + '{cyrS}' (<- 's') + '{cyrT}' (<- 't') + '{cyrCy}' (<- '{cy}') + '{cyrU}' (<- 'u') + '{cyrF}' (<- 'f') + '{cyrH}' (<- 'h') + '{cyrC}' (<- 'c') + '{cyrCx}' (<- '{cx}') + '{cyrDzx}' (<- 'd{zx}') + '{cyrSx}' (<- '{sx}') + ) + ) + +) + +define prelude as ( + + do repeat goto ( + ca ['ije'] ca <- 'e' + ) + + do repeat goto ( + ca ['je'] ca <- 'e' + ) + + do repeat goto ( + ['dj'] <- '{dx}' + ) + +) + +define mark_regions as ( + + $p3 = 0 + + do ( + gopast sa setmark p3 + ) + + $p1 = limit + $p2 = 0 + + do ( + gopast v setmark p1 + ) + do ( + gopast 'r' setmark p2 + $(p1 - p2 > 1) ($p1 = p2) + ) + ($p1 < 2) ( + ($p1 == p2 gopast 'r' gopast non-rg) or ($p1 != p2 gopast v gopast non-v) + setmark p1 + ) + +) + +backwardmode ( + + define R1 as $p1 <= cursor + define R2 as $p3 == 0 + + define Step_1 as ( + [substring] among ( + 'lozi' + 'lozima' (<-'loga') + 'pesi' + 'pesima' (<-'peh') + 'vojci' (<-'vojka') + 'bojci' (<-'bojka') + 'jaci' + 'jacima' (<-'jak') + '{cx}ajan' (<-'{cx}ajni') + 'cajan' (R2 <-'cajni') + 'eran' (<-'erni') + 'laran' (<-'larni') + 'esan' (<-'esni') + 'anjac' (<-'anjca') + 'ajac' + 'ajaca' (<-'ajca') + 'ljaca' + 'ljac' (<-'ljca') + 'ejac' + 'ejaca' (<-'ejca') + 'ojac' + 'ojaca' (<-'ojca') + 'ajaka' (<-'ajka') + 'ojaka' (<-'ojka') + '{sx}aca' + '{sx}ac' (<-'{sx}ca') + 'inzima' + 'inzi' (<-'ing') + 'tvenici' (<-'tvenik') + 'tetici' + 'teticima' (<-'tetika') + 'nstava' (<-'nstva') + 'nicima' (<-'nik') + 'ticima' (<-'tik') + 'zicima' (<-'zik') + 'snici' (<-'snik') + 'kuse' (<-'kusi') + 'kusan' (<-'kusni') + 'kustava' (<-'kustva') + 'du{sx}an' (<-'du{sx}ni') + 'dusan' (R2 <-'dusni') + 'antan' (<-'antni') + 'bilan' (<-'bilni') + 'tilan' (<-'tilni') + 'avilan' (<-'avilni') + 'silan' (<-'silni') + 'gilan' (<-'gilni') + 'rilan' (<-'rilni') + 'nilan' (<-'nilni') + 'alan' (<-'alni') + 'ozan' (<-'ozni') + 'rave' (<-'ravi') + 'stavan' (<-'stavni') + 'pravan' (<-'pravni') + 'tivan' (<-'tivni') + 'sivan' (<-'sivni') + 'atan' (<-'atni') + 'enat' (<-'enta') + 'tetan' (<-'tetni') + 'pletan' (<-'pletni') + '{sx}ave' (<-'{sx}avi') + 'save' (R2 <-'savi') + 'anata' (<-'anta') + 'a{cx}ak' + 'a{cx}aka' (<-'a{cx}ka') + 'acak' + 'acaka' (R2 <-'acka') + 'u{sx}ak' (<-'u{sx}ka') + 'usak' (R2 <-'uska') + 'atak' + 'ataka' + 'atci' + 'atcima' (<-'atka') + 'etak' + 'etaka' (<-'etka') + 'itak' + 'itaka' + 'itci' (<-'itka') + 'otak' + 'otaka' (<-'otka') + 'utak' + 'utaka' + 'utci' + 'utcima' (<-'utka') + 'eskan' (<-'eskna') + 'ti{cx}an' (<-'ti{cx}ni') + 'tican' (R2 <-'ticni') + 'ojsci' (<-'ojska') + 'esama' (<-'esma') + 'metar' + 'metara' (<-'metra') + 'centar' + 'centara' (<-'centra') + 'istar' + 'istara' (<-'istra') + 'o{sx}{cy}u' (<-'osti') + 'oscu' (R2 <-'osti') + 'daba' (<-'dba') + '{cx}cima' + '{cx}ci' (<-'{cx}ka') + 'mac' + 'maca' (<-'mca') + 'naca' + 'nac' (<-'nca') + 'voljan' (<-'voljni') + 'anaka' (<-'anki') + 'vac' + 'vaca' (<-'vca') + 'saca' + 'sac' (<-'sca') + 'raca' + 'rac' (<-'rca') + 'aoca' + 'alaca' + 'alac' (<-'alca') + 'elaca' + 'elac' (<-'elca') + 'olaca' + 'olac' + 'olce' (<-'olca') + 'njac' + 'njaca' (<-'njca') + 'ekata' + 'ekat' (<-'ekta') + 'izam' + 'izama' (<-'izma') + 'jebe' (<-'jebi') + 'baci' (<-'baci') + 'a{sx}an' (<-'a{sx}ni') + 'asan' (R2 <-'asni') + ) + ) + + define Step_2 as ( + [substring] R1 among ( + 'skijima' + 'skijega' + 'skijemu' + 'skijem' + 'skega' + 'skemu' + 'skem' + 'skijim' + 'skijih' + 'skijoj' + 'skijeg' + 'skiji' + 'skije' + 'skija' + 'skoga' + 'skome' + 'skomu' + 'skima' + 'skog' + 'skom' + 'skim' + 'skih' + 'skoj' + 'ski' + 'ske' + 'sko' + 'ska' + 'sku' (<-'sk') + '{sx}kijima' + '{sx}kijega' + '{sx}kijemu' + '{sx}kijem' + '{sx}kega' + '{sx}kemu' + '{sx}kem' + '{sx}kijim' + '{sx}kijih' + '{sx}kijoj' + '{sx}kijeg' + '{sx}kiji' + '{sx}kije' + '{sx}kija' + '{sx}koga' + '{sx}kome' + '{sx}komu' + '{sx}kima' + '{sx}kog' + '{sx}kom' + '{sx}kim' + '{sx}kih' + '{sx}koj' + '{sx}ki' + '{sx}ke' + '{sx}ko' + '{sx}ka' + '{sx}ku' (<-'{sx}k') + 'stvima' + 'stvom' + 'stvo' + 'stva' + 'stvu' (<-'stv') + '{sx}tvima' + '{sx}tvom' + '{sx}tvo' + '{sx}tva' + '{sx}tvu' (<-'{sx}tv') + 'tanijama' + 'tanijima' + 'tanijom' + 'tanija' + 'taniju' + 'tanije' + 'taniji' (<-'tanij') + 'manijama' + 'manijima' + 'manijom' + 'manija' + 'maniju' + 'manije' + 'maniji' (<-'manij') + 'panijama' + 'panijima' + 'panijom' + 'panija' + 'paniju' + 'panije' + 'paniji' (<-'panij') + 'ranijama' + 'ranijima' + 'ranijom' + 'ranija' + 'raniju' + 'ranije' + 'raniji' (<-'ranij') + 'ganijama' + 'ganijima' + 'ganijom' + 'ganija' + 'ganiju' + 'ganije' + 'ganiji' (<-'ganij') + 'aninom' + 'anina' + 'aninu' + 'anine' + 'anima' + 'anin' + 'anom' + 'anu' + 'ani' + 'ana' + 'ane' (<-'an') + 'inima' + 'inama' + 'inom' + 'ina' + 'ine' + 'ini' + 'inu' + 'ino' (<-'in') + 'onovima' + 'onova' + 'onove' + 'onovi' + 'onima' + 'onom' + 'ona' + 'one' + 'oni' + 'onu' (<-'on') + 'nijima' + 'nijega' + 'nijemu' + 'nijeg' + 'nijem' + 'nega' + 'nemu' + 'neg' + 'nem' + 'nijim' + 'nijih' + 'nijoj' + 'niji' + 'nije' + 'nija' + 'niju' + 'nima' + 'nome' + 'nomu' + 'noga' + 'noj' + 'nom' + 'nih' + 'nim' + 'nog' + 'no' + 'ne' + 'na' + 'nu' + 'ni' (<-'n') + 'a{cy}oga' + 'a{cy}ome' + 'a{cy}omu' + 'a{cy}ega' + 'a{cy}emu' + 'a{cy}ima' + 'a{cy}oj' + 'a{cy}ih' + 'a{cy}om' + 'a{cy}eg' + 'a{cy}em' + 'a{cy}og' + 'a{cy}uh' + 'a{cy}im' + 'a{cy}e' + 'a{cy}a' (<-'a{cy}') + 'e{cy}oga' + 'e{cy}ome' + 'e{cy}omu' + 'e{cy}ega' + 'e{cy}emu' + 'e{cy}ima' + 'e{cy}oj' + 'e{cy}ih' + 'e{cy}om' + 'e{cy}eg' + 'e{cy}em' + 'e{cy}og' + 'e{cy}uh' + 'e{cy}im' + 'e{cy}e' + 'e{cy}a' (<-'e{cy}') + 'u{cy}oga' + 'u{cy}ome' + 'u{cy}omu' + 'u{cy}ega' + 'u{cy}emu' + 'u{cy}ima' + 'u{cy}oj' + 'u{cy}ih' + 'u{cy}om' + 'u{cy}eg' + 'u{cy}em' + 'u{cy}og' + 'u{cy}uh' + 'u{cy}im' + 'u{cy}e' + 'u{cy}a' (<-'u{cy}') + 'ugovima' + 'ugovi' + 'ugove' + 'ugova' (<-'ugov') + 'ugama' + 'ugom' + 'uga' + 'uge' + 'ugi' + 'ugu' + 'ugo' (<-'ug') + 'logama' + 'logom' + 'loga' + 'logu' + 'loge' (<-'log') + 'govima' + 'gama' + 'govi' + 'gove' + 'gova' + 'gom' + 'ga' + 'ge' + 'gi' + 'gu' + 'go' (<-'g') + 'rarijem' + 'rarija' + 'rariju' + 'rario' (<-'rari') + 'otijem' + 'otija' + 'otiju' + 'otio' (<-'oti') + 'sijem' + 'sija' + 'siju' + 'sio' (<-'si') + 'lijem' + 'lija' + 'liju' + 'lio' (<-'li') + 'uju{cy}i' + 'ujemo' + 'ujete' + 'ujmo' + 'ujem' + 'uje{sx}' + 'uje' + 'uju' (<-'uj') + 'cajevima' + 'cajevi' + 'cajeva' + 'cajeve' + 'cajama' + 'cajima' + 'cajem' + 'caja' + 'caje' + 'caji' + 'caju' (<-'caj') + '{cx}ajevima' + '{cx}ajevi' + '{cx}ajeva' + '{cx}ajeve' + '{cx}ajama' + '{cx}ajima' + '{cx}ajem' + '{cx}aja' + '{cx}aje' + '{cx}aji' + '{cx}aju' (<-'{cx}aj') + '{cy}ajevima' + '{cy}ajevi' + '{cy}ajeva' + '{cy}ajeve' + '{cy}ajama' + '{cy}ajima' + '{cy}ajem' + '{cy}aja' + '{cy}aje' + '{cy}aji' + '{cy}aju' (<-'{cy}aj') + '{dx}ajevima' + '{dx}ajevi' + '{dx}ajeva' + '{dx}ajeve' + '{dx}ajama' + '{dx}ajima' + '{dx}ajem' + '{dx}aja' + '{dx}aje' + '{dx}aji' + '{dx}aju' (<-'{dx}aj') + 'lajevima' + 'lajevi' + 'lajeva' + 'lajeve' + 'lajama' + 'lajima' + 'lajem' + 'laja' + 'laje' + 'laji' + 'laju' (<-'laj') + 'rajevima' + 'rajevi' + 'rajeva' + 'rajeve' + 'rajama' + 'rajima' + 'rajem' + 'raja' + 'raje' + 'raji' + 'raju' (<-'raj') + 'bijima' + 'bijama' + 'bijom' + 'bija' + 'bije' + 'biji' + 'biju' + 'bijo' (<-'bij') + 'cijima' + 'cijama' + 'cijom' + 'cija' + 'cije' + 'ciji' + 'ciju' + 'cijo' (<-'cij') + 'dijima' + 'dijama' + 'dijom' + 'dija' + 'dije' + 'diji' + 'diju' + 'dijo' (<-'dij') + 'lijima' + 'lijama' + 'lijom' + 'lije' + 'liji' + 'lijo' (<-'lij') + 'nijama' + 'nijom' + 'nijo' (<-'nij') + 'mijima' + 'mijama' + 'mijom' + 'mija' + 'mije' + 'miji' + 'miju' + 'mijo' (<-'mij') + '{zx}ijima' + '{zx}ijama' + '{zx}ijom' + '{zx}ija' + '{zx}ije' + '{zx}iji' + '{zx}iju' + '{zx}ijo' (<-'{zx}ij') + 'gijima' + 'gijama' + 'gijom' + 'gija' + 'gije' + 'giji' + 'giju' + 'gijo' (<-'gij') + 'fijima' + 'fijama' + 'fijom' + 'fija' + 'fije' + 'fiji' + 'fiju' + 'fijo' (<-'fij') + 'pijima' + 'pijama' + 'pijom' + 'pija' + 'pije' + 'piji' + 'piju' + 'pijo' (<-'pij') + 'rijima' + 'rijama' + 'rijom' + 'rija' + 'rije' + 'riji' + 'riju' + 'rijo' (<-'rij') + 'sijima' + 'sijama' + 'sijom' + 'sije' + 'siji' + 'sijo' (<-'sij') + 'tijima' + 'tijama' + 'tijom' + 'tija' + 'tije' + 'tiji' + 'tiju' + 'tijo' (<-'tij') + 'zijima' + 'zijama' + 'zijom' + 'zija' + 'zije' + 'ziji' + 'ziju' + 'zijo' (<-'zij') + 'nalima' + 'nalama' + 'nalom' + 'nala' + 'nale' + 'nali' + 'nalu' + 'nalo' (<-'nal') + 'ijalima' + 'ijalama' + 'ijalom' + 'ijala' + 'ijale' + 'ijali' + 'ijalu' + 'ijalo' (<-'ijal') + 'ozilima' + 'ozilom' + 'ozila' + 'ozile' + 'ozilu' + 'ozili' (<-'ozil') + 'olovima' + 'olovi' + 'olova' + 'olove' (<-'olov') + 'olima' + 'olom' + 'ola' + 'olu' + 'ole' + 'oli' (<-'ol') + 'lemama' + 'lemima' + 'lemom' + 'lema' + 'leme' + 'lemi' + 'lemu' + 'lemo' (<-'lem') + 'ramama' + 'ramom' + 'rama' + 'rame' + 'rami' + 'ramu' + 'ramo' (<-'ram') + 'arama' + 'arima' + 'arom' + 'aru' + 'ara' + 'are' + 'ari' (<-'ar') + 'drama' + 'drima' + 'drom' + 'dru' + 'dra' + 'dre' + 'dri' (<-'dr') + 'erama' + 'erima' + 'erom' + 'eru' + 'era' + 'ere' + 'eri' (<-'er') + 'orama' + 'orima' + 'orom' + 'oru' + 'ora' + 'ore' + 'ori' (<-'or') + 'esima' + 'esom' + 'ese' + 'esa' + 'esu' (<-'es') + 'isima' + 'isom' + 'ise' + 'isa' + 'isu' (<-'is') + 'ta{sx}ama' + 'ta{sx}ima' + 'ta{sx}om' + 'ta{sx}em' + 'ta{sx}a' + 'ta{sx}u' + 'ta{sx}i' + 'ta{sx}e' (<-'ta{sx}') + 'na{sx}ama' + 'na{sx}ima' + 'na{sx}om' + 'na{sx}em' + 'na{sx}a' + 'na{sx}u' + 'na{sx}i' + 'na{sx}e' (<-'na{sx}') + 'ja{sx}ama' + 'ja{sx}ima' + 'ja{sx}om' + 'ja{sx}em' + 'ja{sx}a' + 'ja{sx}u' + 'ja{sx}i' + 'ja{sx}e' (<-'ja{sx}') + 'ka{sx}ama' + 'ka{sx}ima' + 'ka{sx}om' + 'ka{sx}em' + 'ka{sx}a' + 'ka{sx}u' + 'ka{sx}i' + 'ka{sx}e' (<-'ka{sx}') + 'ba{sx}ama' + 'ba{sx}ima' + 'ba{sx}om' + 'ba{sx}em' + 'ba{sx}a' + 'ba{sx}u' + 'ba{sx}i' + 'ba{sx}e' (<-'ba{sx}') + 'ga{sx}ama' + 'ga{sx}ima' + 'ga{sx}om' + 'ga{sx}em' + 'ga{sx}a' + 'ga{sx}u' + 'ga{sx}i' + 'ga{sx}e' (<-'ga{sx}') + 'va{sx}ama' + 'va{sx}ima' + 'va{sx}om' + 'va{sx}em' + 'va{sx}a' + 'va{sx}u' + 'va{sx}i' + 'va{sx}e' (<-'va{sx}') + 'e{sx}ima' + 'e{sx}ama' + 'e{sx}om' + 'e{sx}em' + 'e{sx}i' + 'e{sx}e' + 'e{sx}a' + 'e{sx}u' (<-'e{sx}') + 'i{sx}ima' + 'i{sx}ama' + 'i{sx}om' + 'i{sx}em' + 'i{sx}i' + 'i{sx}e' + 'i{sx}a' + 'i{sx}u' (<-'i{sx}') + 'ikatima' + 'ikatom' + 'ikata' + 'ikate' + 'ikati' + 'ikatu' + 'ikato' (<-'ikat') + 'latima' + 'latom' + 'lata' + 'late' + 'lati' + 'latu' + 'lato' (<-'lat') + 'etama' + 'etima' + 'etom' + 'eta' + 'ete' + 'eti' + 'etu' + 'eto' (<-'et') + 'estima' + 'estama' + 'estom' + 'esta' + 'este' + 'esti' + 'estu' + 'esto' (<-'est') + 'istima' + 'istama' + 'istom' + 'ista' + 'iste' + 'isti' + 'istu' + 'isto' (<-'ist') + 'kstima' + 'kstama' + 'kstom' + 'ksta' + 'kste' + 'ksti' + 'kstu' + 'ksto' (<-'kst') + 'ostima' + 'ostama' + 'ostom' + 'osta' + 'oste' + 'osti' + 'ostu' + 'osto' (<-'ost') + 'i{sx}tima' + 'i{sx}tem' + 'i{sx}ta' + 'i{sx}te' + 'i{sx}tu' (<-'i{sx}t') + 'ovasmo' + 'ovaste' + 'ovahu' + 'ovati' + 'ova{sx}e' + 'ovali' + 'ovala' + 'ovale' + 'ovalo' + 'ovat' + 'ovah' + 'ovao' (<-'ova') + 'avijemu' + 'avijima' + 'avijega' + 'avijeg' + 'avijem' + 'avemu' + 'avega' + 'aveg' + 'avem' + 'avijim' + 'avijih' + 'avijoj' + 'avoga' + 'avome' + 'avomu' + 'avima' + 'avama' + 'aviji' + 'avije' + 'avija' + 'aviju' + 'avim' + 'avih' + 'avoj' + 'avom' + 'avog' + 'avi' + 'ava' + 'avu' + 'ave' + 'avo' (<-'av') + 'evijemu' + 'evijima' + 'evijega' + 'evijeg' + 'evijem' + 'evemu' + 'evega' + 'eveg' + 'evem' + 'evijim' + 'evijih' + 'evijoj' + 'evoga' + 'evome' + 'evomu' + 'evima' + 'evama' + 'eviji' + 'evije' + 'evija' + 'eviju' + 'evim' + 'evih' + 'evoj' + 'evom' + 'evog' + 'evi' + 'eva' + 'evu' + 'eve' + 'evo' (<-'ev') + 'ivijemu' + 'ivijima' + 'ivijega' + 'ivijeg' + 'ivijem' + 'ivemu' + 'ivega' + 'iveg' + 'ivem' + 'ivijim' + 'ivijih' + 'ivijoj' + 'ivoga' + 'ivome' + 'ivomu' + 'ivima' + 'ivama' + 'iviji' + 'ivije' + 'ivija' + 'iviju' + 'ivim' + 'ivih' + 'ivoj' + 'ivom' + 'ivog' + 'ivi' + 'iva' + 'ivu' + 'ive' + 'ivo' (<-'iv') + 'ovijemu' + 'ovijima' + 'ovijega' + 'ovijeg' + 'ovijem' + 'ovemu' + 'ovega' + 'oveg' + 'ovijim' + 'ovijih' + 'ovijoj' + 'ovoga' + 'ovome' + 'ovomu' + 'ovima' + 'oviji' + 'ovije' + 'ovija' + 'oviju' + 'ovim' + 'ovih' + 'ovoj' + 'ovom' + 'ovog' + 'ovi' + 'ova' + 'ovu' + 'ove' + 'ovo' (<-'ov') + 'movima' + 'movom' + 'mova' + 'movu' + 'move' + 'movi' (<-'mov') + 'lovima' + 'lovom' + 'lova' + 'lovu' + 'love' + 'lovi' (<-'lov') + 'elijemu' + 'elijima' + 'elijega' + 'elijeg' + 'elijem' + 'elemu' + 'elega' + 'eleg' + 'elem' + 'elijim' + 'elijih' + 'elijoj' + 'eloga' + 'elome' + 'elomu' + 'elima' + 'eliji' + 'elije' + 'elija' + 'eliju' + 'elim' + 'elih' + 'eloj' + 'elom' + 'elog' + 'eli' + 'ela' + 'elu' + 'ele' + 'elo' (<-'el') + 'anjijemu' + 'anjijima' + 'anjijega' + 'anjijeg' + 'anjijem' + 'anjemu' + 'anjega' + 'anjeg' + 'anjem' + 'anjijim' + 'anjijih' + 'anjijoj' + 'anjoga' + 'anjome' + 'anjomu' + 'anjima' + 'anjiji' + 'anjije' + 'anjija' + 'anjiju' + 'anjim' + 'anjih' + 'anjoj' + 'anjom' + 'anjog' + 'anja' + 'anje' + 'anji' + 'anjo' + 'anju' (<-'anj') + 'enjijemu' + 'enjijima' + 'enjijega' + 'enjijeg' + 'enjijem' + 'enjemu' + 'enjega' + 'enjeg' + 'enjem' + 'enjijim' + 'enjijih' + 'enjijoj' + 'enjoga' + 'enjome' + 'enjomu' + 'enjima' + 'enjiji' + 'enjije' + 'enjija' + 'enjiju' + 'enjim' + 'enjih' + 'enjoj' + 'enjom' + 'enjog' + 'enja' + 'enje' + 'enji' + 'enjo' + 'enju' (<-'enj') + '{sx}njijemu' + '{sx}njijima' + '{sx}njijega' + '{sx}njijeg' + '{sx}njijem' + '{sx}njemu' + '{sx}njega' + '{sx}njeg' + '{sx}njem' + '{sx}njijim' + '{sx}njijih' + '{sx}njijoj' + '{sx}njoga' + '{sx}njome' + '{sx}njomu' + '{sx}njima' + '{sx}njiji' + '{sx}njije' + '{sx}njija' + '{sx}njiju' + '{sx}njim' + '{sx}njih' + '{sx}njoj' + '{sx}njom' + '{sx}njog' + '{sx}nja' + '{sx}nje' + '{sx}nji' + '{sx}njo' + '{sx}nju' (<-'{sx}nj') + 'anemu' + 'anega' + 'aneg' + 'anem' (<-'an') + 'enemu' + 'enega' + 'eneg' + 'enem' (<-'en') + '{sx}nemu' + '{sx}nega' + '{sx}neg' + '{sx}nem' (<-'{sx}n') + '{cx}inama' + '{cx}inome' + '{cx}inomu' + '{cx}inoga' + '{cx}inima' + '{cx}inog' + '{cx}inom' + '{cx}inim' + '{cx}inih' + '{cx}inoj' + '{cx}ina' + '{cx}inu' + '{cx}ini' + '{cx}ino' + '{cx}ine' (<-'{cx}in') + 'ro{sx}iv{sx}i' + 'ro{sx}ismo' + 'ro{sx}iste' + 'ro{sx}i{sx}e' + 'ro{sx}imo' + 'ro{sx}ite' + 'ro{sx}iti' + 'ro{sx}ili' + 'ro{sx}ila' + 'ro{sx}ilo' + 'ro{sx}ile' + 'ro{sx}im' + 'ro{sx}i{sx}' + 'ro{sx}it' + 'ro{sx}ih' + 'ro{sx}io' (<-'ro{sx}i') + 'o{sx}ijemu' + 'o{sx}ijima' + 'o{sx}ijega' + 'o{sx}ijeg' + 'o{sx}ijem' + 'o{sx}emu' + 'o{sx}ega' + 'o{sx}eg' + 'o{sx}em' + 'o{sx}ijim' + 'o{sx}ijih' + 'o{sx}ijoj' + 'o{sx}oga' + 'o{sx}ome' + 'o{sx}omu' + 'o{sx}ima' + 'o{sx}iji' + 'o{sx}ije' + 'o{sx}ija' + 'o{sx}iju' + 'o{sx}im' + 'o{sx}ih' + 'o{sx}oj' + 'o{sx}om' + 'o{sx}og' + 'o{sx}i' + 'o{sx}a' + 'o{sx}u' + 'o{sx}e' (<-'o{sx}') + 'evitijima' + 'evitijega' + 'evitijemu' + 'evitijem' + 'evitega' + 'evitemu' + 'evitem' + 'evitijim' + 'evitijih' + 'evitijoj' + 'evitijeg' + 'evitiji' + 'evitije' + 'evitija' + 'evitoga' + 'evitome' + 'evitomu' + 'evitima' + 'evitog' + 'evitom' + 'evitim' + 'evitih' + 'evitoj' + 'eviti' + 'evite' + 'evito' + 'evita' + 'evitu' (<-'evit') + 'ovitijima' + 'ovitijega' + 'ovitijemu' + 'ovitijem' + 'ovitega' + 'ovitemu' + 'ovitem' + 'ovitijim' + 'ovitijih' + 'ovitijoj' + 'ovitijeg' + 'ovitiji' + 'ovitije' + 'ovitija' + 'ovitoga' + 'ovitome' + 'ovitomu' + 'ovitima' + 'ovitog' + 'ovitom' + 'ovitim' + 'ovitih' + 'ovitoj' + 'oviti' + 'ovite' + 'ovito' + 'ovita' + 'ovitu' (<-'ovit') + 'astijima' + 'astijega' + 'astijemu' + 'astijem' + 'astega' + 'astemu' + 'astem' + 'astijim' + 'astijih' + 'astijoj' + 'astijeg' + 'astiji' + 'astije' + 'astija' + 'astoga' + 'astome' + 'astomu' + 'astima' + 'astog' + 'astom' + 'astim' + 'astih' + 'astoj' + 'asti' + 'aste' + 'asto' + 'asta' + 'astu' (<-'ast') + 'kijemu' + 'kijima' + 'kijega' + 'kijeg' + 'kijem' + 'kemu' + 'kega' + 'keg' + 'kem' + 'kijim' + 'kijih' + 'kijoj' + 'koga' + 'kome' + 'komu' + 'kima' + 'kiji' + 'kije' + 'kija' + 'kiju' + 'kim' + 'kih' + 'koj' + 'kom' + 'kog' + 'kov' + 'ki' + 'ka' + 'ku' + 'ke' + 'ko' (<-'k') + 'evaju{cy}i' + 'evasmo' + 'evaste' + 'evajmo' + 'evajte' + 'evaju' + 'evala' + 'evale' + 'evali' + 'evalo' + 'evamo' + 'evana' + 'evane' + 'evani' + 'evano' + 'evate' + 'evati' + 'eva{sx}e' + 'evahu' + 'evah' + 'evaj' + 'evam' + 'evan' + 'evao' + 'evat' + 'evav' + 'eva{sx}' (<-'eva') + 'avaju{cy}i' + 'avasmo' + 'avaste' + 'avajmo' + 'avajte' + 'avaju' + 'avala' + 'avale' + 'avali' + 'avalo' + 'avamo' + 'avana' + 'avane' + 'avani' + 'avano' + 'avate' + 'avati' + 'ava{sx}e' + 'avahu' + 'avah' + 'avaj' + 'avam' + 'avan' + 'avao' + 'avat' + 'avav' + 'ava{sx}' (<-'ava') + 'ivaju{cy}i' + 'ivasmo' + 'ivaste' + 'ivajmo' + 'ivajte' + 'ivaju' + 'ivala' + 'ivale' + 'ivali' + 'ivalo' + 'ivamo' + 'ivana' + 'ivane' + 'ivani' + 'ivano' + 'ivate' + 'ivati' + 'iva{sx}e' + 'ivahu' + 'ivah' + 'ivaj' + 'ivam' + 'ivan' + 'ivao' + 'ivat' + 'ivav' + 'iva{sx}' (<-'iva') + 'uvaju{cy}i' + 'uvasmo' + 'uvaste' + 'uvajmo' + 'uvajte' + 'uvaju' + 'uvala' + 'uvale' + 'uvali' + 'uvalo' + 'uvamo' + 'uvana' + 'uvane' + 'uvani' + 'uvano' + 'uvate' + 'uvati' + 'uva{sx}e' + 'uvahu' + 'uvah' + 'uvaj' + 'uvam' + 'uvan' + 'uvao' + 'uvat' + 'uvav' + 'uva{sx}' (<-'uva') + 'irujemo' + 'irujete' + 'iruju{cy}i' + 'iraju{cy}i' + 'irivat' + 'irujem' + 'iruje{sx}' + 'irujmo' + 'irujte' + 'irav{sx}i' + 'irasmo' + 'iraste' + 'irati' + 'iramo' + 'irate' + 'iraju' + 'ira{sx}e' + 'irahu' + 'irala' + 'iralo' + 'irali' + 'irale' + 'iruje' + 'iruju' + 'iruj' + 'iral' + 'iran' + 'iram' + 'ira{sx}' + 'irat' + 'irah' + 'irao' (<-'ir') + 'a{cx}ismo' + 'a{cx}iste' + 'a{cx}iti' + 'a{cx}imo' + 'a{cx}ite' + 'a{cx}i{sx}e' + 'a{cx}e{cy}i' + 'a{cx}ila' + 'a{cx}ilo' + 'a{cx}ili' + 'a{cx}ile' + 'a{cx}ena' + 'a{cx}eno' + 'a{cx}eni' + 'a{cx}ene' + 'a{cx}io' + 'a{cx}im' + 'a{cx}i{sx}' + 'a{cx}it' + 'a{cx}ih' + 'a{cx}en' + 'a{cx}i' + 'a{cx}e' (<-'a{cx}') + 'a{cx}av{sx}i' + 'a{cx}asmo' + 'a{cx}aste' + 'a{cx}ahu' + 'a{cx}ati' + 'a{cx}amo' + 'a{cx}ate' + 'a{cx}a{sx}e' + 'a{cx}ala' + 'a{cx}alo' + 'a{cx}ali' + 'a{cx}ale' + 'a{cx}aju' + 'a{cx}ana' + 'a{cx}ano' + 'a{cx}ani' + 'a{cx}ane' + 'a{cx}ao' + 'a{cx}am' + 'a{cx}a{sx}' + 'a{cx}at' + 'a{cx}ah' + 'a{cx}an' (<-'a{cx}a') + 'nuv{sx}i' + 'nusmo' + 'nuste' + 'nu{cy}i' + 'nimo' + 'nite' + 'nemo' + 'nete' + 'nula' + 'nulo' + 'nule' + 'nuli' + 'nuto' + 'nuti' + 'nuta' + 'ne{sx}' + 'nuo' + 'nut' (<-'n') + 'niv{sx}i' + 'nismo' + 'niste' + 'niti' + 'nila' + 'nilo' + 'nile' + 'nili' + 'ni{sx}' + 'nio' (<-'ni') + 'aju{cy}i' + 'av{sx}i' + 'asmo' + 'ajmo' + 'ajte' + 'ajem' + 'aloj' + 'amo' + 'ate' + 'aje' + 'aju' + 'ati' + 'a{sx}e' + 'ahu' + 'ala' + 'ali' + 'ale' + 'alo' + 'ano' + 'at' + 'ah' + 'ao' + 'aj' + 'an' + 'am' + 'a{sx}' (<-'a') + 'uraju{cy}i' + 'urasmo' + 'uraste' + 'urajmo' + 'urajte' + 'uramo' + 'urate' + 'uraju' + 'urati' + 'ura{sx}e' + 'urahu' + 'urala' + 'urali' + 'urale' + 'uralo' + 'urana' + 'urano' + 'urani' + 'urane' + 'ural' + 'urat' + 'urah' + 'urao' + 'uraj' + 'uran' + 'uram' + 'ura{sx}' (<-'ur') + 'astajasmo' + 'astajaste' + 'astajahu' + 'astajati' + 'astajemo' + 'astajete' + 'astaja{sx}e' + 'astajali' + 'astaju{cy}i' + 'astajala' + 'astajalo' + 'astajale' + 'astajmo' + 'astajao' + 'astajem' + 'astaje{sx}' + 'astajat' + 'astajah' + 'astajte' + 'astaje' + 'astaju' (<-'astaj') + 'istajasmo' + 'istajaste' + 'istajahu' + 'istajati' + 'istajemo' + 'istajete' + 'istaja{sx}e' + 'istajali' + 'istaju{cy}i' + 'istajala' + 'istajalo' + 'istajale' + 'istajmo' + 'istajao' + 'istajem' + 'istaje{sx}' + 'istajat' + 'istajah' + 'istajte' + 'istaje' + 'istaju' (<-'istaj') + 'ostajasmo' + 'ostajaste' + 'ostajahu' + 'ostajati' + 'ostajemo' + 'ostajete' + 'ostaja{sx}e' + 'ostajali' + 'ostaju{cy}i' + 'ostajala' + 'ostajalo' + 'ostajale' + 'ostajmo' + 'ostajao' + 'ostajem' + 'ostaje{sx}' + 'ostajat' + 'ostajah' + 'ostajte' + 'ostaje' + 'ostaju' (<-'ostaj') + 'alama' + 'alima' + 'alom' + 'alu' + 'al' (<-'a') + 'ajevima' + 'ajevi' + 'ajeva' + 'ajeve' + 'ajama' + 'ajima' + 'aja' + 'aji' (<-'aj') + 'astadosmo' + 'astadoste' + 'astado{sx}e' + 'astanemo' + 'astademo' + 'astanete' + 'astadete' + 'astanimo' + 'astanite' + 'astanila' + 'astav{sx}i' + 'astanem' + 'astadem' + 'astane{sx}' + 'astade{sx}' + 'astadoh' + 'astade' + 'astati' + 'astane' + 'astanu' + 'astadu' + 'astala' + 'astali' + 'astalo' + 'astale' + 'astat' + 'astao' (<-'asta') + 'istadosmo' + 'istadoste' + 'istado{sx}e' + 'istanemo' + 'istademo' + 'istanete' + 'istadete' + 'istanimo' + 'istanite' + 'istanila' + 'istav{sx}i' + 'istanem' + 'istadem' + 'istane{sx}' + 'istade{sx}' + 'istadoh' + 'istade' + 'istati' + 'istane' + 'istanu' + 'istadu' + 'istala' + 'istali' + 'istalo' + 'istale' + 'istat' + 'istao' (<-'ista') + 'ostadosmo' + 'ostadoste' + 'ostado{sx}e' + 'ostanemo' + 'ostademo' + 'ostanete' + 'ostadete' + 'ostanimo' + 'ostanite' + 'ostanila' + 'ostav{sx}i' + 'ostanem' + 'ostadem' + 'ostane{sx}' + 'ostade{sx}' + 'ostadoh' + 'ostade' + 'ostati' + 'ostane' + 'ostanu' + 'ostadu' + 'ostala' + 'ostali' + 'ostalo' + 'ostale' + 'ostat' + 'ostao' (<-'osta') + 'tasmo' + 'taste' + 'tajmo' + 'tajte' + 'tav{sx}i' + 'tati' + 'tamo' + 'tate' + 'taju' + 'tala' + 'talo' + 'tale' + 'tali' + 'tana' + 'tano' + 'tani' + 'tane' + 'tan' + 'taj' + 'tao' + 'tam' + 'ta{sx}' + 'tat' + 'tah' (<-'ta') + 'injasmo' + 'injaste' + 'injati' + 'injemo' + 'injete' + 'injali' + 'injala' + 'injalo' + 'injale' + 'inja{sx}e' + 'injahu' + 'injem' + 'inje{sx}' + 'injat' + 'injah' + 'injao' (<-'inj') + 'astemo' + 'astete' + 'astimo' + 'astite' + 'astu{cy}i' + 'aste{sx}' + 'asli' + 'asla' + 'aslo' + 'asle' (<-'as') + 'iv{sx}i' + 'ie{cy}i' + 'ismo' + 'imo' + 'ite' + 'iti' + 'ili' + 'ila' + 'ilo' + 'ile' + 'im' + 'i{sx}' + 'it' + 'ih' + 'io' (<-'i') + 'ijemo' + 'ijete' + 'ijem' + 'ije{sx}' + 'ijmo' + 'ijte' + 'iju' + 'ije' + 'ij' + 'ilu' (<-'i') + 'lu{cx}ujete' + 'lu{cx}uju{cy}i' + 'lu{cx}ujemo' + 'lu{cx}ujem' + 'lu{cx}uje{sx}' + 'lu{cx}ismo' + 'lu{cx}iste' + 'lu{cx}ujmo' + 'lu{cx}ujte' + 'lu{cx}uje' + 'lu{cx}uju' + 'lu{cx}i{sx}e' + 'lu{cx}iti' + 'lu{cx}imo' + 'lu{cx}ite' + 'lu{cx}ila' + 'lu{cx}ilo' + 'lu{cx}ili' + 'lu{cx}ile' + 'lu{cx}ena' + 'lu{cx}eno' + 'lu{cx}eni' + 'lu{cx}ene' + 'lu{cx}uj' + 'lu{cx}io' + 'lu{cx}en' + 'lu{cx}im' + 'lu{cx}i{sx}' + 'lu{cx}it' + 'lu{cx}ih' + 'lu{cx}e' + 'lu{cx}i' (<-'lu{cx}') + 'jetismo' + 'jetiste' + 'jeti{sx}e' + 'jetimo' + 'jetite' + 'jetiti' + 'jetili' + 'jetila' + 'jetilo' + 'jetile' + 'jetim' + 'jeti{sx}' + 'jetit' + 'jetih' + 'jetio' (<-'jeti') + 'emo' + 'em' + 'e{sx}' + 'elama' + 'el' (<-'e') + 'ilama' + 'ilima' + 'ilom' + 'il' (<-'i') + 'atijega' + 'atijemu' + 'atijima' + 'atijeg' + 'atijem' + 'atega' + 'atemu' + 'ateg' + 'atem' + 'atijih' + 'atijim' + 'atima' + 'atoga' + 'atome' + 'atomu' + 'atiji' + 'atije' + 'atija' + 'atiju' + 'atoj' + 'atog' + 'atom' + 'atim' + 'atih' + 'ata' + 'atu' + 'ato' (<-'at') + 'etav{sx}i' + 'etu{cy}i' + 'etemo' + 'etimo' + 'etem' + 'ete{sx}' (<-'et') + 'lucujuci' + 'lucujemo' + 'lucujete' + 'lucujem' + 'lucujes' + 'lucujmo' + 'lucujte' + 'lucismo' + 'luciste' + 'luciti' + 'lucite' + 'lucise' + 'lucuje' + 'lucuju' + 'lucila' + 'lucile' + 'lucili' + 'lucilo' + 'lucena' + 'luceni' + 'lucene' + 'luceno' + 'lucimo' + 'lucim' + 'lucis' + 'lucih' + 'lucit' + 'lucio' + 'lucuj' + 'lucen' + 'luce' + 'luci' (R2 <-'luc') + 'snjijima' + 'snjijemu' + 'snjijega' + 'snjijim' + 'snjijih' + 'snjijeg' + 'snjijoj' + 'snjiji' + 'snjija' + 'snjije' + 'snjiju' + 'snjima' + 'snjemu' + 'snjomu' + 'snjome' + 'snjega' + 'snjoga' + 'snjih' + 'snjim' + 'snjem' + 'snjom' + 'snjeg' + 'snjog' + 'snjoj' + 'snja' + 'snje' + 'snji' + 'snjo' + 'snju' (R2 <-'snj') + 'osijima' + 'osijemu' + 'osijega' + 'snjijem' + 'osijih' + 'osijim' + 'osijem' + 'osijeg' + 'osijoj' + 'osima' + 'osemu' + 'osomu' + 'osome' + 'osega' + 'osoga' + 'osija' + 'osije' + 'osiji' + 'osiju' + 'osih' + 'osim' + 'osem' + 'osom' + 'oseg' + 'osog' + 'osoj' + 'osa' + 'ose' + 'osi' + 'osu' (R2 <-'os') + 'acismo' + 'aciste' + 'acima' + 'acimo' + 'acome' + 'acomu' + 'acite' + 'aciti' + 'acise' + 'acila' + 'acile' + 'acili' + 'acilo' + 'acega' + 'acene' + 'aceci' + 'aceni' + 'acemu' + 'acena' + 'aceno' + 'acoga' + 'acoj' + 'acih' + 'acem' + 'acom' + 'acen' + 'acog' + 'acit' + 'acio' + 'aceg' + 'acim' + 'acuh' + 'acis' + 'ace' + 'aca' + 'aci' (R2 <-'ac') + 'ecome' + 'ecoga' + 'ecemu' + 'ecima' + 'ecega' + 'ecomu' + 'ecoj' + 'ecuh' + 'ecom' + 'ecog' + 'eceg' + 'ecih' + 'ecem' + 'ecim' + 'eca' + 'ece' (R2 <-'ec') + 'ucomu' + 'ucome' + 'ucima' + 'ucoga' + 'ucega' + 'ucemu' + 'ucih' + 'ucog' + 'uceg' + 'ucom' + 'ucem' + 'ucim' + 'ucuh' + 'ucoj' + 'uca' + 'uce' (R2 <-'uc') + 'rosismo' + 'rosivsi' + 'rosiste' + 'rositi' + 'rosili' + 'rosise' + 'rosite' + 'rosilo' + 'rosimo' + 'rosile' + 'rosila' + 'rosit' + 'rosis' + 'rosio' + 'rosim' + 'rosih' (R2 <-'rosi') + 'acavsi' + 'acaste' + 'acasmo' + 'acaju' + 'acane' + 'acate' + 'acali' + 'acani' + 'acati' + 'acale' + 'acahu' + 'acase' + 'acano' + 'acamo' + 'acalo' + 'acana' + 'acala' + 'acam' + 'acan' + 'acao' + 'acas' + 'acat' + 'acah' (R2 <-'aca') + 'jasima' + 'jasama' + 'jasem' + 'jasom' + 'jase' + 'jasi' + 'jasa' + 'jasu' (R2 <-'jas') + 'tasima' + 'tasama' + 'tasem' + 'tasom' + 'tase' + 'tasa' + 'tasu' + 'tasi' (R2 <-'tas') + 'gasima' + 'gasama' + 'gasem' + 'gasom' + 'gasi' + 'gasu' + 'gase' + 'gasa' (R2 <-'gas') + 'nasama' + 'nasima' + 'nasem' + 'nasom' + 'nasu' + 'nasi' + 'nase' + 'nasa' (R2 <-'nas') + 'kasama' + 'kasima' + 'kasom' + 'kasem' + 'kasi' + 'kasu' + 'kase' + 'kasa' (R2 <-'kas') + 'vasama' + 'vasima' + 'vasom' + 'vasem' + 'vasi' + 'vase' + 'vasa' + 'vasu' (R2 <-'vas') + 'basama' + 'basima' + 'basom' + 'basem' + 'basi' + 'base' + 'basu' + 'basa' (R2 <-'bas') + 'astuci' + 'astes' (R2 <-'as') + 'cinima' + 'cinome' + 'cinama' + 'cinomu' + 'cinoga' + 'cinom' + 'cinih' + 'cinim' + 'cinog' + 'cinoj' + 'cino' + 'cini' + 'cinu' + 'cine' + 'cina' (R2 <-'cin') + 'astajase' + 'astajuci' + 'astajes' (R2 <-'astaj') + 'istajase' + 'istajuci' + 'istajes' (R2 <-'istaj') + 'ostajase' + 'ostajuci' + 'ostajes' (R2 <-'ostaj') + 'astadose' + 'astades' + 'astanes' + 'astavsi' (R2 <-'asta') + 'istadose' + 'istades' + 'istanes' + 'istavsi' (R2 <-'ista') + 'ostadose' + 'ostades' + 'ostanes' + 'ostavsi' (R2 <-'osta') + 'avajuci' + 'avase' + 'avas' (R2 <-'ava') + 'evajuci' + 'evase' + 'evas' (R2 <-'eva') + 'ivajuci' + 'ivase' + 'ivas' (R2 <-'iva') + 'uvajuci' + 'uvase' + 'uvas' (R2 <-'uva') + 'ovase' (R2 <-'ova') + 'jetise' + 'jetis' (R2 <-'jeti') + 'injase' + 'injes' (R2 <-'inj') + 'istem' (R2 <-'ist') + 'esama' + 'esem' + 'esi' (R2 <-'es') + 'etavsi' + 'etuci' + 'etes' (R2 <-'et') + 'isama' + 'isem' + 'isi' (R2 <-'is') + 'irajuci' + 'irujuci' + 'irujes' + 'iravsi' + 'irase' + 'iras' (R2 <-'ir') + 'urajuci' + 'urase' + 'uras' (R2 <-'ur') + 'ujuci' + 'ujes' (R2 <-'uj') + 'nivsi' + 'nis' (R2 <-'ni') + 'snega' + 'snemu' + 'snem' + 'sneg' (R2 <-'sn') + 'tavsi' + 'tas' (R2 <-'ta') + 'ajuci' + 'avsi' + 'ase' + 'as' (R2 <-'a') + 'ijes' + 'ivsi' + 'ieci' + 'is' (R2 <-'i') + 'es' (R2 <-'e') + 'nuvsi' + 'nuci' + 'nes' (R2 <-'n') + ) + ) + + define Step_3 as ( + [substring] R1 among ( + 'enom' + 'enoj' + 'enog' + 'enim' + 'enih' + 'anoj' + 'anog' + 'anim' + 'anih' + 'ost' + 'eno' + 'eni' + 'oga' + 'ima' + 'enu' + 'ena' + 'ama' + 'ano' + 'ani' + 'om' + 'og' + 'u' + 'o' + 'i' + 'e' + 'a' (<-'') + ) + ) +) + +define stem as ( + do cyr_to_lat + do prelude + do mark_regions + backwards ( + do Step_1 + do (Step_2 or Step_3) + ) +) diff --git a/contrib/snowball/algorithms/spanish.sbl b/contrib/snowball/algorithms/spanish.sbl new file mode 100644 index 0000000..6638f5f --- /dev/null +++ b/contrib/snowball/algorithms/spanish.sbl @@ -0,0 +1,230 @@ +routines ( + postlude mark_regions + RV R1 R2 + attached_pronoun + standard_suffix + y_verb_suffix + verb_suffix + residual_suffix +) + +externals ( stem ) + +integers ( pV p1 p2 ) + +groupings ( v ) + +stringescapes {} + +/* special characters */ + +stringdef a' '{U+00E1}' // a-acute +stringdef e' '{U+00E9}' // e-acute +stringdef i' '{U+00ED}' // i-acute +stringdef o' '{U+00F3}' // o-acute +stringdef u' '{U+00FA}' // u-acute +stringdef u" '{U+00FC}' // u-diaeresis +stringdef n~ '{U+00F1}' // n-tilde + +define v 'aeiou{a'}{e'}{i'}{o'}{u'}{u"}' + +define mark_regions as ( + + $pV = limit + $p1 = limit + $p2 = limit // defaults + + do ( + ( v (non-v gopast v) or (v gopast non-v) ) + or + ( non-v (non-v gopast v) or (v next) ) + setmark pV + ) + do ( + gopast v gopast non-v setmark p1 + gopast v gopast non-v setmark p2 + ) +) + +define postlude as repeat ( + [substring] among( + '{a'}' (<- 'a') + '{e'}' (<- 'e') + '{i'}' (<- 'i') + '{o'}' (<- 'o') + '{u'}' (<- 'u') + // and possibly {u"}->u here, or in prelude + '' (next) + ) //or next +) + +backwardmode ( + + define RV as $pV <= cursor + define R1 as $p1 <= cursor + define R2 as $p2 <= cursor + + define attached_pronoun as ( + [substring] among( + 'me' 'se' 'sela' 'selo' 'selas' 'selos' 'la' 'le' 'lo' + 'las' 'les' 'los' 'nos' + ) + substring RV among( + 'i{e'}ndo' (] <- 'iendo') + '{a'}ndo' (] <- 'ando') + '{a'}r' (] <- 'ar') + '{e'}r' (] <- 'er') + '{i'}r' (] <- 'ir') + 'ando' + 'iendo' + 'ar' 'er' 'ir' + (delete) + 'yendo' ('u' delete) + ) + ) + + define standard_suffix as ( + [substring] among( + + 'anza' 'anzas' + 'ico' 'ica' 'icos' 'icas' + 'ismo' 'ismos' + 'able' 'ables' + 'ible' 'ibles' + 'ista' 'istas' + 'oso' 'osa' 'osos' 'osas' + 'amiento' 'amientos' + 'imiento' 'imientos' + ( + R2 delete + ) + 'adora' 'ador' 'aci{o'}n' + 'adoras' 'adores' 'aciones' + 'ante' 'antes' 'ancia' 'ancias'// Note 1 + ( + R2 delete + try ( ['ic'] R2 delete ) + ) + 'log{i'}a' + 'log{i'}as' + ( + R2 <- 'log' + ) + 'uci{o'}n' 'uciones' + ( + R2 <- 'u' + ) + 'encia' 'encias' + ( + R2 <- 'ente' + ) + 'amente' + ( + R1 delete + try ( + [substring] R2 delete among( + 'iv' (['at'] R2 delete) + 'os' + 'ic' + 'ad' + ) + ) + ) + 'mente' + ( + R2 delete + try ( + [substring] among( + 'ante' // Note 1 + 'able' + 'ible' (R2 delete) + ) + ) + ) + 'idad' + 'idades' + ( + R2 delete + try ( + [substring] among( + 'abil' + 'ic' + 'iv' (R2 delete) + ) + ) + ) + 'iva' 'ivo' + 'ivas' 'ivos' + ( + R2 delete + try ( + ['at'] R2 delete // but not a further ['ic'] R2 delete + ) + ) + ) + ) + + define y_verb_suffix as ( + setlimit tomark pV for ([substring]) among( + 'ya' 'ye' 'yan' 'yen' 'yeron' 'yendo' 'yo' 'y{o'}' + 'yas' 'yes' 'yais' 'yamos' + ('u' delete) + ) + ) + + define verb_suffix as ( + setlimit tomark pV for ([substring]) among( + + 'en' 'es' '{e'}is' 'emos' + (try ('u' test 'g') ] delete) + + 'ar{i'}an' 'ar{i'}as' 'ar{a'}n' 'ar{a'}s' 'ar{i'}ais' + 'ar{i'}a' 'ar{e'}is' 'ar{i'}amos' 'aremos' 'ar{a'}' + 'ar{e'}' + 'er{i'}an' 'er{i'}as' 'er{a'}n' 'er{a'}s' 'er{i'}ais' + 'er{i'}a' 'er{e'}is' 'er{i'}amos' 'eremos' 'er{a'}' + 'er{e'}' + 'ir{i'}an' 'ir{i'}as' 'ir{a'}n' 'ir{a'}s' 'ir{i'}ais' + 'ir{i'}a' 'ir{e'}is' 'ir{i'}amos' 'iremos' 'ir{a'}' + 'ir{e'}' + + 'aba' 'ada' 'ida' '{i'}a' 'ara' 'iera' 'ad' 'ed' + 'id' 'ase' 'iese' 'aste' 'iste' 'an' 'aban' '{i'}an' + 'aran' 'ieran' 'asen' 'iesen' 'aron' 'ieron' 'ado' + 'ido' 'ando' 'iendo' 'i{o'}' 'ar' 'er' 'ir' 'as' + 'abas' 'adas' 'idas' '{i'}as' 'aras' 'ieras' 'ases' + 'ieses' '{i'}s' '{a'}is' 'abais' '{i'}ais' 'arais' + 'ierais' 'aseis' 'ieseis' 'asteis' 'isteis' 'ados' + 'idos' 'amos' '{a'}bamos' '{i'}amos' 'imos' + '{a'}ramos' 'i{e'}ramos' 'i{e'}semos' '{a'}semos' + (delete) + ) + ) + + define residual_suffix as ( + [substring] among( + 'os' + 'a' 'o' '{a'}' '{i'}' '{o'}' + ( RV delete ) + 'e' '{e'}' + ( RV delete try( ['u'] test 'g' RV delete ) ) + ) + ) +) + +define stem as ( + do mark_regions + backwards ( + do attached_pronoun + do ( standard_suffix or + y_verb_suffix or + verb_suffix + ) + do residual_suffix + ) + do postlude +) + +/* + Note 1: additions of 15 Jun 2005 +*/ diff --git a/contrib/snowball/algorithms/swedish.sbl b/contrib/snowball/algorithms/swedish.sbl new file mode 100644 index 0000000..2cbb885 --- /dev/null +++ b/contrib/snowball/algorithms/swedish.sbl @@ -0,0 +1,72 @@ +routines ( + mark_regions + main_suffix + consonant_pair + other_suffix +) + +externals ( stem ) + +integers ( p1 x ) + +groupings ( v s_ending ) + +stringescapes {} + +/* special characters */ + +stringdef a" '{U+00E4}' +stringdef ao '{U+00E5}' +stringdef o" '{U+00F6}' + +define v 'aeiouy{a"}{ao}{o"}' + +define s_ending 'bcdfghjklmnoprtvy' + +define mark_regions as ( + + $p1 = limit + test ( hop 3 setmark x ) + goto v gopast non-v setmark p1 + try ( $p1 < x $p1 = x ) +) + +backwardmode ( + + define main_suffix as ( + setlimit tomark p1 for ([substring]) + among( + + 'a' 'arna' 'erna' 'heterna' 'orna' 'ad' 'e' 'ade' 'ande' 'arne' + 'are' 'aste' 'en' 'anden' 'aren' 'heten' 'ern' 'ar' 'er' 'heter' + 'or' 'as' 'arnas' 'ernas' 'ornas' 'es' 'ades' 'andes' 'ens' 'arens' + 'hetens' 'erns' 'at' 'andet' 'het' 'ast' + (delete) + 's' + (s_ending delete) + ) + ) + + define consonant_pair as setlimit tomark p1 for ( + among('dd' 'gd' 'nn' 'dt' 'gt' 'kt' 'tt') + and ([next] delete) + ) + + define other_suffix as setlimit tomark p1 for ( + [substring] among( + 'lig' 'ig' 'els' (delete) + 'l{o"}st' (<-'l{o"}s') + 'fullt' (<-'full') + ) + ) +) + +define stem as ( + + do mark_regions + backwards ( + do main_suffix + do consonant_pair + do other_suffix + ) +) diff --git a/contrib/snowball/algorithms/tamil.sbl b/contrib/snowball/algorithms/tamil.sbl new file mode 100644 index 0000000..9635777 --- /dev/null +++ b/contrib/snowball/algorithms/tamil.sbl @@ -0,0 +1,405 @@ +/* +* Affix stripping stemming algorithm for Tamil +* By Damodharan Rajalingam +*/ + +stringescapes {} + +/* Aytham */ +stringdef aytham '{U+0B83}' + +/* Uyir - independent vowels */ +stringdef a '{U+0B85}' +stringdef aa '{U+0B86}' +stringdef i '{U+0B87}' +stringdef ii '{U+0B88}' +stringdef u '{U+0B89}' +stringdef uu '{U+0B8A}' +stringdef e '{U+0B8E}' +stringdef ee '{U+0B8F}' +stringdef ai '{U+0B90}' +stringdef o '{U+0B92}' +stringdef oo '{U+0B93}' +stringdef au '{U+0B94}' + +/* Consonants */ +stringdef ka '{U+0B95}' +stringdef nga '{U+0B99}' +stringdef ca '{U+0B9A}' +stringdef ja '{U+0B9C}' +stringdef nya '{U+0B9E}' +stringdef tta '{U+0B9F}' +stringdef nna '{U+0BA3}' +stringdef ta '{U+0BA4}' +stringdef tha '{U+0BA4}' +stringdef na '{U+0BA8}' +stringdef nnna '{U+0BA9}' +stringdef pa '{U+0BAA}' +stringdef ma '{U+0BAE}' +stringdef ya '{U+0BAF}' +stringdef ra '{U+0BB0}' +stringdef rra '{U+0BB1}' +stringdef la '{U+0BB2}' +stringdef lla '{U+0BB3}' +stringdef llla '{U+0BB4}' +stringdef zha '{U+0BB4}' +stringdef va '{U+0BB5}' + +/* Vatamozi - borrowed */ +stringdef sha '{U+0BB6}' +stringdef ssa '{U+0BB7}' +stringdef sa '{U+0BB8}' +stringdef ha '{U+0BB9}' + + +/* Dependent vowel signs (kombu etc.) */ +stringdef vs_aa '{U+0BBE}' +stringdef vs_i '{U+0BBF}' +stringdef vs_ii '{U+0BC0}' +stringdef vs_u '{U+0BC1}' +stringdef vs_uu '{U+0BC2}' +stringdef vs_e '{U+0BC6}' +stringdef vs_ee '{U+0BC7}' +stringdef vs_ai '{U+0BC8}' +stringdef vs_o '{U+0BCA}' +stringdef vs_oo '{U+0BCB}' +stringdef vs_au '{U+0BCC}' + +/* Pulli */ +stringdef pulli '{U+0BCD}' + +/* AU length markk */ +stringdef au_lmark '{U+0BD7}' + + +routines ( + remove_plural_suffix + remove_question_suffixes + remove_question_prefixes + remove_pronoun_prefixes + remove_command_suffixes + remove_um + remove_vetrumai_urupukal + fix_va_start + fix_ending + fix_endings + remove_tense_suffix + remove_tense_suffixes + remove_common_word_endings + has_min_length +) + +externals ( stem ) + +booleans ( + found_a_match + found_vetrumai_urupu +) + +define has_min_length as ( + $(len > 4) +) + +define fix_va_start as ( + (try '{va}{vs_oo}' and [ '{va}{vs_oo}' ] <- '{oo}' ) or + (try '{va}{vs_o}' and [ '{va}{vs_o}' ] <- '{o}' ) or + (try '{va}{vs_u}' and [ '{va}{vs_u}' ] <- '{u}' ) or + (try '{va}{vs_uu}' and [ '{va}{vs_uu}' ] <- '{uu}' ) +) + +define fix_endings as ( + do repeat fix_ending +) + +define remove_question_prefixes as ( + [ ('{e}' ) among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete + do fix_va_start +) + +// Gives signal t if an ending was fixed, signal f otherwise. +define fix_ending as ( + $(len > 3) + backwards ( + ( [among('{na}{pulli}' '{na}{pulli}{ta}' '{na}{pulli}{ta}{pulli}') ] delete ) + or + ( ['{ya}{pulli}' test among('{vs_ai}' '{vs_i}' '{vs_ii}') ] delete ) + or + ( [ '{tta}{pulli}{pa}{pulli}' or '{tta}{pulli}{ka}{pulli}' ] <- '{lla}{pulli}' ) + or + ( [ '{nnna}{pulli}{rra}{pulli}' ] <- '{la}{pulli}' ) + or +// ( [ '{rra}{pulli}{ka}{pulli}' or '{nnna}{pulli}{nnna}{pulli}' ] <- '{la}{pulli}' ) + ( [ '{rra}{pulli}{ka}{pulli}' ] <- '{la}{pulli}' ) + or + ( [ '{tta}{pulli}{tta}{pulli}' ] <- '{tta}{vs_u}' ) + or + ( found_vetrumai_urupu [ '{ta}{pulli}{ta}{pulli}' (test not '{vs_ai}') ] <- '{ma}{pulli}' ] ) + or + ( [ '{vs_u}{ka}{pulli}' or '{vs_u}{ka}{pulli}{ka}{pulli}' ] <- '{pulli}' ) + or + ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete ) + or + ( [ '{vs_u}{ka}{pulli}' ] <- '{pulli}' ) + or + ( [ '{pulli}' among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') ] delete ) + or + ( [ '{pulli}' (among('{ya}' '{ra}' '{la}' '{va}' '{zha}' '{lla}') or among('{nga}' '{nya}' '{nna}' '{na}' '{ma}' '{nnna}')) '{pulli}' ] <- '{pulli}' ) + or + ( [ among('{va}' '{ya}' '{va}{pulli}') ] delete ) + or + ( [ '{nnna}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}')) ] delete ) + or + ( [ '{nga}{pulli}' (test not '{vs_ai}')] <- '{ma}{pulli}' ) + or + ( [ '{nga}{pulli}' ] delete ) + or + ( [ '{pulli}' (test (among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}') or '{pulli}')) ] delete ) + ) +) + +define remove_pronoun_prefixes as ( + unset found_a_match + [ among('{a}' '{i}' '{u}') among('{ka}' '{ca}' '{tha}' '{va}' '{na}' '{pa}' '{ma}' '{ya}' '{nga}' '{nya}') '{pulli}' ] delete + (set found_a_match) + do fix_va_start +) + +define remove_plural_suffix as ( + unset found_a_match + backwards ( + ( [ '{vs_u}{nga}{pulli}{ka}{lla}{pulli}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}')) ] <- '{pulli}' ) or + ( [ '{rra}{pulli}{ka}{lla}{pulli}' ] <- '{la}{pulli}' ) or + ( [ '{tta}{pulli}{ka}{lla}{pulli}' ] <- '{lla}{pulli}' ) or + ( [ '{ka}{lla}{pulli}' ] delete ) + (set found_a_match) + ) +) + +define remove_question_suffixes as ( + has_min_length + unset found_a_match + backwards ( + do ( + [ among('{vs_oo}' '{vs_ee}' '{vs_aa}') ] <- '{pulli}' + (set found_a_match) + ) + ) + do fix_endings +) + +define remove_command_suffixes as ( + has_min_length + unset found_a_match + backwards ( + [ among('{pa}{vs_i}' '{va}{vs_i}') ] delete + (set found_a_match) + ) +) + +define remove_um as ( + unset found_a_match + has_min_length + backwards ( [ '{vs_u}{ma}{pulli}' ] <- '{pulli}' + (set found_a_match) + ) + do fix_ending +) + +define remove_common_word_endings as ( + // These are not suffixes actually but are + // some words that are attached to other words + // but can be removed for stemming + unset found_a_match + has_min_length + backwards ( + test ( [ '{vs_u}{tta}{nnna}{pulli}' or + '{vs_i}{la}{pulli}{la}{vs_ai}' or + '{vs_i}{tta}{ma}{pulli}' or + '{vs_i}{nnna}{pulli}{rra}{vs_i}' or + '{vs_aa}{ka}{vs_i}' or + '{vs_aa}{ka}{vs_i}{ya}' or + '{vs_e}{nnna}{pulli}{rra}{vs_u}' or + '{vs_u}{lla}{pulli}{lla}' or + '{vs_u}{tta}{vs_ai}{ya}' or + '{vs_u}{tta}{vs_ai}' or + '{vs_e}{nnna}{vs_u}{ma}{pulli}' or + ('{la}{pulli}{la}' test (not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or + '{vs_e}{nnna}' or + '{vs_aa}{ka}{vs_i}' ] <- '{pulli}' + (set found_a_match) + ) + or + test ( [ among('{pa}{tta}{vs_u}' + '{pa}{tta}{pulli}{tta}' + '{pa}{tta}{pulli}{tta}{vs_u}' + '{pa}{tta}{pulli}{tta}{ta}{vs_u}' + '{pa}{tta}{pulli}{tta}{nna}' + '{ka}{vs_u}{ra}{vs_i}{ya}' + '{pa}{rra}{pulli}{rra}{vs_i}' + '{va}{vs_i}{tta}{vs_u}' + '{va}{vs_i}{tta}{pulli}{tta}{vs_u}' + '{pa}{tta}{vs_i}{ta}{vs_aa}{nnna}' + '{pa}{tta}{vs_i}' + '{ta}{vs_aa}{nnna}' + '{vs_e}{la}{pulli}{la}{vs_aa}{ma}{pulli}') + ] delete + (set found_a_match) + ) + ) + do fix_endings +) + +define remove_vetrumai_urupukal as ( + unset found_a_match + unset found_vetrumai_urupu + has_min_length + backwards ( + ( + test ( ['{nnna}{vs_ai}'] delete ) + or + test ([ ( '{vs_i}{nnna}{vs_ai}' or + '{vs_ai}' (test not among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}'))) or + ( '{vs_ai}' (test (among('{ka}' '{ca}' '{tta}' '{tha}' '{pa}' '{rra}') '{pulli}'))) + ] <- '{pulli}' + ) + or + test ( [ + '{vs_o}{tta}{vs_u}' or + '{vs_oo}{tta}{vs_u}' or + '{vs_i}{la}{pulli}' or + '{vs_i}{rra}{pulli}' or + ('{vs_i}{nnna}{pulli}' (test not '{ma}')) or + '{vs_i}{nnna}{pulli}{rra}{vs_u}' or + '{vs_i}{ra}{vs_u}{na}{pulli}{ta}{vs_u}' or + '{va}{vs_i}{tta}' or + ($(len >= 7) '{vs_i}{tta}{ma}{pulli}') or + '{vs_aa}{la}{pulli}' or + '{vs_u}{tta}{vs_ai}' or + '{vs_aa}{ma}{la}{pulli}' or + ('{la}{pulli}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or + '{vs_u}{lla}{pulli}' + ] <- '{pulli}' + ) + or + test ( [ + '{ka}{nna}{pulli}' or + '{ma}{vs_u}{nnna}{pulli}' or + '{ma}{vs_ee}{la}{pulli}' or + '{ma}{vs_ee}{rra}{pulli}' or + '{ka}{vs_ii}{llla}{pulli}' or + '{pa}{vs_i}{nnna}{pulli}' or + ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) + ] delete + ) + or + test ([ '{vs_ii}' ] <- '{vs_i}') + ) + (set found_a_match) + (set found_vetrumai_urupu) + do ( [ '{vs_i}{nnna}{pulli}' ] <- '{pulli}' ) + ) + do fix_endings +) + +define remove_tense_suffixes as ( + set found_a_match + repeat ( found_a_match (do remove_tense_suffix) ) +) + +define remove_tense_suffix as ( + unset found_a_match + has_min_length + backwards ( + do ( + test ( [among( + '{ka}{vs_o}{nna}{pulli}{tta}{vs_i}{ra}{pulli}' + '{pa}{tta}{vs_u}' + )] delete + (set found_a_match) + ) + or + test ( [ + '{ma}{vs_aa}{ra}{pulli}' or + '{ma}{vs_i}{nnna}{pulli}' or + '{nnna}{nnna}{pulli}' or + '{nnna}{vs_aa}{nnna}{pulli}' or + '{nnna}{vs_aa}{lla}{pulli}' or + '{nnna}{vs_aa}{ra}{pulli}' or + ('{va}{nnna}{pulli}' test (not among('{a}' '{aa}' '{i}' '{ii}' '{u}' '{uu}' '{e}' '{ee}' '{ai}' '{o}' '{oo}' '{au}')) ) or + '{nnna}{lla}{pulli}' or + '{va}{lla}{pulli}' or + '{nnna}{ra}{pulli}' or + '{va}{ra}{pulli}' or + '{nnna}' or '{pa}' or '{ka}' or '{ta}' or '{ya}' or + '{pa}{nnna}{pulli}' or + '{pa}{lla}{pulli}' or + '{pa}{ra}{pulli}' or + ('{ta}{vs_u}' (test not among('{vs_aa}' '{vs_i}' '{vs_ii}' '{vs_e}' '{vs_ee}' '{vs_u}' '{vs_uu}' '{vs_ai}'))) or + '{vs_i}{rra}{pulli}{rra}{vs_u}' or + '{pa}{ma}{pulli}' or + '{nnna}{ma}{pulli}' or + '{ta}{vs_u}{ma}{pulli}' or + '{rra}{vs_u}{ma}{pulli}' or + '{ka}{vs_u}{ma}{pulli}' or + '{nnna}{vs_e}{nnna}{pulli}' or + '{nnna}{vs_ai}' or + '{va}{vs_ai}' + ] delete + (set found_a_match) + ) + or + test ( [ + ('{vs_aa}{nnna}{pulli}' test (not '{ca}')) or + '{vs_aa}{lla}{pulli}' or + '{vs_aa}{ra}{pulli}' or + '{vs_ee}{nnna}{pulli}' or + '{vs_aa}' or + '{vs_aa}{ma}{pulli}' or + '{vs_e}{ma}{pulli}' or + '{vs_ee}{ma}{pulli}' or + '{vs_oo}{ma}{pulli}' or + '{ka}{vs_u}{ma}{pulli}' or + '{ta}{vs_u}{ma}{pulli}' or + '{tta}{vs_u}{ma}{pulli}' or + '{rra}{vs_u}{ma}{pulli}' or + '{vs_aa}{ya}{pulli}' or + '{nnna}{vs_e}{nnna}{pulli}' or + '{nnna}{vs_i}{ra}{pulli}' or + '{vs_ii}{ra}{pulli}' or + '{vs_ii}{ya}{ra}{pulli}' + ] <- '{pulli}' + (set found_a_match) + ) + or + test ( ([ '{ka}{vs_u}' or '{ta}{vs_u}' ) (test '{pulli}') ] delete + (set found_a_match) + ) + ) + do ([among( + '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}' + '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}{pulli}' + '{ka}{vs_i}{nnna}{pulli}{rra}' + '{ka}{vs_i}{nnna}{pulli}{rra}{pulli}' + '{ka}{vs_i}{rra}' + '{ka}{vs_i}{rra}{pulli}' + )] delete + (set found_a_match) + ) + ) + do fix_endings +) + +define stem as ( + unset found_vetrumai_urupu + do fix_ending + has_min_length + do remove_question_prefixes + do remove_pronoun_prefixes + do remove_question_suffixes + do remove_um + do remove_common_word_endings + do remove_vetrumai_urupukal + do remove_plural_suffix + do remove_command_suffixes + do remove_tense_suffixes +) diff --git a/contrib/snowball/algorithms/turkish.sbl b/contrib/snowball/algorithms/turkish.sbl new file mode 100644 index 0000000..eadd61d --- /dev/null +++ b/contrib/snowball/algorithms/turkish.sbl @@ -0,0 +1,470 @@ +/* Stemmer for Turkish + * author: Evren (Kapusuz) Çilden + * email: evren.kapusuz at gmail.com + * version: 1.0 (15.01.2007) + + + * stems nominal verb suffixes + * stems nominal inflections + * more than one syllable word check + * (y,n,s,U) context check + * vowel harmony check + * last consonant check and conversion (b, c, d, ğ to p, ç, t, k) + + * The stemming algorithm is based on the paper "An Affix Stripping + * Morphological Analyzer for Turkish" by Gülşen Eryiğit and + * Eşref Adalı (Proceedings of the IAESTED International Conference + * ARTIFICIAL INTELLIGENCE AND APPLICATIONS, February 16-18,2004, + * Innsbruck, Austria + + * Turkish is an agglutinative language and has a very rich morphological + * structure. In Turkish, you can form many different words from a single stem + * by appending a sequence of suffixes. Eg. The word "doktoruymuşsunuz" means + * "You had been the doctor of him". The stem of the word is "doktor" and it + * takes three different suffixes -sU, -ymUs, and -sUnUz. The rules about + * the append order of suffixes can be clearly described as FSMs. + * The paper referenced above defines some FSMs for right to left + * morphological analysis. I generated a method for constructing snowball + * expressions from right to left FSMs for stemming suffixes. +*/ + +routines ( + append_U_to_stems_ending_with_d_or_g // for preventing some overstemmings + check_vowel_harmony // tests vowel harmony for suffixes + is_reserved_word // tests whether current string is a reserved word ('ad','soyad') + mark_cAsInA // nominal verb suffix + mark_DA // noun suffix + mark_DAn // noun suffix + mark_DUr // nominal verb suffix + mark_ki // noun suffix + mark_lAr // noun suffix, nominal verb suffix + mark_lArI // noun suffix + mark_nA // noun suffix + mark_ncA // noun suffix + mark_ndA // noun suffix + mark_ndAn // noun suffix + mark_nU // noun suffix + mark_nUn // noun suffix + mark_nUz // nominal verb suffix + mark_sU // noun suffix + mark_sUn // nominal verb suffix + mark_sUnUz // nominal verb suffix + mark_possessives // -(U)m,-(U)n,-(U)mUz,-(U)nUz, + mark_yA // noun suffix + mark_ylA // noun suffix + mark_yU // noun suffix + mark_yUm // nominal verb suffix + mark_yUz // nominal verb suffix + mark_yDU // nominal verb suffix + mark_yken // nominal verb suffix + mark_ymUs_ // nominal verb suffix + mark_ysA // nominal verb suffix + + mark_suffix_with_optional_y_consonant + mark_suffix_with_optional_U_vowel + mark_suffix_with_optional_n_consonant + mark_suffix_with_optional_s_consonant + + more_than_one_syllable_word + + post_process_last_consonants + postlude + + stem_nominal_verb_suffixes + stem_noun_suffixes + stem_suffix_chain_before_ki +) + +stringescapes { } + +/* Special characters in Unicode Latin-1 and Latin Extended-A */ +stringdef c, '{U+00E7}' // LATIN SMALL LETTER C WITH CEDILLA +stringdef g~ '{U+011F}' // LATIN SMALL LETTER G WITH BREVE +stringdef i' '{U+0131}' // LATIN SMALL LETTER I WITHOUT DOT +stringdef o" '{U+00F6}' // LATIN SMALL LETTER O WITH DIAERESIS +stringdef s, '{U+015F}' // LATIN SMALL LETTER S WITH CEDILLA +stringdef u" '{U+00FC}' // LATIN SMALL LETTER U WITH DIAERESIS + +booleans ( continue_stemming_noun_suffixes ) + +groupings ( vowel U vowel1 vowel2 vowel3 vowel4 vowel5 vowel6) + +define vowel 'ae{i'}io{o"}u{u"}' +define U '{i'}iu{u"}' + +// the vowel grouping definitions below are used for checking vowel harmony +define vowel1 'a{i'}ou' // vowels that can end with suffixes containing 'a' +define vowel2 'ei{o"}{u"}' // vowels that can end with suffixes containing 'e' +define vowel3 'a{i'}' // vowels that can end with suffixes containing 'i'' +define vowel4 'ei' // vowels that can end with suffixes containing 'i' +define vowel5 'ou' // vowels that can end with suffixes containing 'o' or 'u' +define vowel6 '{o"}{u"}' // vowels that can end with suffixes containing 'o"' or 'u"' + +externals ( stem ) + +backwardmode ( + // checks vowel harmony for possible suffixes, + // helps to detect whether the candidate for suffix applies to vowel harmony + // this rule is added to prevent over stemming + define check_vowel_harmony as ( + test + ( + (goto vowel) // if there is a vowel + ( + ('a' goto vowel1) or + ('e' goto vowel2) or + ('{i'}' goto vowel3) or + ('i' goto vowel4) or + ('o' goto vowel5) or + ('{o"}' goto vowel6) or + ('u' goto vowel5) or + ('{u"}' goto vowel6) + ) + ) + ) + + // if the last consonant before suffix is vowel and n then advance and delete + // if the last consonant before suffix is non vowel and n do nothing + // if the last consonant before suffix is not n then only delete the suffix + // assumption: slice beginning is set correctly + define mark_suffix_with_optional_n_consonant as ( + ('n' (test vowel)) + or + ((not(test 'n')) test(next vowel)) + + ) + + // if the last consonant before suffix is vowel and s then advance and delete + // if the last consonant before suffix is non vowel and s do nothing + // if the last consonant before suffix is not s then only delete the suffix + // assumption: slice beginning is set correctly + define mark_suffix_with_optional_s_consonant as ( + ('s' (test vowel)) + or + ((not(test 's')) test(next vowel)) + ) + + // if the last consonant before suffix is vowel and y then advance and delete + // if the last consonant before suffix is non vowel and y do nothing + // if the last consonant before suffix is not y then only delete the suffix + // assumption: slice beginning is set correctly + define mark_suffix_with_optional_y_consonant as ( + ('y' (test vowel)) + or + ((not(test 'y')) test(next vowel)) + ) + + define mark_suffix_with_optional_U_vowel as ( + (U (test non-vowel)) + or + ((not(test U)) test(next non-vowel)) + + ) + + define mark_possessives as ( + among ('m{i'}z' 'miz' 'muz' 'm{u"}z' + 'n{i'}z' 'niz' 'nuz' 'n{u"}z' 'm' 'n') + (mark_suffix_with_optional_U_vowel) + ) + + define mark_sU as ( + check_vowel_harmony + U + (mark_suffix_with_optional_s_consonant) + ) + + define mark_lArI as ( + among ('leri' 'lar{i'}') + ) + + define mark_yU as ( + check_vowel_harmony + U + (mark_suffix_with_optional_y_consonant) + ) + + define mark_nU as ( + check_vowel_harmony + among ('n{i'}' 'ni' 'nu' 'n{u"}') + ) + + define mark_nUn as ( + check_vowel_harmony + among ('{i'}n' 'in' 'un' '{u"}n') + (mark_suffix_with_optional_n_consonant) + ) + + define mark_yA as ( + check_vowel_harmony + among('a' 'e') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_nA as ( + check_vowel_harmony + among('na' 'ne') + ) + + define mark_DA as ( + check_vowel_harmony + among('da' 'de' 'ta' 'te') + ) + + define mark_ndA as ( + check_vowel_harmony + among('nda' 'nde') + ) + + define mark_DAn as ( + check_vowel_harmony + among('dan' 'den' 'tan' 'ten') + ) + + define mark_ndAn as ( + check_vowel_harmony + among('ndan' 'nden') + ) + + define mark_ylA as ( + check_vowel_harmony + among('la' 'le') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_ki as ( + 'ki' + ) + + define mark_ncA as ( + check_vowel_harmony + among('ca' 'ce') + (mark_suffix_with_optional_n_consonant) + ) + + define mark_yUm as ( + check_vowel_harmony + among ('{i'}m' 'im' 'um' '{u"}m') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_sUn as ( + check_vowel_harmony + among ('s{i'}n' 'sin' 'sun' 's{u"}n' ) + ) + + define mark_yUz as ( + check_vowel_harmony + among ('{i'}z' 'iz' 'uz' '{u"}z') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_sUnUz as ( + among ('s{i'}n{i'}z' 'siniz' 'sunuz' 's{u"}n{u"}z') + ) + + define mark_lAr as ( + check_vowel_harmony + among ('ler' 'lar') + ) + + define mark_nUz as ( + check_vowel_harmony + among ('n{i'}z' 'niz' 'nuz' 'n{u"}z') + ) + + define mark_DUr as ( + check_vowel_harmony + among ('t{i'}r' 'tir' 'tur' 't{u"}r' 'd{i'}r' 'dir' 'dur' 'd{u"}r') + ) + + define mark_cAsInA as ( + among ('cas{i'}na' 'cesine') + ) + + define mark_yDU as ( + check_vowel_harmony + among ('t{i'}m' 'tim' 'tum' 't{u"}m' 'd{i'}m' 'dim' 'dum' 'd{u"}m' + 't{i'}n' 'tin' 'tun' 't{u"}n' 'd{i'}n' 'din' 'dun' 'd{u"}n' + 't{i'}k' 'tik' 'tuk' 't{u"}k' 'd{i'}k' 'dik' 'duk' 'd{u"}k' + 't{i'}' 'ti' 'tu' 't{u"}' 'd{i'}' 'di' 'du' 'd{u"}') + (mark_suffix_with_optional_y_consonant) + ) + + // does not fully obey vowel harmony + define mark_ysA as ( + among ('sam' 'san' 'sak' 'sem' 'sen' 'sek' 'sa' 'se') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_ymUs_ as ( + check_vowel_harmony + among ('m{i'}{s,}' 'mi{s,}' 'mu{s,}' 'm{u"}{s,}') + (mark_suffix_with_optional_y_consonant) + ) + + define mark_yken as ( + 'ken' (mark_suffix_with_optional_y_consonant) + ) + + define stem_nominal_verb_suffixes as ( + [ + set continue_stemming_noun_suffixes + (mark_ymUs_ or mark_yDU or mark_ysA or mark_yken) + or + (mark_cAsInA (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_) + or + ( + mark_lAr ] delete try([(mark_DUr or mark_yDU or mark_ysA or mark_ymUs_)) + unset continue_stemming_noun_suffixes + ) + or + (mark_nUz (mark_yDU or mark_ysA)) + or + ((mark_sUnUz or mark_yUz or mark_sUn or mark_yUm) ] delete try([ mark_ymUs_)) + or + (mark_DUr ] delete try([ (mark_sUnUz or mark_lAr or mark_yUm or mark_sUn or mark_yUz or true) mark_ymUs_)) + ]delete + ) + + // stems noun suffix chains ending with -ki + define stem_suffix_chain_before_ki as ( + [ + mark_ki + ( + (mark_DA] delete try([ + (mark_lAr] delete try(stem_suffix_chain_before_ki)) + or + (mark_possessives] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + + )) + or + (mark_nUn] delete try([ + (mark_lArI] delete) + or + ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + or + (stem_suffix_chain_before_ki) + )) + or + (mark_ndA ( + (mark_lArI] delete) + or + ((mark_sU] delete try([mark_lAr]delete stem_suffix_chain_before_ki))) + or + (stem_suffix_chain_before_ki) + )) + ) + ) + + define stem_noun_suffixes as ( + ([mark_lAr] delete try(stem_suffix_chain_before_ki)) + or + ([mark_ncA] delete + try( + ([mark_lArI] delete) + or + ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + or + ([mark_lAr] delete stem_suffix_chain_before_ki) + ) + ) + or + ([(mark_ndA or mark_nA) + ( + (mark_lArI] delete) + or + (mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + or + (stem_suffix_chain_before_ki) + ) + ) + or + ([(mark_ndAn or mark_nU) ((mark_sU ] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) or (mark_lArI))) + or + ( [mark_DAn] delete try ([ + ( + (mark_possessives ] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + or + (mark_lAr] delete try(stem_suffix_chain_before_ki)) + or + (stem_suffix_chain_before_ki) + )) + ) + or + ([mark_nUn or mark_ylA] delete + try( + ([mark_lAr] delete stem_suffix_chain_before_ki) + or + ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + or + stem_suffix_chain_before_ki + ) + ) + or + ([mark_lArI] delete) + or + (stem_suffix_chain_before_ki) + or + ([mark_DA or mark_yU or mark_yA] delete try([((mark_possessives] delete try([mark_lAr)) or mark_lAr) ] delete [ stem_suffix_chain_before_ki)) + or + ([mark_possessives or mark_sU] delete try([mark_lAr] delete stem_suffix_chain_before_ki)) + ) + + define post_process_last_consonants as ( + [substring] among ( + 'b' (<- 'p') + 'c' (<- '{c,}') + 'd' (<- 't') + '{g~}' (<- 'k') + ) + ) + + // after stemming if the word ends with 'd' or 'g' most probably last U is overstemmed + // like in 'kedim' -> 'ked' + // Turkish words don't usually end with 'd' or 'g' + // some very well known words are ignored (like 'ad' 'soyad' + // appends U to stems ending with d or g, decides which vowel to add + // based on the last vowel in the stem + define append_U_to_stems_ending_with_d_or_g as ( + test('d' or 'g') + (test((goto vowel) 'a' or '{i'}') <+ '{i'}') + or + (test((goto vowel) 'e' or 'i') <+ 'i') + or + (test((goto vowel) 'o' or 'u') <+ 'u') + or + (test((goto vowel) '{o"}' or '{u"}') <+ '{u"}') + ) + + define is_reserved_word as ( + 'ad' try 'soy' atlimit + ) +) + +// Tests if there are more than one syllables +// In Turkish each vowel indicates a distinct syllable +define more_than_one_syllable_word as ( + test (atleast 2 (gopast vowel)) +) + +define postlude as ( + backwards ( + not(is_reserved_word) + do append_U_to_stems_ending_with_d_or_g + do post_process_last_consonants + + ) +) + +define stem as ( + (more_than_one_syllable_word) + ( + backwards ( + do stem_nominal_verb_suffixes + continue_stemming_noun_suffixes + do stem_noun_suffixes + ) + + postlude + ) +) |