opensextant.advas_phonetics

  1# -*- coding: utf-8 -*-
  2# ----------------------------------------------------------
  3# AdvaS Advanced Search 
  4# module for phonetic algorithms
  5#
  6# (C) 2002 - 2005 Frank Hofmann, Chemnitz, Germany
  7# email fh@efho.de
  8# ----------------------------------------------------------
  9#
 10# changed 2005-01-24
 11# 2012-01-01 MU adapted to support various Unicode transliterations in Metaphone
 12# 2021-03-25 MU migrated to Xponents here
 13
 14import re
 15
 16
 17def soundex(term):
 18    """Return the soundex value to a string argument."""
 19
 20    # Create and compare soundex codes of English words.
 21    #
 22    # Soundex is an algorithm that hashes English strings into
 23    # alpha-numerical value that represents what the word sounds
 24    # like. For more information on soundex and some notes on the
 25    # differences in implemenations visit:
 26    # http://www.bluepoof.com/Soundex/info.html
 27    #
 28    # This version modified by Nathan Heagy at Front Logic Inc., to be
 29    # compatible with php's soundexing and much faster.
 30    #
 31    # eAndroid / Nathan Heagy / Jul 29 2000
 32    # changes by Frank Hofmann / Jan 02 2005
 33
 34    # generate translation table only once. used to translate into soundex numbers
 35    # table = string.maketrans('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ', '0123012002245501262301020201230120022455012623010202')
 36    table = "".maketrans('ABCDEFGHIJKLMNOPQRSTUVWXYZ', '01230120022455012623010202')
 37
 38    # check parameter
 39    if not term:
 40        return "0000"  # could be Z000 for compatibility with other implementations
 41    # end if
 42
 43    # convert into uppercase letters
 44    term = term.upper()
 45    first_char = term[0]
 46
 47    # translate the string into soundex code according to the table above
 48    term = term[1:].translate(table)
 49
 50    # remove all 0s
 51    term = term.replace("0", "")
 52
 53    # remove duplicate numbers in-a-row
 54    str2 = first_char
 55    for x in term:
 56        if x != str2[-1]:
 57            str2 = str2 + x
 58    # end if
 59    # end for
 60
 61    # pad with zeros
 62    str2 = str2 + "0" * len(str2)
 63
 64    # take the first four letters
 65    return_value = str2[:4]
 66
 67    # return value
 68    return return_value
 69
 70
 71# MCU: optimization -- put constant tables in global space, not functional space.
 72#
 73# build translation table
 74metaphone_table = {
 75    "ae": "e",
 76    "gn": "n",
 77    # "kn":"n",  -- Generalization for 'known' or 'knowles' => 'nwn' or 'nwls'; But not "Ken" or "Kane"=> "kn"
 78    "pn": "n",
 79    "wr": "r",
 80    "wh": "w"}
 81
 82# define standard translation table
 83metaphone_std_trans = {
 84    "b": "b",
 85    "c": "k",
 86    "d": "t",
 87    "g": "k",
 88    "h": "h",
 89    "k": "k",
 90    "p": "p",
 91    "n": "n",
 92    "q": "k",
 93    "s": "s",
 94    "t": "t",
 95    "v": "f",
 96    "w": "w",
 97    "x": "ks",
 98    "y": "y",
 99    "z": "s"}
100
101EMPTY_STRING = ''
102
103# cStringIO is about 5% slower than normal + operation for short strings.
104# from cStringIO import StringIO
105
106re_sub_nonalpha = re.compile(u'[^a-zḑḩñşţz̧]')
107re_sub_vowels = re.compile(u'[aeiou]')
108vowels = {'a', 'e', 'i', 'o', 'u'}
109re_CI = re.compile('c[iey]')
110re_SCI = re.compile('sc[iey]')
111re_DG = re.compile('dg[eyi]')
112re_GHvowel = re.compile('gh[aeiouy]')
113# vowels of various Latin forms, preceeding H, where H is followed by consonant.
114re_vowelHvowel = re.compile(u'[aeiouyāīū][hḩ][^aeiouy]')
115re_softH = re.compile('[csptg]h')
116re_SIvowel = re.compile('si[ao]')
117re_TIvowel = re.compile('ti[ao]')
118re_shadowedW = re.compile('w[^aeiouy]')
119
120
121def metaphone(text):
122    """returns metaphone code for a given string"""
123
124    # implementation of the original algorithm from Lawrence Philips
125    # extended/rewritten by M. Kuhn
126    # improvements with thanks to John Machin <sjmachin@lexicon.net>
127    #
128    # 2011-FEB
129    # a) substantial perf improve by Marc Ubaldino <ubaldino@mitre.org> -- put regex in global space for lib. 2.5x faster
130    # b) qualitative fixes:  vowel replacements occur as last step.
131    # c) looking at oddball extended latin, e.g., ḑ,
132    # d) improved repeated chars,  pizza, fayyad, etc.
133
134    # i = 0
135    if not text:
136        # empty string ?
137        return EMPTY_STRING
138    # end if
139
140    # extension #1 (added 2005-01-28)
141    # convert to lowercase
142    term = text.lower()
143
144    # extension #2 (added 2005-01-28)
145    # remove all non-english characters, first
146    term = re_sub_nonalpha.sub('', term)
147    if len(term) == 0:
148        # nothing left
149        return EMPTY_STRING
150    # end if
151
152    # extension #3 (added 2005-01-24)
153    # conflate repeated letters
154    firstChar = term[0]
155    str2 = firstChar
156    for x in term:
157        if x != str2[-1]:
158            str2 = str2 + x
159    # end if
160    # end for
161
162    textnorm = str2
163    # text = str2
164
165    # term = str3
166    term_length = len(textnorm)
167    if term_length == 0:
168        # nothing left
169        return EMPTY_STRING
170    # end if
171
172    # define return value
173    code = ''
174    term = textnorm
175
176    # check for exceptions
177    if (term_length > 1):
178        # get first two characters
179        first_chars = term[0:2]
180
181        kn_start = textnorm.startswith('kn')
182        if first_chars in metaphone_table.keys() or kn_start:
183            term = term[2:]
184            if kn_start:
185                code = 'n'
186            else:
187                code = metaphone_table[first_chars]
188            term_length = len(term)
189    # end if
190
191    elif (term[0] == "x"):
192        term = ""
193        code = "s"
194        term_length = 0
195    # end if
196
197    i = 0
198    while (i < term_length):
199        # init character to add, init basic patterns
200        add_char = ""
201        part_n_2 = ""
202        part_n_3 = ""
203        part_n_4 = ""
204        part_c_2 = ""
205        part_c_3 = ""
206
207        # extract a number of patterns, if possible
208        if (i < (term_length - 1)):
209            part_n_2 = term[i:i + 2]
210
211            if (i > 0):
212                part_c_2 = term[i - 1:i + 1]
213                part_c_3 = term[i - 1:i + 2]
214        # end if
215        # end if
216
217        if (i < (term_length - 2)):
218            part_n_3 = term[i:i + 3]
219        # end if
220
221        if (i < (term_length - 3)):
222            part_n_4 = term[i:i + 4]
223        # end if
224
225        ch = term[i]
226
227        # use table with conditions for translations
228        if (ch == "b"):
229            add_char = metaphone_std_trans["b"]
230            if (i == (term_length - 1)):
231                if (i > 0):
232                    if (term[i - 1] == "m"):
233                        add_char = ""
234                # end if
235            # end if
236        # end if
237        elif (ch == "c"):
238            add_char = metaphone_std_trans["c"]
239            if (part_n_2 == "ch"):
240                add_char = "x"
241            elif re_CI.search(part_n_2):
242                add_char = "s"
243            # end if
244
245            if (part_n_3 == "cia"):
246                add_char = "x"
247            # end if
248
249            if re_SCI.search(part_c_3):
250                add_char = ""
251        # end if
252
253        elif (ch == "d" or ch == u'ḑ'):
254            add_char = metaphone_std_trans["d"]
255            if (re_DG.search(part_n_3)):
256                add_char = "j"
257        # end if
258
259        elif (ch == "g"):
260            add_char = metaphone_std_trans["g"]
261
262            if (part_n_2 == "gh"):
263                if (i == (term_length - 2)):
264                    add_char = ""
265            # end if
266            elif (re_GHvowel.search(part_n_3)):
267                add_char = ""
268            elif (part_n_2 == "gn"):
269                add_char = ""
270            elif (part_n_4 == "gned"):
271                add_char = ""
272            elif re_DG.search(part_c_3):
273                add_char = ""
274            elif (part_n_2 == "gi"):
275                if (part_c_3 != "ggi"):
276                    add_char = "j"
277            # end if
278            elif (part_n_2 == "ge"):
279                if (part_c_3 != "gge"):
280                    add_char = "j"
281            # end if
282            elif (part_n_2 == "gy"):
283                if (part_c_3 != "ggy"):
284                    add_char = "j"
285            # end if
286            elif (part_n_2 == "gg"):
287                add_char = ""
288        # end if
289        elif (ch == "h" or ch == u'ḩ'):
290            add_char = metaphone_std_trans["h"]
291            if (re_vowelHvowel.search(part_c_3)):
292                add_char = ""
293            elif (re_softH.search(part_c_2)):
294                add_char = ""
295        # end if
296        elif (ch == "k"):
297            add_char = metaphone_std_trans["k"]
298            if (part_c_2 == "ck"):
299                add_char = ""
300        # end if
301        elif (ch == "p"):
302            add_char = metaphone_std_trans["p"]
303            if (part_n_2 == "ph"):
304                add_char = "f"
305        # end if
306        elif (ch == "q"):
307            add_char = metaphone_std_trans["q"]
308        elif (ch == "s" or ch == u'ş'):
309            add_char = metaphone_std_trans["s"]
310            if (part_n_2 == "sh"):
311                add_char = "x"
312            # end if
313
314            if re_SIvowel.search(part_n_3):
315                add_char = "x"
316        # end if
317        elif (ch == "t" or ch == u'ţ'):
318            add_char = metaphone_std_trans["t"]
319            if (part_n_2 == "th"):
320                add_char = "0"
321            # end if
322
323            if (re_TIvowel.search(part_n_3)):
324                add_char = "x"
325        # end if
326        elif (ch == "v"):
327            add_char = metaphone_std_trans["v"]
328        elif (ch == "w"):
329            add_char = metaphone_std_trans["w"]
330            if (re_shadowedW.search(part_n_2)):
331                add_char = ""
332        # end if
333        elif (ch == "x"):
334            add_char = metaphone_std_trans["x"]
335        elif (ch == "y"):
336            add_char = metaphone_std_trans["y"]
337        elif (ch == "z" or ch == u'z̧'):
338            add_char = metaphone_std_trans["z"]
339        elif (ch == u'ñ'):
340            add_char = metaphone_std_trans['n']
341        else:
342            # alternative
343            add_char = ch
344        # end if
345
346        if add_char:
347            code = code + add_char
348        i += 1
349    # end while
350
351    # extension #4 (added 2005-01-24)
352    # This was moved from before loop
353    #  "mirance" was coming out as "mrnk"  not "mrns"
354    #  So I refactored and retested all of this.  Vowels are to be stripped out after
355    #  above patterns are run.
356    # remove any vowels unless a vowel is the first letter
357    # firstChar = str2[0]
358    # str3 = firstChar
359    # for x in str2[1:]:
360    #	if x not in vowels:
361    #		str3 = str3 + x
362    # end if
363    # end for
364    # return metaphone code
365
366    c0 = code[0]
367    reduced_code = c0 + re_sub_vowels.sub('', code[1:])
368
369    return reduced_code
370
371
372def nysiis(term):
373    """returns New York State Identification and Intelligence Algorithm (NYSIIS) code for the given term"""
374
375    code = ""
376
377    # i = 0
378    term_length = len(term)
379
380    if (term_length == 0):
381        # empty string ?
382        return code
383    # end if
384
385    # build translation table for the first characters
386    table = {
387        "mac": "mcc",
388        "ph": "ff",
389        "kn": "nn",
390        "pf": "ff",
391        "k": "c",
392        "sch": "sss"
393    }
394
395    table_value_len = 0
396    for table_entry in table.keys():
397        table_value = table[table_entry]  # get table value
398        table_value_len = len(table_value)  # calculate its length
399        first_chars = term[0:table_value_len]
400        if (first_chars == table_entry):
401            term = table_value + term[table_value_len:]
402            break
403    # end if
404    # end for
405
406    # build translation table for the last characters
407    table = {
408        "ee": "y",
409        "ie": "y",
410        "dt": "d",
411        "rt": "d",
412        "rd": "d",
413        "nt": "d",
414        "nd": "d",
415    }
416
417    for table_entry in table.keys():
418        table_value = table[table_entry]  # get table value
419        table_entry_len = len(table_entry)  # calculate its length
420        last_chars = term[(0 - table_entry_len):]
421        # print last_chars, ", ", table_entry, ", ", table_value
422        if (last_chars == table_entry):
423            term = term[:(0 - table_value_len + 1)] + table_value
424            break
425    # end if
426    # end for
427
428    # initialize code
429    code = term
430
431    # transform ev->af
432    code = re.sub(r'ev', r'af', code)
433
434    # transform a,e,i,o,u->a
435    code = re.sub(r'[aeiouy]', r'a', code)
436
437    # transform q->g
438    code = re.sub(r'q', r'g', code)
439
440    # transform z->s
441    code = re.sub(r'z', r's', code)
442
443    # transform m->n
444    code = re.sub(r'm', r'n', code)
445
446    # transform kn->n
447    code = re.sub(r'kn', r'n', code)
448
449    # transform k->c
450    code = re.sub(r'k', r'c', code)
451
452    # transform sch->sss
453    code = re.sub(r'sch', r'sss', code)
454
455    # transform ph->ff
456    code = re.sub(r'ph', r'ff', code)
457
458    # transform h-> if previous or next is nonvowel -> previous
459    occur = re.findall(r'([a-z]{0,1}?)h([a-z]{0,1}?)', code)
460    # print occur
461    for occur_group in occur:
462        occur_item_previous = occur_group[0]
463        occur_item_next = occur_group[1]
464
465        if ((re.match(r'[^aeiouy]', occur_item_previous)) or (re.match(r'[^aeiouy]', occur_item_next))):
466            if (occur_item_previous != ""):
467                # make substitution
468                code = re.sub(occur_item_previous + "h", occur_item_previous * 2, code, 1)
469        # end if
470    # end if
471    # end for
472
473    # transform w-> if previous is vowel -> previous
474    occur = re.findall(r'([aeiouy]{1}?)w', code)
475    # print occur
476    for occur_group in occur:
477        occur_item_previous = occur_group[0]
478        # make substitution
479        code = re.sub(occur_item_previous + "w", occur_item_previous * 2, code, 1)
480    # end for
481
482    # check last character
483    # -s, remove
484    code = re.sub(r's$', r'', code)
485    # -ay, replace by -y
486    code = re.sub(r'ay$', r'y', code)
487    # -a, remove
488    code = re.sub(r'a$', r'', code)
489
490    # return nysiis code
491    return code
492
493
494def caverphone(term):
495    """returns the language key using the caverphone algorithm 2.0"""
496
497    # Developed at the University of Otago, New Zealand.
498    # Project: Caversham Project (http://caversham.otago.ac.nz)
499    # Developer: David Hood, University of Otago, New Zealand
500    # Contact: caversham@otago.ac.nz
501    # Project Technical Paper: http://caversham.otago.ac.nz/files/working/ctp150804.pdf
502    # Version 2.0 (2004-08-15)
503
504    code = ""
505
506    # i = 0
507    term_length = len(term)
508
509    if (term_length == 0):
510        # empty string ?
511        return code
512    # end if
513
514    # convert to lowercase
515    code = term.lower()
516
517    # remove anything not in the standard alphabet (a-z)
518    code = re.sub(r'[^a-z]', '', code)
519
520    # remove final e
521    if code.endswith("e"):
522        code = code[:-1]
523
524    # if the name starts with cough, rough, tough, enough or trough -> cou2f (rou2f, tou2f, enou2f, trough)
525    code = re.sub(r'^([crt]|(en)|(tr))ough', r'\1ou2f', code)
526
527    # if the name starts with gn -> 2n
528    code = re.sub(r'^gn', r'2n', code)
529
530    # if the name ends with mb -> m2
531    code = re.sub(r'mb$', r'm2', code)
532
533    # replace cq -> 2q
534    code = re.sub(r'cq', r'2q', code)
535
536    # replace c[i,e,y] -> s[i,e,y]
537    code = re.sub(r'c([iey])', r's\1', code)
538
539    # replace tch -> 2ch
540    code = re.sub(r'tch', r'2ch', code)
541
542    # replace c,q,x -> k
543    code = re.sub(r'[cqx]', r'k', code)
544
545    # replace v -> f
546    code = re.sub(r'v', r'f', code)
547
548    # replace dg -> 2g
549    code = re.sub(r'dg', r'2g', code)
550
551    # replace ti[o,a] -> si[o,a]
552    code = re.sub(r'ti([oa])', r'si\1', code)
553
554    # replace d -> t
555    code = re.sub(r'd', r't', code)
556
557    # replace ph -> fh
558    code = re.sub(r'ph', r'fh', code)
559
560    # replace b -> p
561    code = re.sub(r'b', r'p', code)
562
563    # replace sh -> s2
564    code = re.sub(r'sh', r's2', code)
565
566    # replace z -> s
567    code = re.sub(r'z', r's', code)
568
569    # replace initial vowel [aeiou] -> A
570    code = re.sub(r'^[aeiou]', r'A', code)
571
572    # replace all other vowels [aeiou] -> 3
573    code = re.sub(r'[aeiou]', r'3', code)
574
575    # replace j -> y
576    code = re.sub(r'j', r'y', code)
577
578    # replace an initial y3 -> Y3
579    code = re.sub(r'^y3', r'Y3', code)
580
581    # replace an initial y -> A
582    code = re.sub(r'^y', r'A', code)
583
584    # replace y -> 3
585    code = re.sub(r'y', r'3', code)
586
587    # replace 3gh3 -> 3kh3
588    code = re.sub(r'3gh3', r'3kh3', code)
589
590    # replace gh -> 22
591    code = re.sub(r'gh', r'22', code)
592
593    # replace g -> k
594    code = re.sub(r'g', r'k', code)
595
596    # replace groups of s,t,p,k,f,m,n by its single, upper-case equivalent
597    for single_letter in ["s", "t", "p", "k", "f", "m", "n"]:
598        otherParts = re.split(single_letter + "+", code)
599        letter = single_letter.upper()
600        code = letter.join(otherParts)
601
602    # replace w[3,h3] by W[3,h3]
603    code = re.sub(r'w(h?3)', r'W\1', code)
604
605    # replace final w with 3
606    code = re.sub(r'w$', r'3', code)
607
608    # replace w -> 2
609    code = re.sub(r'w', r'2', code)
610
611    # replace h at the beginning with an A
612    code = re.sub(r'^h', r'A', code)
613
614    # replace all other occurrences of h with a 2
615    code = re.sub(r'h', r'2', code)
616
617    # replace r3 with R3
618    code = re.sub(r'r3', r'R3', code)
619
620    # replace final r -> 3
621    code = re.sub(r'r$', r'3', code)
622
623    # replace r with 2
624    code = re.sub(r'r', r'2', code)
625
626    # replace l3 with L3
627    code = re.sub(r'l3', r'L3', code)
628
629    # replace final l -> 3
630    code = re.sub(r'l$', r'3', code)
631
632    # replace l with 2
633    code = re.sub(r'l', r'2', code)
634
635    # remove all 2's
636    code = re.sub(r'2', r'', code)
637
638    # replace the final 3 -> A
639    code = re.sub(r'3$', r'A', code)
640
641    # remove all 3's
642    code = re.sub(r'3', r'', code)
643
644    # extend the code by 10 '1' (one)
645    code += '1' * 10
646
647    # take the first 10 characters
648    caverphoneCode = code[:10]
649
650    # return caverphone code
651    return caverphoneCode