机器翻译词典合并
elisa-patch的Python项目详细描述
依赖关系
unidecode emoji *kenlm fuzzy scikit-learn pyxdameraulevenshtein pygtrie numpy
*从github安装kenlm包装器：
pip install https://github.com/kpu/kenlm/archive/master.zip
用法
# Load english dictionaryenglish_vocab=load_english_vocab(...)english_vocab.update(load_english_vocab(...))# Load bilingual lexicon dictionaryforeign_dict=load_lexicon_norm(...)# Load target language modellm=kenlm.Model(...)# Train a ngram model if needed# ngram_train(foreign_dict, 'hin-tfidf-ngram_algo')# Ulf's romanizerromanizer=partial(romanize,romanization_path=...,language_code="hin")soundex_inst=fuzzy.DMetaphone()soundex_algo=lambdax:soundex_inst(x)[0].decode('utf-8')ifsoundex_inst(x)[0]isnotNoneelsexenglish_encoded_vocab={e:soundex_algo(e)foreinenglish_vocabife}# load the ngram modelngram_algo=pickle.loads(open(...,"rb").read())soundex_model=partial(soundex_similarity,encoded_english_vocab=english_encoded_vocab,romanizer=romanizer,soundex=soundex_algo)lev_model=partial(lev_similarity,backup=soundex_model)ngram_model=partial(ngram_similarity,model=ngram_algo,backup=lev_model)final_model=partial(exact_similarity,backup=ngram_model)forlineinopen(...):source,target=line.strip('\n').split('\t')oovs=extract_oov(target,source,english_vocab=english_vocab,romanization=True)best,mods=translate_oov(target,oovs,foreign_dict,final_model,lm.score)ifbest!=target:foroovinoovs:alt=list(mods[oov].keys())[0]trans=mods[oov][alt]debug.debug(f"{romanizer(oov)} -> {romanizer(alt)} : {list(trans)}")debug.debug(best)debug.debug("*"*100)
或
python -m elisa_patch --help
欢迎加入QQ群-->： 979659372
