3 e_H@sddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z m Z ddZddZd d Zd d Zd dZddZddZddZddZddZddZddZddZdd ZdS)!N) word_tokenize sent_tokenize) stopwords)LancasterStemmerWordNetLemmatizercCstj|}|S)z-Convert text into list of tokenized sentences)nltkr)text sentencesr L/var/www/html/talenthub/backend/ai/resume-to-job-match/text_preprocessing.pytext_to_sentence_tokens s r cCstj|}|S)z)Convert text into list of tokenized words)rr)rwordsr r r text_to_words_tokenss rcCs t|}|S)z0Convert string text into list of tokenized words)r)rr r r r text_to_wordssrc Csg}x|D]}ddiddiddiddidd id d id d idd idd ig }x8|D]0}x*|jD]\}}tj|}|j||}q\WqNW|j}|jdd}|j|q W|S)z/Clean special tags from list of tokenized wordsz>\s+>z\s+ z\s*\s* z\s*z\s*z z.*<\s*(/head|body)[^>]*>z]*>.*z\1z[ \t]*<[^<]*?/?>z^\s+z\n)itemsrecompilesubrstripreplaceappend) r new_wordswordZrulesrulekvregexnew_wordr r r remove_special_tagss&     r"cCs<g}x2|D]*}tjd|jddjdd}|j|q W|S)z8Remove non-ASCII characters from list of tokenized wordsZNFKDasciiignorezutf-8) unicodedata normalizeencodedecoder)r rrr!r r r remove_non_ascii8s  r)cCs0g}x&|D]}tjj|dd}|j|q W|S)z.Remove apostrophe from list of tokenized words'r)npcharrr)r rrr!r r r remove_apostrophe@s  r-cCs(g}x|D]}|j}|j|q W|S)z@Convert all characters to lowercase from list of tokenized words)lowerr)r rrr!r r r to_lowercaseIs  r/cCs6g}x,|D]$}tjdd|}|dkr |j|q W|S)z/Remove punctuation from list of tokenized wordsz[^\w\s]r)rrr)r rrr!r r r remove_punctuationQs  r0cCs~g}xt|D]l}d}x8tt|D](}tjj|||d}tjj|dd}q Wtjj|dd}tjj|dd}|j|q W|S)z6Remove special characters from list of tokenized wordsz!"#$%&()*+-./:;<=>?@[\]^_`{|}~ rz ,rr*)rangelenr+r,rr)r rrZsymbolsir!r r r remove_special_chars\s r5cCsFtj}g}x4|D],}|jr4|j|}|j|q|j|qW|S)zWReplace all interger occurrences in list of tokenized words with textual representation)inflectengineisdigitZnumber_to_wordsr)r prrr!r r r replace_numbersms   r:cCs.g}x$|D]}|tjdkr |j|q W|S)z.Remove stop words from list of tokenized wordsenglish)rr r)r rrr r r remove_stopwordsys  r<cCs0t}g}x |D]}|j|}|j|qW|S)z%Stem words in list of tokenized words)rstemr)r ZstemmerZstemsrr=r r r stemming_wordss   r>cCs4t}g}x$|D]}|j|dd}|j|qW|S)z*Lemmatize verbs in list of tokenized wordsr)pos)r lemmatizer)r Z lemmatizerlemmasrlemmar r r lemmatize_verbss  rCcCsDt|}t|}t|}t|}t|}t|}t|}t|}|S)z1 preprocess the text into list of tokenized words)rr)r/r0r:r<r>rC)rZ words_tokensr r r preprocess_textsrD)rstringr%numpyr+rr6rrZ nltk.corpusrZ nltk.stemrrr rrr"r)r-r/r0r5r:r<r>rCrDr r r r s(