用別人寫的語言 想弄一下英文詞頻分析 運行后出現以上情況怎么辦啊
import sys,re,collections,nltk
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize
# patterns that used to find or/and replace particular chars or words
# to find chars that are not a letter, a blank or a quotation
pat_letter = re.compile(r'[^a-zA-Z \']+')
# to find the 's following the pronouns. re.I is refers to ignore case
pat_is = re.compile("(it|he|she|that|this|there|here)(\'s)", re.I)
# to find the 's following the letters
pat_s = re.compile("(?<=[a-zA-Z])\'s")
# to find the ' following the words ending by s
pat_s2 = re.compile("(?<=s)\'s?")
# to find the abbreviation of not
pat_not = re.compile("(?<=[a-zA-Z])n\'t")
# to find the abbreviation of would
pat_would = re.compile("(?<=[a-zA-Z])\'d")
# to find the abbreviation of will
pat_will = re.compile("(?<=[a-zA-Z])\'ll")
# to find the abbreviation of am
pat_am = re.compile("(?<=[I|i])\'m")
# to find the abbreviation of are
pat_are = re.compile("(?<=[a-zA-Z])\'re")
# to find the abbreviation of have
pat_ve = re.compile("(?<=[a-zA-Z])\'ve")
lmtzr = WordNetLemmatizer()
def get_words(file):
with open (file) as f:
words_box=[]
pat = re.compile(r'[^a-zA-Z \']+')
for line in f:
#if re.match(r'[a-zA-Z]*',line):
# words_box.extend(line.strip().strip('\'\"\.,').lower().split())
# words_box.extend(pat.sub(' ', line).strip().lower().split())
words_box.extend(merge(replace_abbreviations(line).split()))
return collections.Counter(words_box)
def merge(words):
new_words = []
for word in words:
if word:
tag = nltk.pos_tag(word_tokenize(word)) # tag is like [('bigger', 'JJR')]
pos = get_wordnet_pos(tag[0][1])
if pos:
lemmatized_word = lmtzr.lemmatize(word, pos)
new_words.append(lemmatized_word)
else:
new_words.append(word)
return new_words
def get_wordnet_pos(treebank_tag):
if treebank_tag.startswith('J'):
return nltk.corpus.wordnet.ADJ
elif treebank_tag.startswith('V'):
return nltk.corpus.wordnet.VERB
elif treebank_tag.startswith('N'):
return nltk.corpus.wordnet.NOUN
elif treebank_tag.startswith('R'):
return nltk.corpus.wordnet.ADV
else:
return ''
def replace_abbreviations(text):
new_text = text
new_text = pat_letter.sub(' ', text).strip().lower()
new_text = pat_is.sub(r"\1 is", new_text)
new_text = pat_s.sub("", new_text)
new_text = pat_s2.sub("", new_text)
new_text = pat_not.sub(" not", new_text)
new_text = pat_would.sub(" would", new_text)
new_text = pat_will.sub(" will", new_text)
new_text = pat_am.sub(" am", new_text)
new_text = pat_are.sub(" are", new_text)
new_text = pat_ve.sub(" have", new_text)
new_text = new_text.replace('\'', ' ')
return new_text
def append_ext(words):
new_words = []
for item in words:
word, count = item
tag = nltk.pos_tag(word_tokenize(word))[0][1] # tag is like [('bigger', 'JJR')]
new_words.append((word, count, tag))
return new_words
def write_to_file(words, file='results.txt'):
f = open(file, 'w')
for item in words:
for field in item:
f.write(str(field)+',')
f.write('\n')
print('results.txt')
if __name__=='__main__':
book = sys.argv[1]
print ('counting...')
words = get_words(book)
print ('writing file...')
write_to_file(append_ext(words.most_common()))
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/267210.html
上一篇：如何獲取p 站鏈接？