from__future__importdivision
importnltk
fromnltk.probabilityimportFreqDist
fromnltk.corpusimportbrown, inaugural, stopwords,swadesh, wordnetaswn, state_union,names,gutenberg
#from nltk.book import *
?4
# print(state_union.fileids())
# cfd=nltk.ConditionalFreqDist((words, fileid[:4])
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for fileid in state_union.fileids()
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for words in state_union.words(fileid)
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? )
# words=['men','women','people']
# print(cfd.plot(conditions=words))
8
# cfd=nltk.ConditionalFreqDist((fileid,name.lower()[0])
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for fileid in names.fileids()
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for name in names.words(fileid))
#
# print(cfd.plot())
15
# fd=nltk.FreqDist(brown.words())
# more_than_tree_times=[word for word in fd if fd[word]>=3]
16
# def word_diversity(words):
#? ? num_word=len(words)
#? ? num_vocab=len(set([word.lower()for word in words]))
#? ? word_diversity=int(num_word/num_vocab)
#? ? return word_diversity
#
# def word_diversity_categories():
#? ? for categorie in brown.categories():
#? ? ? ? word_diversity_categories = word_diversity(brown.words(categories=categorie))
#? ? ? ? print(categorie,':', word_diversity_categories)
#
# word_diversity_categories()
17
# stopwords=stopwords.words('english')
# def frequentest_words(text):
#? ? words=[word for word in text if word.lower() not in stopwords and word.isalpha()]
#? ? cfd=nltk.FreqDist(words)
#? ? frequentest_words = sorted(cfd.items(),key=lambda item:item[1], reverse=True)
#? ? print (frequentest_words[:50])
#
# frequentest_words(gutenberg.words('austen-emma.txt'))
19
# cfd=nltk.ConditionalFreqDist((genre,word)
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for genre in brown.categories()
#? ? ? ? ? ? ? ? ? ? ? ? ? ? ? for word in brown.words(categories=genre))
# for genre in brown.categories():
#? ? fdist=cfd[genre]
#? ? sorted_words=sorted(fdist.keys(),key=lambda x:fdist[x],reverse=True)
#? ? print (type(fdist), genre, sorted_words[:10], sorted_words[-10::])
