需要解决的问题？
txt在线文档下载
分词
创建text
根据内容定义开始与结尾
HTML下载
HTML解析
过滤无关内容
读取本地文件
正则表达式
查找ed结尾的词汇
字谜：8个字母，第3个字母是j，第6个字母是t
9宫格输入判断
正则表达式中的+
提取字符块
查找词干
搜索已分词文本
规范化文本
词干提取器
词性归并
分割
链表与字符串
字符串与格式
排列
--------------------------------------------------------------------------------------------------------------------------
# coding: utf-8
# In[1]:
from __future__ import division # Python 2 users only
import nltk, re, pprint
from nltk import word_tokenize
from nltk.data import PathPointer, ZipFilePathPointer, find
# In[2]:
#txt在线文档下载
from urllib import urlopen
url = "http://www.gutenberg.org/files/2554/2554.txt"
response = urlopen(url)
raw = response.read().decode('utf8')
print type(raw)
print len(raw)
print raw[:75]
# In[3]:
#分词
tokens = word_tokenize(raw)
print type(tokens)
print len(tokens)
print tokens[:10]
# In[4]:
#创建text
text = nltk.Text(tokens)
print type(text)
print text[1024:1062]
print text.collocations()
# In[10]:
#根据内容定义开始与结尾
print raw.find("PART I")
print raw.rfind("End of Project Gutenberg's Crime")
raw = raw[5338:1157746]
# raw=raw[raw.find("PART I"):raw.rfind("End of Project Gutenberg's Crime")]
print raw.find("PART I")
# In[5]:
#HTML下载
url = "http://news.bbc.co.uk/2/hi/health/2284783.stm"
html = urlopen(url).read().decode('utf8')
html[:60]
# In[13]:
print html
# In[7]:
#HTML解析
from bs4 import BeautifulSoup
raw = BeautifulSoup(html,'lxml').get_text()
tokens = word_tokenize(raw)
tokens
# In[35]:
bs = BeautifulSoup(html,'lxml')
print bs.find("div",class_='bodytext').get_text()
# In[8]:
#过滤无关内容
tokens = tokens[110:390]
text = nltk.Text(tokens)
text.concordance('gene')
# In[9]:
print text
# In[14]:
#读取本地文件
f = open('document.txt')
# In[15]:
f = open('d:/data/document.txt')
f.read()
# In[16]:
f = open('d:/data/document.txt','rU')
for line in f:
print(line.strip())
# In[13]:
raw = open('d:/data/document.txt').read()
print type(raw)
tokens = word_tokenize(raw)
print type(tokens)
words = [w.lower() for w in tokens]
print type(words)
vocab = sorted(set(words))
print type(vocab)
# In[28]:
vocab.append('blog')
raw.append('blog')
# In[29]:
query = 'Who knows?'
beatles = ['john', 'paul', 'george', 'ringo']
query + beatles
# In[24]:
#Unicode字符
path = nltk.data.find('corpora/unicode_samples/polish-lat2.txt')
f= path.open(encoding='latin2')
for line in f:
line = line.strip()
print(line)
# In[46]:
f= path.open()
for line in f:
line = line.strip()
print(line)
# In[47]:
ord('a')
# In[48]:
a=u'\u0061'
a
# In[49]:
print a
# In[18]:
ord(u'ń')
# In[20]:
nacute =u'\u0144'
nacute
# In[21]:
nacute.encode('utf8')
# In[22]:
print nacute.encode('utf8')
# In[25]:
import unicodedata
lines = path.open( encoding='latin2').readlines()
line = lines[2]
print(line.encode('unicode_escape'))
for c in line:
if ord(c) > 127:
print('%s U+%04x %s'% (c.encode('utf8'), ord(c), unicodedata.name(c)))
# In[26]:
line.find('zosta\u0142y')
line = line.lower()
line
# In[27]:
line.encode('unicode_escape')
# In[76]:
import re
m = re.search(u'\u015b\w*', line)
m.group()
# In[77]:
word_tokenize(line)
# In[28]:
#正则表达式
import re
wordlist = [w for w in nltk.corpus.words.words('en') if w.islower()]
# In[79]:
#查找ed结尾的词汇
[w for w in wordlist if re.search('ed$', w)]
# In[29]:
#字谜：8个字母，第3个字母是j，第6个字母是t
[w for w in wordlist if re.search('^..j..t..$', w)]
# In[81]:
#9宫格输入判断
[w for w in wordlist if re.search('^[ghi][mno][jlk][def]$', w)]
# In[30]:
#正则表达式中的+
chat_words = sorted(set(w for w in nltk.corpus.nps_chat.words()))
[w for w in chat_words if re.search('^m+i+n+e+$', w)]
# In[31]:
[w for w in chat_words if re.search('^[ha]+$', w)]
# In[32]:
wsj = sorted(set(nltk.corpus.treebank.words()))
[w for w in wsj if re.search('^[0-9]+\.[0-9]+$', w)]
# In[84]:
[w for w in wsj if re.search(r'^[A-Z]+\$$', w)]
# In[33]:
[w for w in wsj if re.search('^[0-9]{4}$', w)]
# In[34]:
[w for w in wsj if re.search('^[0-9]+-[a-z]{3,5}$', w)]
# In[87]:
[w for w in wsj if re.search('^[a-z]{5,}-[a-z]{2,3}-[a-z]{,6}$', w)]
# In[35]:
[w for w in wsj if re.search('(ed|ing)$', w)]
# In[36]:
#提取字符块
word = 'supercalifragilisticexpialidocious'
print re.findall(r'[aeiou]', word)
print len(re.findall(r'[aeiou]', word))
# In[90]:
wsj = sorted(set(nltk.corpus.treebank.words()))
fd = nltk.FreqDist(vs for word in wsj for vs in re.findall(r'[aeiou]{2,}', word))
fd.most_common(12)
# In[91]:
regexp = r'^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]'
def compress(word):
pieces = re.findall(regexp, word)
return ''.join(pieces)
english_udhr = nltk.corpus.udhr.words('English-Latin1')
print(nltk.tokenwrap(compress(w) for w in english_udhr[:75]))
# In[38]:
rotokas_words = nltk.corpus.toolbox.words('rotokas.dic')
cvs = [cv for w in rotokas_words for cv in re.findall(r'[ptksvr][aeiou]', w)]
cfd = nltk.ConditionalFreqDist(cvs)
cfd.tabulate()
# In[39]:
cv_word_pairs = [(cv, w) for w in rotokas_words for cv in re.findall(r'[ptksvr][aeiou]', w)]
cv_index = nltk.Index(cv_word_pairs)
print cv_index['su']
print cv_index['po']
# In[40]:
#查找词干
def stem(word):
for suffix in ['ing', 'ly', 'ed', 'ious', 'ies', 'ive', 'es', 's', 'ment']:
if word.endswith(suffix):
return word[:-len(suffix)]
return word
# In[95]:
re.findall(r'^.*(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')
# In[96]:
re.findall(r'^.*(?:ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')
# In[97]:
re.findall(r'^(.*)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')
# In[98]:
re.findall(r'^(.*)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processes')
# In[99]:
re.findall(r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processes')
# In[100]:
re.findall(r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$', 'language')
# In[41]:
def stem(word):
regexp = r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$'
stem, suffix = re.findall(regexp, word)[0]
return stem
raw = """DENNIS: Listen, strange women lying in ponds distributing swords
is no basis for a system of government. Supreme executive power derives from
a mandate from the masses, not from some farcical aquatic ceremony."""
tokens = word_tokenize(raw)
[stem(t) for t in tokens]
# In[102]:
#搜索已分词文本
from nltk.corpus import gutenberg, nps_chat
moby = nltk.Text(gutenberg.words('melville-moby_dick.txt'))
print moby.findall(r"<a> (<.*>) <man>")
chat = nltk.Text(nps_chat.words())
print chat.findall(r"<.*> <.*> <bro>")
print chat.findall(r"<l.*>{3,}")
# In[103]:
from nltk.corpus import brown
hobbies_learned = nltk.Text(brown.words(categories=['hobbies', 'learned']))
hobbies_learned.findall(r"<\w*> <and> <other> <\w*s>")
# In[42]:
###规范化文本###
raw = """DENNIS: Listen, strange women lying in ponds distributing swords
is no basis for a system of government. Supreme executive power derives from
a mandate from the masses, not from some farcical aquatic ceremony."""
tokens = word_tokenize(raw)
# In[43]:
#词干提取器
porter = nltk.PorterStemmer()
lancaster = nltk.LancasterStemmer()
[porter.stem(t) for t in tokens]
# In[45]:
[lancaster.stem(t) for t in tokens]
# In[108]:
#词性归并
wnl = nltk.WordNetLemmatizer()
[wnl.lemmatize(t) for t in tokens]
# In[121]:
####分割####
# In[123]:
len(nltk.corpus.brown.words()) / len(nltk.corpus.brown.sents())
# In[124]:
text = nltk.corpus.gutenberg.raw('chesterton-thursday.txt')
sents = nltk.sent_tokenize(text)
pprint.pprint(sents[79:89])
# In[125]:
#分词
# In[126]:
def segment(text, segs):
words = []
last = 0
for i in range(len(segs)):
if segs[i] == '1':
words.append(text[last:i+1])
last = i+1
words.append(text[last:])
return words
# In[127]:
text = "doyouseethekittyseethedoggydoyoulikethekittylikethedoggy"
seg1 = "0000000000000001000000000010000000000000000100000000000"
seg2 = "0100100100100001001001000010100100010010000100010010000"
segment(text, seg1)
# In[128]:
segment(text, seg2)
# In[134]:
def evaluate(text, segs):
words = segment(text, segs)
text_size = len(words)
lexicon_size = len(' '.join(list(set(words))))
return text_size + lexicon_size
# In[136]:
text = "doyouseethekittyseethedoggydoyoulikethekittylikethedoggy"
seg1 = "0000000000000001000000000010000000000000000100000000000"
seg2 = "0100100100100001001001000010100100010010000100010010000"
seg3 = "0000100100000011001000000110000100010000001100010000001"
print evaluate(text, seg3)
print evaluate(text, seg2)
print evaluate(text, seg1)
# In[50]:
from random import randint
def flip(segs, pos):
return segs[:pos] + str(1-int(segs[pos])) + segs[pos+1:]
def flip_n(segs, n):
for i in range(n):
segs = flip(segs, randint(0, len(segs)-1))
return segs
def anneal(text, segs, iterations, cooling_rate):
temperature = float(len(segs))
while temperature > 0.5:
best_segs, best = segs, evaluate(text, segs)
for i in range(iterations):
guess = flip_n(segs, int(round(temperature,0)))
score = evaluate(text, guess)
if score < best:
best, best_segs = score, guess
score, segs = best, best_segs
temperature = temperature / cooling_rate
print(evaluate(text, segs), segment(text, segs))
print()
return segs
# In[151]:
text = "doyouseethekittyseethedoggydoyoulikethekittylikethedoggy"
seg1 = "0000000000000001000000000010000000000000000100000000000"
anneal(text, seg1, 5000, 1.2)
# In[152]:
####链表与字符串####
# In[155]:
#链表到字符串
silly = ['We', 'called', 'him', 'Tortoise', 'because', 'he', 'taught', 'us', '.']
print ' '.join(silly)
print ';'.join(silly)
print ''.join(silly)
# In[51]:
#字符串与格式
word = 'cat'
sentence = """hello
world"""
print(word)
print(sentence)
# In[52]:
word
# In[159]:
sentence
# In[53]:
fdist = nltk.FreqDist(['dog', 'cat', 'dog', 'cat', 'dog', 'snake', 'dog', 'cat'])
for word in sorted(fdist):
print(word, '->', fdist[word], '; ')
# In[54]:
for word in sorted(fdist):
print '%s->%d;' % (word, fdist[word]),
# In[55]:
'%s->%d;' % ('cat',3)
# In[56]:
'%s->%d;' % 'cat'
# In[57]:
'%s->'%'cat'
# In[58]:
'%d'% 3
# In[59]:
'I want a %s right now' % 'coffee'
# In[60]:
print '%s wants a %s %s'%('Lee', 'sandwich', 'for lunch')
# In[67]:
template = 'Lee wants a %s right now'
menu = ('sandwich', 'spam fritter', 'pancake')
for snack in menu:
print template % snack
# In[69]:
#排列
'%6s' % 'dog'
# In[179]:
'%-6s' % 'dog'
# In[70]:
width = 6
'%-*s' % (width, 'dog')
# In[181]:
count, total = 3205, 9375
"accuracy for %d words: %2.4f%%" % (total, 100 * count / total)
# In[183]:
def tabulate(cfdist, words, categories):
print '%-16s' % 'Category',
for word in words: # column headings
print '%6s' % word,
for category in categories:
print '%-16s' % category, # row heading
for word in words: # for each word
print '%6d' % cfdist[category][word], # print table cell
print # end the row
from nltk.corpus import brown
cfd = nltk.ConditionalFreqDist(
(genre, word)
for genre in brown.categories()
for word in brown.words(categories=genre))
genres = ['news', 'religion', 'hobbies', 'science_fiction', 'romance', 'humor']
modals = ['can', 'could', 'may', 'might', 'must', 'will']
tabulate(cfd, modals, genres)
# In[71]:
output_file = open('output.txt', 'w')
words = set(nltk.corpus.genesis.words('english-kjv.txt'))
for word in sorted(words):
output_file.write(word + "\n")
# In[185]:
len(words)
# In[186]:
str(len(words))
# In[72]:
output_file.write(str(len(words)) + "\n")
output_file.close()
# In[ ]:
