第30页 第三章: 发现群组(单词向量)

需求如下:
1. 统计指定范围的词汇(在10%~50%博客中出现的词汇), 在博客中出现的次数

首先, 获取博客的所有单词统计数:
import feedparser
import re

def getwordcounts(url):
    # 解析订阅源
    d = feedparser.parse(url)
    wc = {}
    
    for e in d.entries:
        if 'summary' in e:
            summary = e.summary
        else:
            summary = e.description
        
        # 提取一个单词列表
        words = getwords(e.title + ' ' + summary)
        for word in words:
            wc.setdefault(word, 0)
            wc[word] += 1
    return d.feed.title, wc
    
def getwords(html):
    # 去除所有html标记
    txt = re.compile(r'<[^>]+>').sub('', html)
    # 利用所有非字母字符拆分出单词
    words = re.compile(r'[^a-z^A-Z]+').split(txt)
    return [word.lower() for word in words if word != '']

然后, 我们统计每篇博客的单词统计, 以及出现这些单词的博客数目:
# 存储出现单词的博客数目
apcount = {}
wordcounts = {}
path = '/Users/lgt/PCI/PCI_Code/chapter3'
feedlist = [line for line in open(path + '/feedlist.txt')]
for feedurl in feedlist:
    title, wc = getwordcounts(feedurl)
    wordcounts[title] = wc
    for word, count in wc.items():
        apcount.setdefault(word, 0)
        if count > 1:
            apcount[word] += 1

我们要保证统计的单词出现在10%~50%的博客中:
wordlist = []
for w, bc in apcount.items():
    frac = float(bc) / len(feedlist)
    if frac > 0.1 and frac < 0.5:
        wordlist.append(w)

最后, 记录针对每个博客的所有单词的统计情况:
out = open('blogdata.txt', 'w')
out.write('Blog')
for word in wordlist:
    out.write('\t%s' % word)
out.write('\n')
for blog, wc in wordcounts.items():
    out.write(blog)
    for word in wordlist:
        if word in wc:
            out.write('\t%d' % wc[word])
        else:
            out.write('\t0')
    out.write('\n')