python 中文词频统计 python实现简单中文词频统计示例

import codecs
import matplotlib.pyplot as plt
from pylab import mpl
mpl.rcParams['font.sans-serif'] = ['FangSong'] # 指定默认字体
mpl.rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'-'显示为方块的问题

word = []
counter = {}

with codecs.open('data.txt') as fr:
 for line in fr:
  line = line.strip()
  if len(line) == 0:
   continue
  for w in line:
   if not w in word:
    word.append(w)
   if not w in counter:
    counter[w] = 0
   else:
    counter[w] += 1

counter_list = sorted(counter.items(), key=lambda x: x[1], reverse=True)

print(counter_list[:50])

label = list(map(lambda x: x[0], counter_list[:50]))
value = list(map(lambda y: y[1], counter_list[:50]))

plt.bar(range(len(value)), value, tick_label=label)
plt.show()

import sys 
reload(sys) 
sys.setdefaultencoding("utf-8") 
 
import jieba 
import jieba.analyse 
 
wf = open('clean_title.txt','w+') 
for line in open('/root/clean_data/clean_data.csv'): 
 
  item = line.strip('\n\r').split('\t') //制表格切分 
  # print item[1] 
  tags = jieba.analyse.extract_tags(item[1]) //jieba分词 
  tagsw = ",".join(tags) //逗号连接切分的词 
  wf.write(tagsw) 
 
wf.close() 

#!/usr/bin/python 
# -*- coding:utf-8 -*- 
 
word_lst = [] 
word_dict= {} 
with open('/root/clean_data/clean_title.txt') as wf,open("word.txt",'w') as wf2: //打开文件 
 
  for word in wf: 
    word_lst.append(word.split(',')) //使用逗号进行切分 
    for item in word_lst: 
       for item2 in item: 
        if item2 not in word_dict: //统计数量 
          word_dict[item2] = 1 
        else: 
          word_dict[item2] += 1 
 
  for key in word_dict: 
    print key,word_dict[key] 
    wf2.write(key+' '+str(word_dict[key])+'\n') //写入文档