利用pandas进行大文件计数处理 利用pandas进行大文件计数处理的方法

import pandas as pd
data = pd.read_table('filename.txt', iterator=True)
chunk = data.get_chunk(5) 

Out[4]: 
332.977889999979 -0.0164794921875 
0 332.97790 -0.022278 
1 332.97791 -0.026855 
2 332.97792 -0.030518 
3 332.97793 -0.045776 
4 332.97794 -0.032654

import pandas as pd
data = pd.read_table('filename.txt', iterator=True， names=['time', 'energe'])
chunk = data.get_chunk(5) 
data['energe'] = df['energe'].astype('int')

DataFrame(data, columns=[‘year', ‘month', ‘day'], 
index=[‘one', ‘two', ‘three'])

series = data.ix[0]
data - series

data.sub(series1,axis=0) 

start = time.time()
data = pd.read_table('Eu155_Na22_K40_MR_0CM_3Min.csv', names=['time', 'energe'])
end = time.time()
data.index
print("The time is %f s" % (end - start))
plus = data['energe']
plus[plus < 0] = 0

The time is 29.403917 s 
RangeIndex(start=0, stop=68319232, step=1)

import pandas as pd


def split_file(filename, size):
 name = filename.split('.')[0]
 data = pd.read_table(filename, chunksize=size, names=['time', 'intension'])
 i = 1
 for piece in data:
 outname = name + str(i) + '.csv'
 piece.to_csv(outname, index=False, names = ['time', 'intension'])
 i += 1

def split_csvfile(filename, size):
 name = filename.split('.')[0]
 data = pd.read_csv(filename, chunksize=size, names=['time', 'intension'])
 i = 1
 for piece in data:
 outname = name + str(i) + '.csv'
 piece = piece['intension']
 piece.to_csv(outname, index=False)
 i += 1

rom scipy import signal
import os
import time
import pandas as pd
import numpy as np
from multiprocessing import Pool
import matplotlib.pylab as plt
from functools import partial


def findpeak(pluse):
 pluse[pluse < 0.05] = 0
 print('Sub process %s.' % os.getpid())
 start = time.time()
 peaks = signal.find_peaks_cwt(pluse, np.arange(1, 10)) # 返回一个列表
 end = time.time()
 print("The time is %f s" % (end - start))
 pks = [pluse[x] for x in peaks]
 return pks


def histcnt(pks, edge=None, channel=None):
 cnt = plt.hist(pks, edge)
 res = pd.DataFrame(cnt[0], index=channel, columns=['cnt'])
 return res


if __name__ == '__main__':
 with Pool(processes=8) as p:
 start = time.time()
 print('Parent process %s.' % os.getpid())
 pluse = pd.read_csv('data/samples.csv', chunksize=50000, names=['time', 'energe'])
 channel = pd.read_csv('data/channels.txt', names=['value'])
 edges = channel * 2
 edges = pd.DataFrame({'value': [0]}).append(edges, ignore_index=True)
 specal = []
 for data in pluse:
 total = p.apply_async(findpeak, (data['energe'],),
   callback=partial(histcnt, edge=edges['value'], channel=channel['value']))
 specal.append(total)
 print('Waiting for all subprocesses done...')
 p.close()
 p.join()
 print('All subprocesses done.')
 spec = sum(specal)
 plt.figure()
 plt.plot(spec['cnt'])
 spec.to_csv('data/spec1.csv', header=False)
 print('every is OK')
 end = time.time()
 print("The time is %f s" % (end - start))

import pandas as pd
import time
import scipy.signal as signal
import numpy as np
from multiprocessing import Pool
import os
import matplotlib.pyplot as plt


def findpeak(pluse):
 pluse[pluse < 0] = 0
 pluse[pluse > 100] = 0
 print('Sub process %s.' % os.getpid())
 start = time.time()
 peaks = signal.find_peaks_cwt(pluse, np.arange(1, 10))
 end = time.time()
 print("The time is %f s" % (end - start))
 res = [pluse[x] for x in peaks]
 return res


if __name__ == '__main__':
 with Pool(processes=8) as p:
 start = time.time()
 print('Parent process %s.' % os.getpid())
 pluse = pd.read_csv('data/sample.csv', chunksize=200000, names=['time', 'energe'])
 pks = []
 for data in pluse:
 pks.append(p.apply_async(findpeak, (data['energe'],)))
 print('Waiting for all subprocesses done...')
 p.close()
 p.join()
 print('All subprocesses done.')
 peaks = []
 for i, ele in enumerate(pks):
 peaks.extend(ele.get())
 peaks = pd.DataFrame(peaks, columns=['energe'])
 peaks.to_csv('peaks.csv', index=False, header=False, chunksize=50000)
 channel = pd.read_csv('data/channels.txt', names=['value'])
 channel *= 2
 channel = pd.DataFrame({'value': [0]}).append(channel, ignore_index=True)
 plt.figure()
 spec = plt.hist(peaks['energe'], channel['value'])
 # out.plot.hist(bins=1024)
 # print(out)
 # cnt = peaks.value_counts(bins=1024)
 # cnt.to_csv('data/cnt.csv', index=False, header=False)
 print('every is OK')
 end = time.time()
 print("The time is %f s" % (end - start))