benchmark_uint8.py

from os.path import join, exists
from sys import argv
from tempfile import mkdtemp
from json_tricks import load as jt_load, dump as jt_dump
from matplotlib.pyplot import show
import numpy as np
from numpy import mean, loadtxt, array, std
from numpy.random import RandomState
from scipy import sparse
from methods import METHODS
from visualize import plot_results


class Benchmark(object):
	extension = 'data'
	
	def __init__(self, cls, data, data_name=None, reps=50):
		self.cls = cls
		self.data = data
		if data_name:
			self.data_name = data_name
		else:
			self.data_name = str(hash(data.tobytes())).replace('-', '')[:8]
		self.reps = int(reps)
		self.todo = []
		self.done = []
		self.label = self.cls.__name__
		for k in range(reps):
			pth = 'cache/{0:s}.{1:s}.{2:03d}.json'.format(self.cls.__name__, self.data_name, k)
			if exists(pth):
				self.done.append(jt_load(pth))
			else:
				inst = self.cls()
				inst._cache = pth
				self.todo.append(inst)
	
	@property
	def save_time(self):
		assert self.done
		return mean(tuple(inst.save_time for inst in self.done))
	
	@property
	def load_time(self):
		assert self.done
		return mean(tuple(inst.load_time for inst in self.done))
	
	@property
	def storage_space(self):
		assert self.done
		return mean(tuple(inst.storage_space for inst in self.done))
	
	@property
	def save_time_std(self):
		assert self.done
		return std(tuple(inst.save_time for inst in self.done))
	
	@property
	def load_time_std(self):
		assert self.done
		return std(tuple(inst.load_time for inst in self.done))
	
	@property
	def storage_space_std(self):
		assert self.done
		return std(tuple(inst.storage_space for inst in self.done))
	
	def __str__(self):
		return 'benchmark {0:s} {2:d}/{1:d}'.format(self.cls.__name__, self.reps, len(self.done))
	
	def run(self):
		while self.todo:
			inst = self.todo.pop()
			pth = join(mkdtemp(), '{0:s}_{1:d}.{2:s}'.format(self.__class__.__name__, len(self.done), inst.extension))
			inst.time_save(self.data, pth)
			inst.time_load(self.data, pth)
			jt_dump(inst, inst._cache)
			self.done.append(inst)
	
	def log(self):
		print('{0:12s}  {4:2d}/{5:2d}  {1:8.6f}+-{6:8.6f}s  {2:8.6f}+-{7:8.6f}s  {3:6.0f}+-{8:8.6f}kb'.format(self.cls.__name__, self.save_time,
			self.load_time, self.storage_space/1024., len(self.done), self.reps, self.save_time_std, self.load_time_std, self.storage_space_std/1024.))


def random_data(size, is_sparse=False, is_big=True):
	rs = RandomState(seed=123456789)
	if is_sparse:
		arr = array(sparse.rand(size[0], size[1], density=0.01, random_state=rs).todense())
	else:
		arr = rs.randint(256,size = size, dtype=np.uint8)#.astype('float64')
	if is_big:
		# don't use the full range, since some formats (Stata) uses the highest values for special meanings.
		arr = (arr - 0.5) * 1.7976931348623157e+308
	return arr


def load_example_data():
	return loadtxt('testdata.csv', delimiter=',')


if __name__ == '__main__':
	reps = int(argv[1]) if len(argv) > 1 else 30
	for data, name, label in (
		(random_data((1000, 400)), 'random_integer', 'Random integer array'),
		# (random_data((1000, 400), is_sparse=True), 'sparse', 'Sparse (0.01)'),
		# (random_data((100000, 3), is_big=False), 'long', 'Long array'),
		# (load_example_data(), 'example', 'Real data'),
	):
		print('>> benchmark {0:s} <<'.format(name))
		insts = tuple(Benchmark(cls, data, data_name=name, reps=reps) for cls in METHODS)
		for bm in insts:
			bm.run()
			bm.log()
		# sinsts = sorted(insts, key=lambda inst: (inst.save_time + inst.load_time) * inst.storage_space)
		fig, ax = plot_results(insts, fname='bm_{0:s}.png'.format(name),
			suptitle='{1:s} storage performance ({2:d}x{3:d}, avg of {0:d}x)'.format(reps, label, *data.shape))
	show()