ht.go

// Copyright © 2014,2015 Lawrence E. Bakst. All rights reserved.
package main

// based on http://amsoftware.narod.ru/algo.html
// However I can't replicate the result

import (
	"bufio"
	cr "crypto/rand"
	"crypto/sha1"
	"flag"
	"fmt"
	"hash"
	"io"
	. "leb.io/hashland/hashf"     // cleaved
	. "leb.io/hashland/hashtable" // cleaved
	"leb.io/hashland/nhash"
	"leb.io/hashland/smhasher"
	"leb.io/hrff"
	_ "math"
	"math/rand"
	"os"
	"sort"
	"time"

	// remove these at some point
	"leb.io/aeshash"           // remove
	"leb.io/hashland/gomap"    // remove
	"leb.io/hashland/jenkins"  // remove
	"leb.io/hashland/keccak"   // remove
	"leb.io/hashland/keccakpg" // remove
	"leb.io/hashland/nullhash" // remove
	"leb.io/hashland/siphash"  // remove
	"leb.io/hashland/spooky"
)

func ReadFile(file string, cb func(line string)) int {
	var lines int
	//fmt.Printf("ReadFile: file=%q\n", file)
	f, err := os.Open(file)
	if err != nil {
		panic("ReadFile: opening file")
	}
	defer f.Close()

	rl := bufio.NewReader(f)
	//rs := csv.NewReader(f)
	// rs.Comma = '\t'      // Use tab-separated values

	for {
		//r, err := rs.Read()
		s, err := rl.ReadString(10) // 0x0A separator = newline
		if err == io.EOF {
			// fmt.Printf("ReadFile: EOF\n")
			return lines
		} else if err != nil {
			panic("reading file")
		}
		if s[len(s)-1] == '\n' {
			s = s[:len(s)-1]
		}
		if s[len(s)-1] == '\r' {
			s = s[:len(s)-1]
		}
		if s[len(s)-1] == ' ' {
			s = s[:len(s)-1]
		}
		//fmt.Printf("%q\n", s)
		if cb != nil {
			cb(s)
		}
		lines++
	}
}

func Test0(file string, lines int, hf2 string) (ht *HashTable) {
	var cnt int
	var countlines = func(line string) {
		cnt++
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, countlines)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestA(file string, lines int, hf2 string) (ht *HashTable) {
	//var lines int
	/*
		var countlines = func(line string) {
			lines++
		}
	*/
	var addLine = func(line string) {
		ht.Insert([]byte(line))
	}

	//fmt.Printf("\t%20q: ", hf2)
	//fmt.Printf("run: file=%q\n", file)
	//fmt.Printf("TestA: lines=%d, hf2=%q\n", lines, hf2)
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	//fmt.Printf("ht=%v\n", ht)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestB(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line += "\n"
		ht.Insert([]byte(line))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestC(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line += line + "\n\n\n\n"
		ht.Insert([]byte(line))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestD(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line = "ABCDE" + line
		ht.Insert([]byte(line))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestE(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line = line + line
		ht.Insert([]byte(line))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestF(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line = line + line + line + line
		ht.Insert([]byte(line))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func reverse(s string) string {
	if len(s) == 0 {
		return ""
	}
	return reverse(s[1:]) + string(s[0])
}

func TestG(file string, lines int, hf2 string) (ht *HashTable) {
	var addLine = func(line string) {
		line2 := reverse(line)
		//fmt.Printf("line=%q, line2=%q", line, line2)
		ht.Insert([]byte(line2))
	}
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	ReadFile(file, addLine)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

func TestH(file string, lines int, hf2 string) (ht *HashTable) {
	var cnt int
	var counter = func(word string) {
		cnt++
	}
	var addWord = func(word string) {
		ht.Insert([]byte(word))
	}
	//test := []string{"abcdefgh", "efghijkl", "ijklmnop", "mnopqrst", "qrstuvwx", "uvwxyz01"} // 262144 words

	genWords(letters, counter)
	ht = NewHashTable(cnt, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	genWords(letters, addWord)
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

// integers 0 to n
func TestI(file string, lines int, hf2 string) (ht *HashTable) {
	//r = rand.New(rand.NewSource(int64(bseed)))
	fmt.Printf("n=%d, ni=%d\n", *n, *ni)
	bs := make([]byte, 4, 4)
	if *dru {
		*n = -*n
	}
	ht = NewHashTable(*n, seed, *extra, *pd, *oa, *prime) // ??? @@@
	start := time.Now()
	for i := 0; i < *ni; i++ {
		bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
		ht.Insert(bs)
		//fmt.Printf("i=%d, 0x%08x\n", i, i)
	}
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

// marching 1
func TestJ(file string, lines int, hf2 string) (ht *HashTable) {
	length := 900
	keys := length * 8
	key := make([]byte, length, length)
	key = key[:]
	ht = NewHashTable(keys, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	for k := range key {
		for i := uint(0); i < 8; i++ {
			key[k] = 1 << i
			//fmt.Printf("k=%d, i=%d, key=%#0x2\n", k, i, key)
			ht.Insert(key)
			key[k] = 0
		}
	}
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

// all possible 3 bytes keys must have distinct hashses
func TestL(file string, lines int, hf2 string) (ht *HashTable) {
	type HashSet struct {
		m map[uint64]struct{} // set of hashes added
		n int                 // number of hashes added
	}
	hs := &HashSet{make(map[uint64]struct{}), 0}
	key := make([]byte, 3, 3)
	key = key[:]
	ht = NewHashTable(256*256*256, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	for i := 0; i < 256; i++ {
		for j := 0; j < 256; j++ {
			for k := 0; k < 256; k++ {
				key[0] = byte(i)
				key[1] = byte(j)
				key[2] = byte(k)
				h := Hashf(key, ht.Seed)
				hs.m[h] = struct{}{}
				hs.n++
				//fmt.Printf("i=%d, j=%d, k=%d, key=%#v\n", i, j, k, key)
				ht.Insert(key)
			}
		}
	}
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	ht.Dups2 = hs.n - len(hs.m)
	//fmt.Printf("collisions=%d\n", ht.Dups2)
	return
}

// hash function mod test
func TestM(file string, lines int, hf2 string) (ht *HashTable) {
	//r = rand.New(rand.NewSource(int64(bseed)))
	fmt.Printf("n=%d, ni=%d\n", *n, *ni)
	bs := make([]byte, 4, 4)
	if *dru {
		*n = -*n
	}
	hist := make([]int, *n)
	ht = NewHashTable(*n, seed, *extra, *pd, *oa, *prime) // ??? @@@
	mod := uint64(*n)
	start := time.Now()
	for i := 0; i < *ni; i++ {
		bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
		h := Hashf(bs, ht.Seed)
		hist[h%mod]++
		//fmt.Printf("i=%d, 0x%08x\n", i, i)
	}
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	fmt.Printf("expected=%d\n", *ni / *n)
	fmt.Printf("hist=%v\n", hist)
	return
}

func unhex(c byte) uint8 {
	switch {
	case '0' <= c && c <= '9':
		return c - '0'
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10
	}
	panic("unhex: bad input")
}

func hexToBytes(s string) []byte {
	var data = make([]byte, 1000, 1000)
	data = data[0 : len(s)/2]

	n := len(s)
	if (n & 1) == 1 {
		panic("gethex: string must be even")
	}
	for i := range data {
		data[i] = unhex(s[2*i])<<4 | unhex(s[2*i+1])
	}
	//fmt.Printf("hexToBytes: len(data)=%d, len(s)=%d\n", len(data), len(s))
	return data[0 : len(s)/2]
}

var r *rand.Rand

func newRand(seed int64) {
	r = rand.New(rand.NewSource(seed))
}

func rbetween(a uint64, b uint64) uint64 {
	rf := r.Float64()
	diff := float64(b - a + 1)
	r2 := rf * diff
	r3 := r2 + float64(a)
	//      fmt.Printf("rbetween: a=%d, b=%d, rf=%f, diff=%f, r2=%f, r3=%f\n", a, b, rf, diff, r2, r3)
	ret := uint64(r3)
	return ret
}

func TestK(file string, lines int, hf2 string) (ht *HashTable) {
	var aseed uint64
	var rseed uint64
	var b = make([]byte, 1000, 1000)
	var hashLine = func(line string) {
		//fmt.Printf("line=%q\n", line)
		h := Hashf(b, rseed)
		b := hexToBytes(line)
		fmt.Printf("\t\tseed=%d, key=%x, hash=0x%x\n", rseed, b, h)
	}
	fmt.Printf("\n")
	ht = NewHashTable(lines, seed, *extra, *pd, *oa, *prime)
	start := time.Now()
	for aseed = 0; aseed < uint64(*ns); aseed++ {
		rseed = rbetween(0, uint64(1)<<63)
		ReadFile(file, hashLine)
	}
	stop := time.Now()
	ht.Dur = tdiff(start, stop)
	return
}

/*
func TestS(file string, lines int, hf2 string) (ht *HashTable) {
	var seed uint64
	var rseed uint64

	sha1160 := sha1.New()
	fp20 := make([]byte, 20, 20)

	sha1160.Reset()
	sha1160.Write(k)
	fp20 = fp20[0:0]
	fp20 = sha1160.Sum(fp20)

	return
}
*/

// [ABCDEFGH][EFGHIJKL][IJKLMNOP][MNOPQRST][QRSTUVWX][UVWXYZ01]
// given a slice of strings, generate all the combinations in order
func genWords(perms []string, f func(word string)) {
	var indices = make([]int, len(perms), len(perms))
	var idx int
	var inc = func() bool {
		// increment counter with carry
		for idx = 0; ; {
			indices[idx]++
			if indices[idx] >= len(perms[idx]) {
				indices[idx] = 0
				idx++
				if idx >= len(perms) {
					return true
				}
				continue
			} else {
				break
			}
		}
		return false
	}
	var letter = func(idx int, s string) string {
		return string(s[indices[idx]])
	}
	var word func(p []string) string
	word = func(p []string) string {
		if len(p) == 0 {
			return ""
		}
		l := len(p)
		idx := len(perms) - l
		tmp := letter(idx, p[0]) + word(p[1:])
		return tmp
	}
	// generate a word, hand it out, bump counter, repeat
	for {
		aword := word(perms)
		f(aword)
		if inc() {
			return
		}
	}
}

func tdiff(begin, end time.Time) time.Duration {
	d := end.Sub(begin)
	return d
}

// IntSlice attaches the methods of Interface to []int, sorting in increasing order.
type Uint32Slice []uint32

func (p Uint32Slice) Len() int           { return len(p) }
func (p Uint32Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Uint32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p Uint32Slice) Sort() { sort.Sort(p) }

func checkForDups32(u Uint32Slice) (dups, mrun int) {
	i := 0
	run := 0
	for k, v := range u {
		if k == 0 || i == k {
			continue
		}
		if u[i] == v {
			run++
			dups++
			continue
		} else {
			if run > mrun {
				mrun = run
			}
			run = 0
			i = k
		}
	}
	return
}

var keySizes = []int{4, 8, 16, 32, 64, 512, 1024, 4096}

// WARNING: The benchmark run here is hardcoded (in two places)
// You must recompile to change the benchmark
func benchmark32s(n int) {
	//var hashes = make(Uint32Slice, n)
	const nbytes = 16384 // fix
	bs := make([]byte, nbytes, nbytes)
	bs = bs[:]
	//fp20 := make([]byte, 20, 20)
	Hf2 = "nullhash"
	for _, ksiz := range keySizes {
		if ksiz == 512 {
			n = n / 10
		}
		bs = bs[:ksiz]
		//hashes = make(Uint32Slice, n, n)
		//hashes = hashes[:]
		//fmt.Printf("ksiz=%d, len(bs)=%d\n", ksiz, len(bs))
		pn := hrff.Int64{int64(n), ""}
		ps := hrff.Int64{int64(n * ksiz), "B"}
		//fmt.Printf("benchmark32s: gen n=%d, n=%h, keySize=%d, size=%h\n", n, pn, ksiz, ps)
		if false {
			_ = gomap.Hash64(bs, 0)
			_ = aeshash.Hash(bs, 0)
		}
		start, stop := time.Now(), time.Now()
		switch ksiz {
		case 4:
			start = time.Now()
			for i := 0; i < n; i++ {
				//_ = aeshash.Hash(bs, 0)
				//_, _ = jenkins.Jenkins364(bs, 0, 0, 0)
				//_ = jenkins.Hash232(bs, 0)
				//_, _ = jenkins.Jenkins364(bs, 0, 0, 0)
				_ = spooky.Hash64(bs, 0)
				// sha1160.Reset()
				// sha1160.Write(bs)
				// fp20 = fp20[0:0]
				// fp20 = sha1160.Sum(fp20)
				// _ = uint64(fp20[0])<<56 | uint64(fp20[1])<<48 | uint64(fp20[2])<<40 | uint64(fp20[3])<<32 |
				// 	uint64(fp20[4])<<24 | uint64(fp20[5])<<16 | uint64(fp20[6])<<8  | uint64(fp20[7])<<0
				//k224.Reset()
				//k224.Write(bs)
				//_ = k224.Sum(nil)
				//bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
				//_ = gomap.Hash64(bs, 0)
				//_ =  nhf64.Hash64S(bs, 0) // chnage below too
				//nh.Reset()
				//nh.Write(bs)
				//_ = nh.Sum64()
				//Hashf(bs, 0)
				//_ = siphash.Hash(0, 0, bs)
				//hashes[i] = h
				//fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, h)
				//hashes[i] = h
				//fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, h)
			}
			stop = time.Now()
		default:
			start = time.Now()
			for i := 0; i < n; i++ {
				//_ = aeshash.Hash(bs, 0)
				//_, _ = jenkins.Jenkins364(bs, 0, 0, 0)
				_ = spooky.Hash64(bs, 0)
				// sha1160.Reset()
				// sha1160.Write(bs)
				// fp20 = fp20[0:0]
				// fp20 = sha1160.Sum(fp20)
				// _ = uint64(fp20[0])<<56 | uint64(fp20[1])<<48 | uint64(fp20[2])<<40 | uint64(fp20[3])<<32 |
				// 	uint64(fp20[4])<<24 | uint64(fp20[5])<<16 | uint64(fp20[6])<<8  | uint64(fp20[7])<<0
				//k224.Reset()
				//k224.Write(bs)
				//_ = k224.Sum(nil)
				//bs[0], bs[1], bs[2], bs[3], bs[4], bs[5], bs[6], bs[7] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24), byte(i>>32), byte(i>>40), byte(i>>48), byte(i>>56)
				//_ = aeshash.Hash(bs, 0)
				//_ = gomap.Hash64(bs, 0)
				//_ = siphash.Hash(0, 0, bs)
				//_ =  nhf64.Hash64S(bs, 0)
				//nh.Reset()
				//nh.Write(bs)
				//_ = nh.Sum64()
				//Hashf(bs, 0)
			}
			stop = time.Now()
		}
		d := tdiff(start, stop)
		hsec := hrff.Float64{(float64(n) / d.Seconds()), "hashes/sec"}
		bsec := hrff.Float64{(float64(n) * float64(ksiz) / d.Seconds()), "B/sec"}
		//fmt.Printf("benchmark32s: %h\n", hsec)
		//fmt.Printf("benchmark32s: %.2h\n\n", bsec)
		fmt.Printf("\tksize=%d, n=%h, size=%h, %h, %.1h, time=%v\n", ksiz, pn, ps, hsec, bsec, d)
	}
	return

	fmt.Printf("benchmark32s: sort n=%d\n", n)
	//hashes.Sort()
	/*
		for i := 0; i < n; i++ {
			fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, hashes[i])
		}
	*/
	fmt.Printf("benchmark32s: dup check n=%d\n", n)
	//dups, mrun := checkForDups32(hashes)
	//fmt.Printf("benchmark32: dups=%d, mrun=%d\n", dups, mrun)
}

func benchmark32g(h hash.Hash64, hf2 string, n int) {
	//var hashes Uint32Slice
	const nbytes = 16384 // fix

	Hf2 = hf2
	bs := make([]byte, nbytes, nbytes)
	bs = bs[:]
	for _, ksiz := range keySizes {
		if ksiz == 512 {
			n = n / 10
		}
		bs = bs[:ksiz]
		//hashes = make(Uint32Slice, n, n)
		//hashes = hashes[:]
		//fmt.Printf("ksiz=%d, len(bs)=%d\n", ksiz, len(bs))
		pn := hrff.Int64{int64(n), ""}
		ps := hrff.Int64{int64(n * ksiz), "B"}
		//fmt.Printf("benchmark32g: gen n=%d, n=%h, keySize=%d, size=%h\n", n, pn, ksiz, ps)
		start, stop := time.Now(), time.Now()
		if h != nil {
			switch ksiz {
			case 4:
				start = time.Now()
				for i := 0; i < n; i++ {
					bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
					h.Reset()
					h.Write(bs)
					_ = h.Sum64()
				}
				stop = time.Now()
			default:
				start = time.Now()
				for i := 0; i < n; i++ {
					bs[0], bs[1], bs[2], bs[3], bs[4], bs[5], bs[6], bs[7] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24), byte(i>>32), byte(i>>40), byte(i>>48), byte(i>>56)
					h.Reset()
					h.Write(bs)
					_ = h.Sum64()
					//hashes[i] = h
					//fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, h)
				}
				stop = time.Now()
			}
		} else {
			switch ksiz {
			case 4:
				start = time.Now()
				for i := 0; i < n; i++ {
					//bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
					Hashf(bs, 0) // the generic adapter is very inefficient, as much as 6X slower, however same for everyone
				}
				stop = time.Now()
			default:
				start = time.Now()
				for i := 0; i < n; i++ {
					//bs[0], bs[1], bs[2], bs[3], bs[4], bs[5], bs[6], bs[7] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24), byte(i>>32), byte(i>>40), byte(i>>48), byte(i>>56)
					Hashf(bs, 0) // the generic adapter is very inefficient, as much as 6X slower, however same for everyone
					//hashes[i] = h
					//fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, h)
				}
				stop = time.Now()
			}
		}
		d := tdiff(start, stop)
		hsec := hrff.Float64{(float64(n) / d.Seconds()), "hashes/sec"}
		bsec := hrff.Float64{(float64(n) * float64(ksiz) / d.Seconds()), "B/sec"}
		//fmt.Printf("benchmark32g: %h\n", hsec)
		//fmt.Printf("benchmark32g: %.2h\n\n", bsec)
		fmt.Printf("\tksize=%d, n=%h, size=%h, %h, %.1h, time=%v\n", ksiz, pn, ps, hsec, bsec, d)
	}

	if *cd {
		fmt.Printf("benchmark32g: sort n=%d\n", n)
		//hashes.Sort()
		/*
			for i := 0; i < n; i++ {
				fmt.Printf("i=%d, 0x%08x, h=0x%08x\n", i, i, hashes[i])
			}
		*/
		fmt.Printf("benchmark32g: dup check n=%d\n", n)
		//dups, mrun := checkForDups32(hashes)
		//fmt.Printf("benchmark32: dups=%d, mrun=%d\n", dups, mrun)
	}
}

var benchmarks = []string{"j332c", "j232", "sbox", "CrapWow"}

//var benchmarks = []string{"j332c"}

func benchmark(hashes []string, hf2 string, n int) {
	//for _, v := range hashes {
	//hf32 := Halloc(v)
	//fmt.Printf("benchmark32g: %q\n", v)
	benchmark32g(nil, hf2, n)
	fmt.Printf("\n")
	//}
}

type Test struct {
	name string
	flag **bool
	ptf  func(file string, lines int, hashf string) (ht *HashTable)
	desc string
}

var Tests = []Test{
	{"TestA", &A, TestA, "insert keys"},
	{"TestB", &B, TestB, "add newline to key"},
	{"TestC", &C, TestC, "add 4 newlines to key"},
	{"TestD", &D, TestD, "prepend ABCDE to key"},
	{"TestE", &E, TestE, "add 1 duplicate key"},
	{"TestF", &F, TestF, "add 3 duplicate keys"},
	{"TestG", &G, TestF, "reverse letter order in key"},
	{"TestH", &H, TestH, "words from letter combinations in wc"},
	{"TestI", &I, TestI, "integers from 0 to ni-1 (does not read file)"},
	{"TestJ", &J, TestJ, "one bit keys (does not read file)"},
	{"TestK", &K, TestK, "read file of keys and print hashes"},
	{"TestL", &L, TestL, "all possible 3 byte keys"},
	{"TestL", &M, TestM, "hash function mod test"},
}

func runTestsWithFileAndHashes(file string, lines int, hf []string) {
	var test Test
	if file != "" {
		if lines <= 0 {
			lines = ReadFile(file, nil)
		}
		//fmt.Printf("file=%q, lines=%d\n", file, lines)
		if *T0 {
			fmt.Printf("Test0 - ReadFile\n\t%20q: ", "ReadFile")
			ht := Test0(file, lines, "")
			ht.Print()
		}
	}
	for _, test = range Tests {
		if **test.flag {
			fmt.Printf("%s - %s\n", test.name, test.desc)
			for _, Hf2 = range hf {
				hi := HashFunctions[Hf2]
				if *c && !hi.Crypto {
					continue
				}
				if *h32 && hi.Size != 32 {
					continue
				}
				if *h64 && hi.Size != 64 {
					continue
				}
				fmt.Printf("\t%20q: ", Hf2)
				ht := test.ptf(file, lines, Hf2)
				if *rr || *s != 0 {
					fmt.Printf("seed=%#016x\n", seed)
				}
				ht.Print()
			}
		}
	}
}

var file = flag.String("file", "", "words to read")
var lines = flag.Int("lines", 0, "number of lines to read in file")
var hf = flag.String("hf", "all", "hash function")
var extra = flag.Int("e", 1, "extra bis in table size")
var prime = flag.Bool("p", false, "table size is prime, use mod")
var all = flag.Bool("a", false, "run all tests")
var pd = flag.Bool("pd", false, "print duplicate hashes")
var oa = flag.Bool("oa", false, "open addressing (no buckets)")
var s = flag.Int("s", 0, "seed for random number gen.")
var rr = flag.Bool("rr", false, "random run, i.e. random hash seed")

var seed int64

var c = flag.Bool("c", false, "only test crypto hash functions")
var h32 = flag.Bool("h32", false, "only test 32 bit has functions")
var h64 = flag.Bool("h64", false, "only test 64 bit has functions")

var b = flag.Bool("b", false, "run benchmarks")
var hcb = flag.Bool("hcb", false, "run hard coded benchmark")
var sm = flag.Bool("sm", false, "run SMHasher")
var v = flag.Bool("v", false, "verbose")
var cd = flag.Bool("cd", false, "check for duplicate hashs when running benchmarks")

//var wc = flags.String("wc", "abcdefgh, efghijkl, ijklmnop, mnopqrst, qrstuvwx, uvwxyz01", "letter combinations for word") // 262144 words)

var _ni = hrff.Int{V: 4096, U: "keys"}
var ni *int = &_ni.V // flag.Int("ni", 200000, "number of integer keys to be inserted")

var _n = hrff.Int{V: 4096, U: "buckets"}
var n *int = &_n.V // var n = flag.Int("n", 1000000, "number of buckets")

var dru = flag.Bool("dru", false, "don't round (n) up")
var ns = flag.Int("ns", 1, "number of seeds to test")
var trace = flag.Bool("t", false, "generate trace file")

var T0 = flag.Bool("0", false, "test 0")
var A = flag.Bool("A", false, "test A")
var B = flag.Bool("B", false, "test B")
var C = flag.Bool("C", false, "test C")
var D = flag.Bool("D", false, "test D")
var E = flag.Bool("E", false, "test E")
var F = flag.Bool("F", false, "test F")
var G = flag.Bool("G", false, "test G")
var H = flag.Bool("H", false, "test H")
var I = flag.Bool("I", false, "test I")
var J = flag.Bool("J", false, "test J")
var K = flag.Bool("K", false, "test K")
var L = flag.Bool("L", false, "test L")
var M = flag.Bool("M", false, "test M")
var S = flag.Bool("S", false, "test S")

var letters = []string{"abcdefgh", "efghijkl", "ijklmnop", "mnopqrst", "qrstuvwx", "uvwxyz01"} // 262144 words
var TestPointers = []**bool{&A, &B, &C, &D, &E, &F, &G, &H, &I, &J, &K, &L, &M}

func allTestsOn() {
	*A, *B, *C, *D, *E, *F, *G, *H, *I, *J, *L, *M = true, true, true, true, true, true, true, true, true, true, true, true
}

func allTestsOff() {
	*A, *B, *C, *D, *E, *F, *G, *H, *I, *J, *L, *M = false, false, false, false, false, false, false, false, false, false, false, false
}

func main() {
	flag.Var(&_ni, "ni", "number of integer keys to be inserted.")
	flag.Var(&_n, "n", "number of buckets in hash table.")
	/*
		var cnt int
		var f = func(word string) {
			cnt++
			//fmt.Printf("%q\n", word)
		}

		test := []string{"ab", "cd"}
		test := []string{"abcdefgh", "efghijkl", "ijklmnop", "mnopqrst", "qrstuvwx", "uvwxyz01"} // 262144 words
		genWords(test, f)
		fmt.Printf("cnt=%d\n", cnt)
		return
	*/
	flag.Parse()
	*ni = _ni.V
	*n = _n.V
	Trace = *trace
	if *all {
		allTestsOn()
	}
	if *hcb {
		*b = true
	}

	if *rr {
		seed = time.Now().UTC().UnixNano()
		// fixed pattern or different values each time
		b := make([]byte, 8)
		_, err := cr.Read(b)
		if err != nil {
			fmt.Println("error:", err)
			return
		}
		seed = int64(uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 |
			uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7])<<0)
	} else {
		seed = int64(*s)
	}
	newRand(seed)
	//fmt.Printf("%d lines read\n", lines)

	// read file and count lines
	// create table
	// read file and insert
	// stats

	switch {
	case *sm:
		if *hf == "all" {
			for _, Hf2 = range TestHashFunctions {
				hi := HashFunctions[Hf2]
				if *c && !hi.Crypto {
					continue
				}
				if *h32 && hi.Size != 32 {
					continue
				}
				if *h64 && hi.Size != 64 {
					continue
				}
				fmt.Printf("%q\n", Hf2)
				smhasher.RunSMhasher(Hf2, *v)
			}
		} else {
			Hf2 = *hf
			smhasher.RunSMhasher(Hf2, *v)
		}
		return
	case *b:
		fmt.Printf("\n")
		if *hf == "all" {
			for _, Hf2 = range TestHashFunctions {
				hi := HashFunctions[Hf2]
				if *c && !hi.Crypto {
					continue
				}
				if *h32 && hi.Size != 32 {
					continue
				}
				if *h64 && hi.Size != 64 {
					continue
				}
				fmt.Printf("%q\n", Hf2)
				benchmark(benchmarks, Hf2, *n)
			}
		} else {
			if *hcb { // this benchmark is hard coded
				fmt.Printf("%q\n", "hardcoded benchmark")
				benchmark32s(*n)
			} else {
				Hf2 = *hf
				fmt.Printf("%q\n", Hf2)
				benchmark32g(nil, *hf, *n)
			}
		}
		return
	case *file != "":
		if *hf == "all" {
			runTestsWithFileAndHashes(*file, *lines, TestHashFunctions)
		} else {
			Hf2 = *hf
			runTestsWithFileAndHashes(*file, *lines, []string{*hf})
		}
	case len(flag.Args()) != 0:
		for _, v := range flag.Args() {
			if *hf == "all" {
				runTestsWithFileAndHashes(v, *lines, TestHashFunctions)
			} else {
				Hf2 = *hf
				runTestsWithFileAndHashes(v, *lines, []string{*hf})
			}
		}
	case len(flag.Args()) == 0 && !*b:
		// no files specified run the only two tests we can with the specified hash functions
		cnt := 0
		for _, v := range TestPointers {
			if **v == true {
				cnt++
			}
		}
		if cnt == 0 {
			allTestsOff()
			*I, *J = true, true
		}
		fmt.Printf("here\n")
		if *hf == "all" {
			runTestsWithFileAndHashes("", *lines, TestHashFunctions)
		} else {
			Hf2 = *hf
			runTestsWithFileAndHashes("", *lines, []string{*hf})
		}
	}
}

var nhf64 nhash.HashF64
var nh hash.Hash64
var k224 hash.Hash
var k643 hash.Hash
var sha1160 hash.Hash

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s: [flags] [dictionary-files]\n", os.Args[0])
		flag.PrintDefaults()
	}

	bs := make([]byte, 4, 4)
	bs = bs[:]
	i := 0
	bs[0], bs[1], bs[2], bs[3] = byte(i), byte(i>>8), byte(i>>16), byte(i>>24)
	_, _ = jenkins.Jenkins364(bs, 0, 0, 0)
	_ = jenkins.Hash232(bs, 0)
	_, _ = jenkins.Jenkins364(bs, 0, 0, 0)
	_ = aeshash.Hash(bs, 0)
	_ = siphash.Hash(0, 0, bs)
	nh = nullhash.New()
	nhf64 = nullhash.NewF64()
	k224 = keccak.New224()
	k643 = keccakpg.NewCustom(64, 3)
	sha1160 = sha1.New()
}