Add a toy to exercise the SMT writing code. (#80)

merkle/map_hasher.go

@@ -8,15 +8,15 @@ import (
 // "null" hashes for the unused sections of a SparseMerkleTree.
 type MapHasher struct {
 	TreeHasher
-	keyHasher  keyHashFunc
+	HashKey    keyHashFunc
 	nullHashes []trillian.Hash
 }
 
 // NewMapHasher creates a new MapHasher based on the passed in hash function.
 func NewMapHasher(th TreeHasher) MapHasher {
 	return MapHasher{
 		TreeHasher: th,
-		keyHasher:  keyHasher(th.Hasher),
+		HashKey:    keyHasher(th.Hasher),
 		nullHashes: createNullHashes(th),
 	}
 }

merkle/sparse_merkle_tree.go

@@ -384,7 +384,7 @@ func (s SparseMerkleTreeReader) RootAtRevision(rev int64) (trillian.Hash, error)
 // specified key at the specified revision.
 // If the revision does not exist it will return ErrNoSuchRevision error.
 func (s SparseMerkleTreeReader) InclusionProof(rev int64, key trillian.Key) ([]trillian.Hash, error) {
-	kh := s.hasher.keyHasher(key)
+	kh := s.hasher.HashKey(key)
 	nid := storage.NewNodeIDFromHash(kh)
 	sibs := nid.Siblings()
 	nodes, err := s.tx.GetMerkleNodes(rev, sibs)

merkle/sparse_merkle_tree_test.go

@@ -232,7 +232,7 @@ func TestInclusionProofGetsTooManyNodes(t *testing.T) {
 	const rev = 100
 	r, tx := getSparseMerkleTreeReaderWithMockTX(rev)
 	const key = "SomeArbitraryKey"
-	keyHash := r.hasher.keyHasher([]byte(key))
+	keyHash := r.hasher.HashKey([]byte(key))
 	// going to return one too many nodes
 	nodes := make([]storage.Node, 257, 257)
 	// First build a plausible looking set of proof nodes.
@@ -278,7 +278,7 @@ func testSparseTreeCalculatedRoot(t *testing.T, vec sparseTestVector) {
 func testSparseTreeCalculatedRootWithWriter(t *testing.T, rev int64, vec sparseTestVector, w *SparseMerkleTreeWriter) {
 	leaves := make([]HashKeyValue, 0)
 	for _, kv := range vec.kv {
-		leaves = append(leaves, HashKeyValue{w.hasher.keyHasher([]byte(kv.k)), w.hasher.HashLeaf([]byte(kv.v))})
+		leaves = append(leaves, HashKeyValue{w.hasher.HashKey([]byte(kv.k)), w.hasher.HashLeaf([]byte(kv.v))})
 	}
 
 	if err := w.SetLeaves(leaves); err != nil {
@@ -319,7 +319,7 @@ func testSparseTreeFetches(t *testing.T, vec sparseTestVector) {
 
 		// calculate the set of expected node reads.
 		for i, kv := range vec.kv {
-			keyHash := w.hasher.keyHasher([]byte(kv.k))
+			keyHash := w.hasher.HashKey([]byte(kv.k))
 			nodeID := storage.NewNodeIDFromHash(keyHash)
 			leafNodeIDs = append(leafNodeIDs, nodeID)
 			sibs := nodeID.Siblings()
@@ -488,7 +488,7 @@ func DISABLEDTestSparseMerkleTreeWriterBigBatch(t *testing.T) {
 	for x := 0; x < numBatches; x++ {
 		h := make([]HashKeyValue, batchSize)
 		for y := 0; y < batchSize; y++ {
-			h[y].HashedKey = w.hasher.keyHasher([]byte(fmt.Sprintf("key-%d-%d", x, y)))
+			h[y].HashedKey = w.hasher.HashKey([]byte(fmt.Sprintf("key-%d-%d", x, y)))
 			h[y].HashedValue = w.hasher.TreeHasher.HashLeaf([]byte(fmt.Sprintf("value-%d-%d", x, y)))
 		}
 		if err := w.SetLeaves(h); err != nil {

storage/mysql/drop_storage.sql

@@ -5,6 +5,8 @@ DROP TABLE IF EXISTS Subtree;
 DROP TABLE IF EXISTS SequencedLeafData;
 DROP TABLE IF EXISTS TreeHead;
 DROP TABLE IF EXISTS LeafData;
+DROP TABLE IF EXISTS MapLeaf;
+DROP TABLE IF EXISTS MapHead;
 DROP TABLE IF EXISTS TreeControl;
 DROP TABLE IF EXISTS MapHead;
 DROP TABLE IF EXISTS MapLeaf;

vmap/toy/vmap_toy.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"flag"
+	"fmt"
+
+	"github.com/golang/glog"
+	"github.com/google/trillian"
+	"github.com/google/trillian/merkle"
+	"github.com/google/trillian/storage"
+	"github.com/google/trillian/storage/mysql"
+	"github.com/google/trillian/testonly"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+var mysqlUriFlag = flag.String("mysql_uri", "test:zaphod@tcp(127.0.0.1:3306)/test", "")
+
+func main() {
+	flag.Parse()
+	glog.Info("Starting...")
+	ms, err := mysql.NewMapStorage(trillian.MapID{[]byte("TODO"), 1}, *mysqlUriFlag)
+	if err != nil {
+		glog.Fatalf("Failed to open mysql storage: %v", err)
+	}
+
+	hasher := merkle.NewMapHasher(merkle.NewRFC6962TreeHasher(trillian.NewSHA256()))
+
+	testVecs := []struct {
+		batchSize       int
+		numBatches      int
+		expectedRootB64 string
+	}{
+		// roots calculated using python code.
+		{1024, 4, "Av30xkERsepT6F/AgbZX3sp91TUmV1TKaXE6QPFfUZA="},
+		{10, 4, "6Pk5sprCr3ACfo0OLRZw7sAGdIBTc+7+MxfdW3n76Pc="},
+		{6, 4, "QZJ42Te4bw+uGdUaIqzhelxpERU5Ru6uLdy0ixJAuWQ="},
+		{4, 4, "9myL1k8Ik6m3Q3JXljHLzfNQHS2d5X6CCbpE/x3mixg="},
+		{5, 4, "4xyGOe2DQYi2Qb4aBto9R7jSmiRYqfJ+TcMxUZTXMkM="},
+		{6, 3, "FeB/9D+Gzo6oYB2Zi2JMHdrr9KvfvMk7o6DOzjPYG4w="},
+		{10, 3, "RfJ6JPERbkDiwlov8/alCqr4yeYYIWl3dWWS3trHsiY="},
+		{1, 4, "pQhTahkoXM3WTeAO1o8BYKhgMNzS1yG03vg/fQSVyIc="},
+		{2, 4, "RdcEkg5qEuW5eV3VJJLr6uSzvlc27D55AZChG76LHGA="},
+		{4, 1, "3dpnVw5Le3HDq/GAkGoSYT9VkzJRV8z18huOk5qMbco="},
+		{1024, 1, "7R5uvGy5MJ2Y8xrQr4/mnn3aPw39vYscghmg9KBJaKc="},
+		{1, 2, "cZIYiv7ZQ/3rBfpCrha1NKdUnQ8NsTm21WWdV3P4qcU="},
+		{1, 3, "KUaQinjLtPQ/ZAek4nHrR7tVXDxLt5QsvZK3vGopDkA="}}
+
+	const testIndex = 0
+
+	batchSize := testVecs[testIndex].batchSize
+	numBatches := testVecs[testIndex].numBatches
+	expectedRootB64 := testVecs[testIndex].expectedRootB64
+
+	rev := int64(0)
+	var root trillian.Hash
+	for x := 0; x < numBatches; x++ {
+		w, err := merkle.NewSparseMerkleTreeWriter(rev, hasher,
+			func() (storage.TreeTX, error) {
+				return ms.Begin()
+			})
+		if err != nil {
+			glog.Fatalf("Failed to create new SMTWriter: %v", err)
+		}
+		tx, err := ms.Begin()
+		if err != nil {
+			glog.Fatalf("Failed to Begin() a new tx: %v", err)
+		}
+
+		glog.Infof("Starting batch %d...", x)
+		h := make([]merkle.HashKeyValue, batchSize)
+		for y := 0; y < batchSize; y++ {
+			h[y].HashedKey = hasher.HashKey([]byte(fmt.Sprintf("key-%d-%d", x, y)))
+			h[y].HashedValue = hasher.TreeHasher.HashLeaf([]byte(fmt.Sprintf("value-%d-%d", x, y)))
+		}
+		glog.Infof("Created %d k/v pairs...", len(h))
+
+		glog.Info("SetLeaves...")
+		if err := w.SetLeaves(h); err != nil {
+			glog.Fatalf("Failed to batch %d: %v", x, err)
+		}
+		glog.Info("SetLeaves done.")
+
+		glog.Info("CalculateRoot...")
+		root, err = w.CalculateRoot()
+		if err != nil {
+			glog.Fatalf("Failed to calculate root hash: %v", err)
+		}
+		glog.Infof("CalculateRoot (%d), root: %s", x, base64.StdEncoding.EncodeToString(root))
+
+		err = tx.Commit()
+		if err != nil {
+			glog.Fatalf("Failed to Commit() tx: %v", err)
+		}
+		rev++
+	}
+
+	if expected, got := testonly.MustDecodeBase64(expectedRootB64), root; !bytes.Equal(expected, root) {
+		glog.Fatalf("Expected root %s, got root: %s", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got))
+	}
+	glog.Infof("Finished, root: %s", base64.StdEncoding.EncodeToString(root))
+
+}