Merge pull request #10 from gohornet/database-changes

Database changes

.github/workflows/release.yml

@@ -17,6 +17,6 @@ jobs:
       uses: actions/checkout@v1
 
     - name: Release HORNET
-      run: goreleaser --release-notes=CHANGELOG.md --rm-dist
+      run: goreleaser --rm-dist
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.goreleaser.yml

@@ -120,17 +120,9 @@ checksum:
 snapshot:
   name_template: "{{ .Tag }}"
 
-# Changelog (if --release-notes not set)
+# Changelog
 changelog:
-  sort: asc
-  filters:
-    exclude:
-    - '^docs:'
-    - '^test:'
-    - '^dev:'
-    - 'README'
-    - Merge pull request
-    - Merge branch
+  skip: true
 
 # Release
 release:

CHANGELOG.md

@@ -1,6 +1,23 @@
-# Version 0.1.0
+# HORNET Changelog
+All notable changes to this project will be documented in this file.
 
-Release note
+## [0.2.0] - 12.12.2019
 
-#### What's new
+### Added
+    - DB version number
+    - Configurable zmq host
+    - Solidification timestamp of transactions
+    - Docker files
+
+### Changed
+    - Database layout (breaking change)
+
+### Fixed
+    - Trinity compatibility
+    - WebAPI CORS headers
+
+
+## [0.1.0] - 11.12.2019
+
+### Added
     - First beta release

packages/database/database.go

@@ -15,30 +15,24 @@ var (
 
 type prefixDb struct {
 	db     *badger.DB
-	name   string
 	prefix []byte
 }
 
-func getPrefix(name string) []byte {
-	return []byte(name + "_")
-}
-
-func Get(name string) (Database, error) {
+func Get(dbPrefix byte) (Database, error) {
 	mu.Lock()
 	defer mu.Unlock()
 
-	if db, exists := dbMap[name]; exists {
+	if db, exists := dbMap[string(dbPrefix)]; exists {
 		return db, nil
 	}
 
 	badger := GetBadgerInstance()
 	db := &prefixDb{
 		db:     badger,
-		name:   name,
-		prefix: getPrefix(name),
+		prefix: []byte{dbPrefix},
 	}
 
-	dbMap[name] = db
+	dbMap[string(dbPrefix)] = db
 
 	return db, nil
 }
@@ -52,7 +46,7 @@ func (pdb *prefixDb) keyWithPrefix(key Key) Key {
 }
 
 func (pdb *prefixDb) keyWithoutPrefix(key Key) Key {
-	return key[len(pdb.prefix):]
+	return key[1:]
 }
 
 func (k Key) keyWithoutKeyPrefix(prefix KeyPrefix) Key {

packages/model/hornet/transaction.go

@@ -1,6 +1,8 @@
 package hornet
 
 import (
+	"time"
+
 	"github.com/iotaledger/iota.go/transaction"
 	"github.com/iotaledger/iota.go/trinary"
 
@@ -37,6 +39,9 @@ type Transaction struct {
 	// TxTimestamp or, if available, AttachmentTimestamp
 	timestamp int64
 
+	// Unix time when the Tx became solid (needed for local modifiers for tipselection)
+	solidificationTimestamp int32
+
 	// The index of the milestone which confirmed this tx
 	confirmationIndex milestone_index.MilestoneIndex
 
@@ -76,14 +81,15 @@ func NewTransactionFromGossip(transaction *transaction.Transaction, transactionB
 	}
 }
 
-func NewTransactionFromDatabase(transaction *transaction.Transaction, transactionBytes []byte, confirmationIndex milestone_index.MilestoneIndex, metadata byte) *Transaction {
+func NewTransactionFromDatabase(transaction *transaction.Transaction, transactionBytes []byte, solidificationTimestamp int32, confirmationIndex milestone_index.MilestoneIndex, metadata byte) *Transaction {
 	return &Transaction{
-		Tx:                transaction,
-		RawBytes:          transactionBytes,
-		timestamp:         getTimestampFromTx(transaction),
-		confirmationIndex: confirmationIndex,
-		metadata:          bitutils.BitMask(metadata),
-		modified:          false,
+		Tx:                      transaction,
+		RawBytes:                transactionBytes,
+		timestamp:               getTimestampFromTx(transaction),
+		solidificationTimestamp: solidificationTimestamp,
+		confirmationIndex:       confirmationIndex,
+		metadata:                bitutils.BitMask(metadata),
+		modified:                false,
 	}
 }
 
@@ -113,6 +119,10 @@ func (tx *Transaction) GetTimestamp() int64 {
 	return tx.timestamp
 }
 
+func (tx *Transaction) GetSolidificationTimestamp() int32 {
+	return tx.solidificationTimestamp
+}
+
 func (tx *Transaction) IsTail() bool {
 	return tx.Tx.CurrentIndex == 0
 }
@@ -133,6 +143,7 @@ func (tx *Transaction) SetSolid(solid bool) {
 	defer tx.metadataMutex.Unlock()
 
 	if solid != tx.metadata.HasFlag(HORNET_TX_METADATA_SOLID) {
+		tx.solidificationTimestamp = int32(time.Now().Unix())
 		tx.metadata = tx.metadata.ModifyingFlag(HORNET_TX_METADATA_SOLID, solid)
 		tx.SetModified(true)
 	}

packages/model/tangle/approvers_db.go

@@ -9,7 +9,7 @@ import (
 var approversDatabase database.Database
 
 func configureApproversDatabase() {
-	if db, err := database.Get("approvers"); err != nil {
+	if db, err := database.Get(DBPrefixApprovers); err != nil {
 		panic(err)
 	} else {
 		approversDatabase = db

packages/model/tangle/bundle_db.go

@@ -3,6 +3,7 @@ package tangle
 import (
 	"github.com/iotaledger/iota.go/trinary"
 	"github.com/pkg/errors"
+
 	"github.com/gohornet/hornet/packages/bitutils"
 	"github.com/gohornet/hornet/packages/database"
 	"github.com/gohornet/hornet/packages/model/hornet"
@@ -11,7 +12,7 @@ import (
 var bundleDatabase database.Database
 
 func configureBundleDatabase() {
-	if db, err := database.Get("bundle"); err != nil {
+	if db, err := database.Get(DBPrefixBundles); err != nil {
 		panic(err)
 	} else {
 		bundleDatabase = db

packages/model/tangle/database_prefixes.go

@@ -0,0 +1,14 @@
+package tangle
+
+const (
+	DBPrefixHealth                  byte = 0
+	DBPrefixTransactions            byte = 1
+	DBPrefixBundles                 byte = 2
+	DBPrefixAddresses               byte = 3
+	DBPrefixMilestones              byte = 4
+	DBPrefixLedgerState             byte = 5
+	DBPrefixApprovers               byte = 6
+	DBPrefixSnapshot                byte = 7
+	DBPrefixUnconfirmedTransactions byte = 8
+	DBPrefixSpentAddresses          byte = 9
+)

packages/model/tangle/health_db.go

@@ -2,20 +2,27 @@ package tangle
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/gohornet/hornet/packages/database"
 	"github.com/gohornet/hornet/packages/typeutils"
 )
 
+const (
+	DbVersion = 1
+)
+
 var (
 	healthDatabase database.Database
 )
 
 func configureHealthDatabase() {
-	if db, err := database.Get("health"); err != nil {
+	if db, err := database.Get(DBPrefixHealth); err != nil {
 		panic(err)
 	} else {
 		healthDatabase = db
 	}
+
+	setDatabaseVersion()
 }
 
 func MarkDatabaseCorrupted() {
@@ -43,3 +50,31 @@ func IsDatabaseCorrupted() bool {
 	}
 	return contains
 }
+
+func setDatabaseVersion() {
+	_, err := healthDatabase.Get(typeutils.StringToBytes("dbVersion"))
+	if err == database.ErrKeyNotFound {
+		// Only create the entry, if it doesn't exist already (fresh database)
+		if err := healthDatabase.Set(
+			database.Entry{
+				Key:   typeutils.StringToBytes("dbVersion"),
+				Value: []byte{DbVersion},
+			}); err != nil {
+			panic(errors.Wrap(NewDatabaseError(err), "failed to set database version"))
+		}
+	}
+}
+
+func IsCorrectDatabaseVersion() bool {
+
+	entry, err := healthDatabase.Get(typeutils.StringToBytes("dbVersion"))
+	if err != nil {
+		panic(errors.Wrap(NewDatabaseError(err), "failed to read database version"))
+	}
+
+	if len(entry.Value) > 0 {
+		return entry.Value[0] == DbVersion
+	}
+
+	return false
+}

packages/model/tangle/ledger_db.go

@@ -41,7 +41,7 @@ func WriteUnlockLedger() {
 }
 
 func configureLedgerDatabase() {
-	if db, err := database.Get("ledger"); err != nil {
+	if db, err := database.Get(DBPrefixLedgerState); err != nil {
 		panic(err)
 	} else {
 		ledgerDatabase = db

packages/model/tangle/milestones_db.go

@@ -13,7 +13,7 @@ import (
 var milestoneDatabase database.Database
 
 func configureMilestoneDatabase() {
-	if db, err := database.Get("ms"); err != nil {
+	if db, err := database.Get(DBPrefixMilestones); err != nil {
 		panic(err)
 	} else {
 		milestoneDatabase = db

packages/model/tangle/snapshot_db.go

@@ -14,7 +14,7 @@ import (
 var snapshotDatabase database.Database
 
 func configureSnapshotDatabase() {
-	if db, err := database.Get("ls"); err != nil {
+	if db, err := database.Get(DBPrefixSnapshot); err != nil {
 		panic(err)
 	} else {
 		snapshotDatabase = db

packages/model/tangle/spent_db.go

@@ -11,7 +11,7 @@ var (
 )
 
 func configureSpentAddressesDatabase() {
-	if db, err := database.Get("spent"); err != nil {
+	if db, err := database.Get(DBPrefixSpentAddresses); err != nil {
 		panic(err)
 	} else {
 		spentAddressesDatabase = db

packages/model/tangle/transaction_db.go

@@ -15,7 +15,7 @@ import (
 var transactionDatabase database.Database
 
 func configureTransactionDatabase() {
-	if db, err := database.Get("tx"); err != nil {
+	if db, err := database.Get(DBPrefixTransactions); err != nil {
 		panic(err)
 	} else {
 		transactionDatabase = db
@@ -41,14 +41,15 @@ func StoreTransactionsInDatabase(transactions []*hornet.Transaction) error {
 	var modifiedTx []*hornet.Transaction
 	for _, transaction := range transactions {
 		if transaction.IsModified() {
-			value := make([]byte, 4, 4+len(transaction.RawBytes))
+			value := make([]byte, 8, 8+len(transaction.RawBytes))
 			confirmed, confirmationIndex := transaction.GetConfirmed()
 
 			if !confirmed {
 				confirmationIndex = 0
 			}
 
 			binary.LittleEndian.PutUint32(value, uint32(confirmationIndex))
+			binary.LittleEndian.PutUint32(value[4:], uint32(transaction.GetSolidificationTimestamp()))
 			value = append(value, transaction.RawBytes...)
 
 			entry := database.Entry{
@@ -100,13 +101,14 @@ func readTransactionFromDatabase(transactionHash trinary.Hash) (*hornet.Transact
 	}
 
 	confirmationIndex := milestone_index.MilestoneIndex(binary.LittleEndian.Uint32(entry.Value[:4]))
-	rawBytes := entry.Value[4:]
+	solidificationTimestamp := int32(binary.LittleEndian.Uint32(entry.Value[4:8]))
+	rawBytes := entry.Value[8:]
 
 	tx, err := compressed.TransactionFromCompressedBytes(rawBytes, transactionHash)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to decompress tx")
 	} else {
-		return hornet.NewTransactionFromDatabase(tx, rawBytes, confirmationIndex, entry.Meta), nil
+		return hornet.NewTransactionFromDatabase(tx, rawBytes, solidificationTimestamp, confirmationIndex, entry.Meta), nil
 	}
 }
 

packages/model/tangle/tx_per_address.go

@@ -12,7 +12,7 @@ var (
 )
 
 func configureTransactionHashesForAddressDatabase() {
-	if db, err := database.Get("address"); err != nil {
+	if db, err := database.Get(DBPrefixAddresses); err != nil {
 		panic(err)
 	} else {
 		transactionsHashesForAddressDatabase = db

packages/model/tangle/unconfirmed_tx.go

@@ -3,6 +3,7 @@ package tangle
 import (
 	"github.com/iotaledger/iota.go/trinary"
 	"github.com/pkg/errors"
+
 	"github.com/gohornet/hornet/packages/database"
 	"github.com/gohornet/hornet/packages/model/milestone_index"
 )
@@ -12,7 +13,7 @@ var (
 )
 
 func configureUnconfirmedTransactionsDatabase() {
-	if db, err := database.Get("unconfirmed"); err != nil {
+	if db, err := database.Get(DBPrefixUnconfirmedTransactions); err != nil {
 		panic(err)
 	} else {
 		unconfirmedTransactionDatabase = db

plugins/cli/plugin.go

@@ -5,14 +5,15 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/iotaledger/hive.go/events"
 	"github.com/gohornet/hornet/packages/node"
+	"github.com/iotaledger/hive.go/events"
 	"github.com/iotaledger/hive.go/parameter"
 )
 
 var (
 	// AppVersion version number
-	AppVersion = "0.1.0"
+	AppVersion = "0.2.0"
+
 	// AppName app code name
 	AppName = "HORNET"
 )

plugins/tangle/plugin.go

@@ -40,6 +40,11 @@ func configure(plugin *node.Plugin) {
 	if tangle.IsDatabaseCorrupted() {
 		log.Panic("HORNET was not shut down correctly. Database is corrupted. Please delete the database folder and start with a new local snapshot.")
 	}
+
+	if !tangle.IsCorrectDatabaseVersion() {
+		log.Panic("HORNET database version mismatch. The database scheme was updated. Please delete the database folder and start with a new local snapshot.")
+	}
+
 	tangle.MarkDatabaseCorrupted()
 
 	tangle.ConfigureMilestones(