Map Server as a Service (#44)

* New mapserver command that starts the mapserver.
* MapResponder created with RSA key argument.
* LogFetcher can retrieve the size of the CT log server.
* Map udater has a NextBatch() and GetNextBatch methods.
* Connect in testdb takes a tests.T
* Store and retrieve last index in DB.
* Mapserver updater can continue from last go.
* MapServer uses same timer to update and prune.
* Add Prune certs to DB.
* Add mapserver systemd service file.
* Add tool to fix the permissions of mysql.
* Add the STH as part of the server state.
* Multiple fetchers per updater.
* MapServer integration test.
* Add a benchmark for the MapServer responder API.
* Added benchmark for GetPayloads too.
* Skip parsing X509 certs if CSV indicates cert is expired.
* API point to obtain policy payloads.

Makefile

@@ -1,56 +1,37 @@
 .PHONY: all clean test policy_log
 
-all: build_policy_log build_integration_test build_benchmark
+all: build_mapserver build_ingest build_policy_log build_integration_test
 
 clean:
 	@rm -f bin/*
 
 test:
 	@go test ./...
+	@echo "Tests OK"
+
+integration: build_integration_test
+	@# ./bin/test_policylog_interaction
+	@# ./bin/test_smt
+	./bin/test_mapserver
+	@echo "All integration tests OK"
+
+build_mapserver:
+	@go build -o bin/mapserver ./cmd/mapserver/
+
+build_ingest:
+	@go build -o bin/ingest ./cmd/ingest/
 
 build_policy_log:
 	@go build -o bin/logserver_exec cmd/logserver/logserver_exec.go
 	@go build -o bin/logsigner_exec cmd/logsigner/logsigner_exec.go
 
-setup_db: create_log_database create_fpki_table
-
 create_fpki_schema_replace_old:
 	@./tools/create_schema.sh
 
-create_log_database:
-	@./scripts/reset_db/resetdb.sh
-
 build_integration_test:
-	@go build -o ./bin/test_policylog_interaction  ./tests/integration/policylog_interaction
-	@go build -o ./bin/test_domainowner_pca_policlog_interaction  ./tests/integration/domainowner_pca_policlog_interaction
-	@go build -o ./bin/test_mapserver  ./tests/integration/mapserver
-	@go build -o ./bin/test_smt  ./tests/integration/smt
-	@go build -o ./bin/test_grpc  ./tests/integration/grpc_test
+	@# @go build -o ./bin/test_policylog_interaction  ./tests/integration/policylog_interaction
+	@# @go build -o ./bin/test_smt  ./tests/integration/smt
+	@go build -o ./bin/test_mapserver ./tests/integration/mapserver/
 
 drop_cacheTable:
 	@mysql -u root -e "DROP TABLE map.deleteTest;"
-
-run_integration_test:
-	@./scripts/integration_tests.sh
-
-build_benchmark:
-	@go build -o ./bin/log_benchmark  ./tests/benchmark/logserver_benchmark
-	@go build -o ./bin/smt_benchmark  ./tests/benchmark/smt_benchmark
-	@go build -o ./bin/db_benchmark  ./tests/benchmark/db_benchmark
-	@go build -o ./bin/updater_benchmark  ./tests/benchmark/mapserver_benchmark/updater_benchmark
-	@go build -o ./bin/responder_benchmark  ./tests/benchmark/mapserver_benchmark/responder_benchmark
-
-run_log_benchmark:
-	@./scripts/log_benchmark.sh
-
-run_smt_benchmark:
-	@./bin/smt_benchmark
-
-run_db_benchmark:
-	@./bin/db_benchmark
-
-run_updater_benchmark:
-	@./bin/updater_benchmark
-
-run_responder_benchmark:
-	@./bin/responder_benchmark

README.md

@@ -1,5 +1,7 @@
 # FPKI
 
+TODO: This information is old and outdated.
+
 ## Features
 
 - Issuance and logging of RPC (Root Policy Certificate)

cmd/ingest/main.go

@@ -135,7 +135,7 @@ func mainFunction() int {
 
 	// Now update the SMT Trie with the changed domains:
 	fmt.Println("Starting SMT update ...")
-	err = updater.UpdateSMT(ctx, conn, 32)
+	err = updater.UpdateSMT(ctx, conn)
 	exitIfError(err)
 	fmt.Println("Done SMT update.")
 

cmd/ingest/processor.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -172,6 +173,17 @@ func (p *Processor) ingestWithCSV(fileReader io.Reader) error {
 	reader.FieldsPerRecord = -1 // don't check number of fields
 
 	parseFunction := func(fields []string, lineNo int) error {
+		// First avoid even parsing already expired certs.
+		n, err := getExpiration(fields)
+		if err != nil {
+			return err
+		}
+		if p.now.After(time.Unix(n, 0)) {
+			// Skip this certificate.
+			return nil
+		}
+
+		// From this point on, we need to parse the certificate.
 		rawBytes, err := base64.StdEncoding.DecodeString(fields[CertificateColumn])
 		if err != nil {
 			return err
@@ -277,3 +289,22 @@ func (p *Processor) processErrorChannel() error {
 	}
 	return nil
 }
+
+// getExpiration returns the expiration time in seconds. It is stored already in seconds on the
+// last column of the CSV entry, usually index 7.
+func getExpiration(fields []string) (int64, error) {
+	// Because some entries in the CSVs are malformed by not escaping their SAN field, we cannot
+	// reliably use a column index, but the last column of the entry.
+	expirationColumn := len(fields) - 1
+
+	s := strings.Split(fields[expirationColumn], ".")
+	if len(s) != 2 {
+		return 0, fmt.Errorf("unrecognized timestamp in the last column: %s", fields[expirationColumn])
+	}
+	exp, err := strconv.ParseInt(s[0], 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parsing the expiration time \"%s\" got: %w",
+			fields[expirationColumn], err)
+	}
+	return exp, nil
+}

cmd/mapserver/config_sample.json

@@ -0,0 +1,26 @@
+{
+    "CTLogServerURLs": [
+        "https://ct.googleapis.com/logs/xenon2023/"
+    ],
+    "DBConfig": {
+        "Dsn": "",
+        "DBName": "",
+        "Values": {
+            "DBNAME": "fpki",
+            "MYSQL_HOST": "127.0.0.1",
+            "MYSQL_LOCALSOCKET": "/var/run/mysqld/mysqld.sock",
+            "MYSQL_PASSWORD": "",
+            "MYSQL_PORT": "",
+            "MYSQL_USER": "root",
+            "collation": "binary",
+            "interpolateParams": "true",
+            "maxAllowedPacket": "1073741824",
+            "parseTime": "true"
+        },
+        "CheckSchema": false
+    },
+    "CertificatePemFile": "tests/testdata/servercert.pem",
+    "PrivateKeyPemFile": "tests/testdata/serverkey.pem",
+    "UpdateAt": "03:00:00",
+    "UpdateTimer": "1d"
+}

cmd/mapserver/main.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/netsec-ethz/fpki/pkg/db"
+	"github.com/netsec-ethz/fpki/pkg/db/mysql"
+	"github.com/netsec-ethz/fpki/pkg/mapserver"
+	"github.com/netsec-ethz/fpki/pkg/mapserver/config"
+	"github.com/netsec-ethz/fpki/pkg/util"
+)
+
+const waitForExitBeforePanicTime = 10 * time.Second
+
+func main() {
+	os.Exit(mainFunc())
+}
+
+func mainFunc() int {
+	// Because some packages (glog) change the flags to main, and we don't want/need them, reset
+	// the flags before touching them.
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// Prepare our flags.
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage:\n%s configuration_file\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	updateVar := flag.Bool("updateNow", true, "Immediately trigger an update cycle")
+	createSampleConfig := flag.Bool("createSampleConfig", false,
+		"Create configuration file specified by positional argument")
+	flag.Parse()
+
+	// We need the configuration file as the first positional argument.
+	if flag.NArg() != 1 {
+		flag.Usage()
+		return 1
+	}
+
+	var err error
+	if *createSampleConfig {
+		err = writeSampleConfig()
+	} else {
+		err = run(*updateVar)
+	}
+
+	// We have finished. Probably the context created in run was been cancelled (exit request).
+	// Print message in case of error.
+	return manageError(err)
+}
+
+func writeSampleConfig() error {
+	dbConfig := db.NewConfig(
+		mysql.WithDefaults(),
+		mysql.WithEnvironment(),
+		mysql.WithLocalSocket("/var/run/mysqld/mysqld.sock"),
+	)
+	conf := &config.Config{
+		DBConfig:           dbConfig,
+		CTLogServerURLs:    []string{"https://ct.googleapis.com/logs/xenon2023/"},
+		CertificatePemFile: "tests/testdata/servercert.pem",
+		PrivateKeyPemFile:  "tests/testdata/serverkey.pem",
+
+		UpdateAt: util.NewTimeOfDay(3, 00, 00, 00),
+		UpdateTimer: util.DurationWrap{
+			Duration: 24 * time.Hour,
+		},
+	}
+
+	return config.WriteConfigurationToFile(flag.Arg(0), conf)
+}
+
+func run(updateNow bool) error {
+	ctx := context.Background()
+
+	// Set SIGTERM handler. The context we get is cancelled if one of those signals is caught.
+	ctx = util.SetSignalHandler(ctx, waitForExitBeforePanicTime, syscall.SIGTERM, syscall.SIGINT)
+
+	// Load configuration and run with it.
+	config, err := config.ReadConfigFromFile(flag.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	return runWithConfig(ctx, config, updateNow)
+}
+
+// runWithConfig examines the configuration, and according to its values, starts a timer to
+// run the update cycle at the corresponding time.
+func runWithConfig(
+	ctx context.Context,
+	conf *config.Config,
+	updateNow bool,
+) error {
+
+	server, err := mapserver.NewMapServer(ctx, conf)
+	if err != nil {
+		return err
+	}
+
+	// Should update now?
+	if updateNow {
+		err := server.PruneAndUpdate(ctx)
+		if err != nil {
+			return fmt.Errorf("performing initial update: %w", err)
+		}
+	}
+
+	// Set update cycle timer.
+	util.RunWhen(ctx, conf.UpdateAt.NextTimeOfDay(), conf.UpdateTimer.Duration,
+		func(ctx context.Context) {
+			err := server.PruneAndUpdate(ctx)
+			if err != nil {
+				fmt.Printf("ERROR: update returned %s\n", err)
+			}
+		})
+
+	// Listen in responder.
+	err = server.Listen(ctx)
+
+	// Regardless of the error, clean everything up.
+	cleanUp()
+
+	// Return the error from the responder.
+	return err
+}
+
+func cleanUp() {
+	fmt.Println("cleaning up")
+}
+
+func manageError(err error) int {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 1
+	}
+
+	fmt.Println("exiting")
+	return 0
+}

cmd/mapserver/mapserver.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Map Server Service. Updates automatically depending on configuration.
+Documentation=https://github.com/netsec-ethz/fpki
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=fpki
+Group=fpki
+ExecStart=/usr/bin/mapserver --config /etc/fpki/config.json
+Restart=always
+RestartSec=5            # wait 5 seconds if app crashes
+RemainAfterExit=False   # report status bad if process is not running
+KillMode=control-group
+
+[Install]
+WantedBy=multi-user.target

go.mod

@@ -12,7 +12,8 @@ require (
 	github.com/stretchr/testify v1.7.4
 	github.com/transparency-dev/merkle v0.0.1
 	go.uber.org/atomic v1.9.0
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
+	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
+	golang.org/x/net v0.1.0
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/grpc v1.47.0
 	google.golang.org/protobuf v1.28.0
@@ -105,12 +106,12 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/tools v0.2.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/api v0.77.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect