Map Server as a Service

Makefile

@@ -1,56 +1,37 @@
 .PHONY: all clean test policy_log
 
-all: build_policy_log build_integration_test build_benchmark
+all: build_mapserver build_ingest build_policy_log build_integration_test
 
 clean:
 	@rm -f bin/*
 
 test:
 	@go test ./...
+	@echo "Tests OK"
+
+integration: build_integration_test
+	@# ./bin/test_policylog_interaction
+	@# ./bin/test_smt
+	./bin/test_mapserver
+	@echo "All integration tests OK"
+
+build_mapserver:
+	@go build -o bin/mapserver ./cmd/mapserver/
+
+build_ingest:
+	@go build -o bin/ingest ./cmd/ingest/
 
 build_policy_log:
 	@go build -o bin/logserver_exec cmd/logserver/logserver_exec.go
 	@go build -o bin/logsigner_exec cmd/logsigner/logsigner_exec.go
 
-setup_db: create_log_database create_fpki_table
-
 create_fpki_schema_replace_old:
 	@./tools/create_schema.sh
 
-create_log_database:
-	@./scripts/reset_db/resetdb.sh
-
 build_integration_test:
-	@go build -o ./bin/test_policylog_interaction  ./tests/integration/policylog_interaction
-	@go build -o ./bin/test_domainowner_pca_policlog_interaction  ./tests/integration/domainowner_pca_policlog_interaction
-	@go build -o ./bin/test_mapserver  ./tests/integration/mapserver
-	@go build -o ./bin/test_smt  ./tests/integration/smt
-	@go build -o ./bin/test_grpc  ./tests/integration/grpc_test
+	@# @go build -o ./bin/test_policylog_interaction  ./tests/integration/policylog_interaction
+	@# @go build -o ./bin/test_smt  ./tests/integration/smt
+	@go build -o ./bin/test_mapserver ./tests/integration/mapserver/
 
 drop_cacheTable:
 	@mysql -u root -e "DROP TABLE map.deleteTest;"
-
-run_integration_test:
-	@./scripts/integration_tests.sh
-
-build_benchmark:
-	@go build -o ./bin/log_benchmark  ./tests/benchmark/logserver_benchmark
-	@go build -o ./bin/smt_benchmark  ./tests/benchmark/smt_benchmark
-	@go build -o ./bin/db_benchmark  ./tests/benchmark/db_benchmark
-	@go build -o ./bin/updater_benchmark  ./tests/benchmark/mapserver_benchmark/updater_benchmark
-	@go build -o ./bin/responder_benchmark  ./tests/benchmark/mapserver_benchmark/responder_benchmark
-
-run_log_benchmark:
-	@./scripts/log_benchmark.sh
-
-run_smt_benchmark:
-	@./bin/smt_benchmark
-
-run_db_benchmark:
-	@./bin/db_benchmark
-
-run_updater_benchmark:
-	@./bin/updater_benchmark
-
-run_responder_benchmark:
-	@./bin/responder_benchmark

README.md

@@ -1,5 +1,7 @@
 # FPKI
 
+TODO: This information is old and outdated.
+
 ## Features
 
 - Issuance and logging of RPC (Root Policy Certificate)

cmd/ingest/main.go

@@ -28,6 +28,7 @@ const (
 const (
 	CertificateColumn = 3
 	CertChainColumn   = 4
+	ExpirationColumn  = 7 // In seconds
 )
 
 // Times gathered at jupiter, 64 gz files, no CSV
@@ -135,7 +136,7 @@ func mainFunction() int {
 
 	// Now update the SMT Trie with the changed domains:
 	fmt.Println("Starting SMT update ...")
-	err = updater.UpdateSMT(ctx, conn, 32)
+	err = updater.UpdateSMT(ctx, conn)
 	exitIfError(err)
 	fmt.Println("Done SMT update.")
 

cmd/ingest/processor.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -172,6 +173,22 @@ func (p *Processor) ingestWithCSV(fileReader io.Reader) error {
 	reader.FieldsPerRecord = -1 // don't check number of fields
 
 	parseFunction := func(fields []string, lineNo int) error {
+		// First avoid even parsing already expired certs.
+		exp := strings.Split(fields[ExpirationColumn], ".")
+		if len(exp) != 2 {
+			return fmt.Errorf("unrecognized timestamp in 7th column: %s", fields[ExpirationColumn])
+		}
+		n, err := strconv.ParseInt(exp[0], 10, 64)
+		if err != nil {
+			return fmt.Errorf("parsing the expiration time \"%s\" got: %w",
+				fields[ExpirationColumn], err)
+		}
+		if p.now.After(time.Unix(n, 0)) {
+			// Skip this certificate.
+			return nil
+		}
+
+		// From this point on, we need to parse the certificate.
 		rawBytes, err := base64.StdEncoding.DecodeString(fields[CertificateColumn])
 		if err != nil {
 			return err

cmd/mapserver/config_sample.json

@@ -0,0 +1,26 @@
+{
+    "CTLogServerURLs": [
+        "https://ct.googleapis.com/logs/xenon2023/"
+    ],
+    "DBConfig": {
+        "Dsn": "",
+        "DBName": "",
+        "Values": {
+            "DBNAME": "fpki",
+            "MYSQL_HOST": "127.0.0.1",
+            "MYSQL_LOCALSOCKET": "/var/run/mysqld/mysqld.sock",
+            "MYSQL_PASSWORD": "",
+            "MYSQL_PORT": "",
+            "MYSQL_USER": "root",
+            "collation": "binary",
+            "interpolateParams": "true",
+            "maxAllowedPacket": "1073741824",
+            "parseTime": "true"
+        },
+        "CheckSchema": false
+    },
+    "CertificatePemFile": "tests/testdata/servercert.pem",
+    "PrivateKeyPemFile": "tests/testdata/serverkey.pem",
+    "UpdateAt": "03:00:00",
+    "UpdateTimer": "1d"
+}

cmd/mapserver/main.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/netsec-ethz/fpki/pkg/db"
+	"github.com/netsec-ethz/fpki/pkg/db/mysql"
+	"github.com/netsec-ethz/fpki/pkg/mapserver"
+	"github.com/netsec-ethz/fpki/pkg/mapserver/config"
+	"github.com/netsec-ethz/fpki/pkg/util"
+)
+
+const waitForExitBeforePanicTime = 10 * time.Second
+
+func main() {
+	os.Exit(mainFunc())
+}
+
+func mainFunc() int {
+	// Because some packages (glog) change the flags to main, and we don't want/need them, reset
+	// the flags before touching them.
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// Prepare our flags.
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage:\n%s configuration_file\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	updateVar := flag.Bool("updateNow", true, "Immediately trigger an update cycle")
+	createSampleConfig := flag.Bool("createSampleConfig", false,
+		"Create configuration file specified by positional argument")
+	flag.Parse()
+
+	// We need the configuration file as the first positional argument.
+	if flag.NArg() != 1 {
+		flag.Usage()
+		return 1
+	}
+
+	var err error
+	if *createSampleConfig {
+		err = writeSampleConfig()
+	} else {
+		err = run(*updateVar)
+	}
+
+	// We have finished. Probably the context created in run was been cancelled (exit request).
+	// Print message in case of error.
+	return manageError(err)
+}
+
+func writeSampleConfig() error {
+	dbConfig := db.NewConfig(
+		mysql.WithDefaults(),
+		mysql.WithEnvironment(),
+		mysql.WithLocalSocket("/var/run/mysqld/mysqld.sock"),
+	)
+	conf := &config.Config{
+		DBConfig:           dbConfig,
+		CTLogServerURLs:    []string{"https://ct.googleapis.com/logs/xenon2023/"},
+		CertificatePemFile: "tests/testdata/servercert.pem",
+		PrivateKeyPemFile:  "tests/testdata/serverkey.pem",
+
+		UpdateAt: util.NewTimeOfDay(3, 00, 00, 00),
+		UpdateTimer: util.DurationWrap{
+			Duration: 24 * time.Hour,
+		},
+	}
+
+	return config.WriteConfigurationToFile(flag.Arg(0), conf)
+}
+
+func run(updateNow bool) error {
+	ctx := context.Background()
+
+	// Set SIGTERM handler. The context we get is cancelled if one of those signals is caught.
+	ctx = util.SetSignalHandler(ctx, waitForExitBeforePanicTime, syscall.SIGTERM, syscall.SIGINT)
+
+	// Load configuration and run with it.
+	config, err := config.ReadConfigFromFile(flag.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	return runWithConfig(ctx, config, updateNow)
+}
+
+// runWithConfig examines the configuration, and according to its values, starts a timer to
+// run the update cycle at the corresponding time.
+func runWithConfig(
+	ctx context.Context,
+	conf *config.Config,
+	updateNow bool,
+) error {
+
+	server, err := mapserver.NewMapServer(ctx, conf)
+	if err != nil {
+		return err
+	}
+
+	// Should update now?
+	if updateNow {
+		err := server.PruneAndUpdate(ctx)
+		if err != nil {
+			return fmt.Errorf("performing initial update: %w", err)
+		}
+	}
+
+	// Set update cycle timer.
+	util.RunWhen(ctx, conf.UpdateAt.NextTimeOfDay(), conf.UpdateTimer.Duration,
+		func(ctx context.Context) {
+			err := server.PruneAndUpdate(ctx)
+			if err != nil {
+				fmt.Printf("ERROR: update returned %s\n", err)
+			}
+		})
+
+	// Listen in responder.
+	err = server.Listen(ctx)
+
+	// Regardless of the error, clean everything up.
+	cleanUp()
+
+	// Return the error from the responder.
+	return err
+}
+
+func cleanUp() {
+	fmt.Println("cleaning up")
+}
+
+func manageError(err error) int {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 1
+	}
+
+	fmt.Println("exiting")
+	return 0
+}

cmd/mapserver/mapserver.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Map Server Service. Updates automatically depending on configuration.
+Documentation=https://github.com/netsec-ethz/fpki
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=fpki
+Group=fpki
+ExecStart=/usr/bin/mapserver --config /etc/fpki/config.json
+Restart=always
+RestartSec=5            # wait 5 seconds if app crashes
+RemainAfterExit=False   # report status bad if process is not running
+KillMode=control-group
+
+[Install]
+WantedBy=multi-user.target

go.mod

@@ -12,7 +12,8 @@ require (
 	github.com/stretchr/testify v1.7.4
 	github.com/transparency-dev/merkle v0.0.1
 	go.uber.org/atomic v1.9.0
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
+	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
+	golang.org/x/net v0.1.0
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/grpc v1.47.0
 	google.golang.org/protobuf v1.28.0
@@ -105,12 +106,12 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/tools v0.2.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/api v0.77.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -934,6 +934,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
+golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU=
+golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -960,8 +962,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1024,8 +1026,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220421235706-1d1ef9303861/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1250,8 +1252,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=