Merge pull request #35 from cloudflare/feature/filter

Filter and assertion implementation (#34)

Makefile

@@ -24,6 +24,7 @@ vet:
 .PHONY: test
 test:
 	go test -v github.com/cloudflare/gortr/lib
+	go test -v github.com/cloudflare/gortr/prefixfile
 
 .PHONY: prepare
 prepare:

README.md

@@ -188,12 +188,72 @@ And to configure a bypass for every SSH key:
 $ ./gortr -ssh.bind :8282 -ssh.key private.pem -ssh.method.key=true -ssh.auth.key.bypass=true -bind ""
 ```
 
+## Configure filters and overrides (SLURM)
+
+GoRTR supports SLURM configuration files ([RFC8416](https://tools.ietf.org/html/rfc8416)).
+
+Create a json file (`slurm.json`):
+
+```
+{
+    "slurmVersion": 1,
+    "validationOutputFilters": {
+     "prefixFilters": [
+       {
+        "prefix": "10.0.0.0/8",
+        "comment": "Everything inside will be removed"
+       },
+       {
+        "asn": 65001,
+       },
+       {
+        "asn": 65002,
+        "prefix": "192.168.0.0/24",
+       },
+     ],
+     "bgpsecFilters": []
+    },
+    "locallyAddedAssertions": {
+     "prefixAssertions": [
+       {
+        "asn": 65001,
+        "prefix": "2001:db8::/32",
+        "maxPrefixLength": 48,
+        "comment": "Manual add"
+       }
+     ],
+     "bgpsecAssertions": [
+     ]
+    }
+  }
+```
+
+When starting GoRTR, add the `-slurm ./slurm.json` argument.
+
+The log should display something similar to the following:
+
+```
+INFO[0001] Slurm filtering: 112214 kept, 159 removed, 1 asserted
+INFO[0002] New update (112215 uniques, 112215 total prefixes).
+```
+
+For instance, if the original JSON fetched contains the ROA: `10.0.0.0/24-24 AS65001`,
+it will be removed.
+
+The JSON exported by GoRTR will contain the overrides and the file can be signed again.
+Others GoRTR can be configured to fetch the ROAs from the filtering GoRTR:
+the operator manages one SLURM file on a leader GoRTR.
+
 ## Debug the content
 
+You can check the content provided over RTR with rtrdump tool
+
 ```bash
 $ ./rtrdump -connect 127.0.0.1:8282 -file debug.json
 ```
 
+You can also fetch the re-generated JSON from the `-export.path` endpoint (default: `http://localhost:8080/rpki.json`)
+
 ### Data sources
 
 Use your own validator, as long as the JSON source follows the following schema:

cmd/gortr/gortr.go

@@ -26,6 +26,7 @@ import (
 	"os/signal"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -46,7 +47,11 @@ var (
 
 	MetricsAddr = flag.String("metrics.addr", ":8080", "Metrics address")
 	MetricsPath = flag.String("metrics.path", "/metrics", "Metrics path")
-	RTRVersion  = flag.Int("protocol", 1, "RTR protocol version")
+
+	ExportPath = flag.String("export.path", "/rpki.json", "Export path")
+	ExportSign = flag.String("export.sign", "", "Sign export with key")
+
+	RTRVersion = flag.Int("protocol", 1, "RTR protocol version")
 
 	Bind = flag.String("bind", ":8282", "Bind address")
 
@@ -75,6 +80,9 @@ var (
 	MaxConn         = flag.Int("maxconn", 0, "Max simultaneous connections (0 to disable limit)")
 	SendNotifs      = flag.Bool("notifications", true, "Send notifications to clients")
 
+	Slurm        = flag.String("slurm", "", "Slurm configuration file (filters and assertions)")
+	SlurmRefresh = flag.Bool("slurm.refresh", true, "Refresh along the cache")
+
 	LogLevel = flag.String("loglevel", "info", "Log level")
 	Version  = flag.Bool("version", false, "Print version")
 
@@ -201,15 +209,15 @@ func decodeJSON(data []byte) (*prefixfile.ROAList, error) {
 	return &roalistjson, err
 }
 
-func processData(roalistjson *prefixfile.ROAList) ([]rtr.ROA, int, int, int) {
+func processData(roalistjson []prefixfile.ROAJson) ([]rtr.ROA, int, int, int) {
 	filterDuplicates := make(map[string]bool)
 
 	roalist := make([]rtr.ROA, 0)
 
 	var count int
 	var countv4 int
 	var countv6 int
-	for _, v := range roalistjson.Data {
+	for _, v := range roalistjson {
 		prefix, err := v.GetPrefix2()
 		if err != nil {
 			log.Error(err)
@@ -300,7 +308,37 @@ func (s *state) updateFile(file string) error {
 		log.Debugf("Signature verified")
 	}
 
-	roas, count, countv4, countv6 := processData(roalistjson)
+	roasjson := roalistjson.Data
+	if s.slurm != nil {
+		kept, removed := s.slurm.FilterOnROAs(roasjson)
+		asserted := s.slurm.AssertROAs()
+		log.Infof("Slurm filtering: %v kept, %v removed, %v asserted", len(kept), len(removed), len(asserted))
+		roasjson = append(kept, asserted...)
+	}
+	s.lockJson.Lock()
+	s.exported = prefixfile.ROAList{
+		Metadata: prefixfile.MetaData{
+			Counts:    len(roasjson),
+			Generated: roalistjson.Metadata.Generated,
+			Valid:     roalistjson.Metadata.Valid,
+			/*Signature:     roalistjson.Metadata.Signature,
+			SignatureDate: roalistjson.Metadata.SignatureDate,*/
+		},
+		Data: roasjson,
+	}
+
+	if s.key != nil {
+		signdate, sign, err := s.exported.Sign(s.key)
+		if err != nil {
+			log.Error(err)
+		}
+		s.exported.Metadata.Signature = sign
+		s.exported.Metadata.SignatureDate = signdate
+	}
+
+	s.lockJson.Unlock()
+
+	roas, count, countv4, countv6 := processData(roasjson)
 	if err != nil {
 		return err
 	}
@@ -333,8 +371,26 @@ func (s *state) updateFile(file string) error {
 	return nil
 }
 
-func (s *state) routineUpdate(file string, interval int) {
-	log.Debugf("Starting refresh routine (file: %v, interval: %vs)", file, interval)
+func (s *state) updateSlurm(file string) error {
+	log.Debugf("Refreshing slurm from %v", file)
+	data, err := fetchFile(file, s.userAgent)
+	if err != nil {
+		log.Error(err)
+		return err
+	}
+
+	buf := bytes.NewBuffer(data)
+
+	slurm, err := prefixfile.DecodeJSONSlurm(buf)
+	if err != nil {
+		return err
+	}
+	s.slurm = slurm
+	return nil
+}
+
+func (s *state) routineUpdate(file string, interval int, slurmFile string) {
+	log.Debugf("Starting refresh routine (file: %v, interval: %vs, slurm: %v)", file, interval, slurmFile)
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGHUP)
 	for {
@@ -345,6 +401,12 @@ func (s *state) routineUpdate(file string, interval int) {
 			log.Debug("Received HUP signal")
 		}
 		delay.Stop()
+		if slurmFile != "" {
+			err := s.updateSlurm(slurmFile)
+			if err != nil {
+				log.Errorf("Slurm: %v", err)
+			}
+		}
 		err := s.updateFile(file)
 		if err != nil {
 			switch err.(type) {
@@ -357,6 +419,14 @@ func (s *state) routineUpdate(file string, interval int) {
 	}
 }
 
+func (s *state) exporter(wr http.ResponseWriter, r *http.Request) {
+	s.lockJson.RLock()
+	toExport := s.exported
+	s.lockJson.RUnlock()
+	enc := json.NewEncoder(wr)
+	enc.Encode(toExport)
+}
+
 type state struct {
 	lastdata      []byte
 	lastconverted []byte
@@ -369,6 +439,12 @@ type state struct {
 
 	metricsEvent *metricsEvent
 
+	exported prefixfile.ROAList
+	lockJson *sync.RWMutex
+	key      *ecdsa.PrivateKey
+
+	slurm *prefixfile.SlurmConfig
+
 	pubkey    *ecdsa.PublicKey
 	verify    bool
 	checktime bool
@@ -420,6 +496,19 @@ func ReadPublicKey(key []byte, isPem bool) (*ecdsa.PublicKey, error) {
 	return kconv, nil
 }
 
+func ReadKey(key []byte, isPem bool) (*ecdsa.PrivateKey, error) {
+	if isPem {
+		block, _ := pem.Decode(key)
+		key = block.Bytes
+	}
+
+	k, err := x509.ParseECPrivateKey(key)
+	if err != nil {
+		return nil, err
+	}
+	return k, nil
+}
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 
@@ -443,10 +532,11 @@ func main() {
 	}
 
 	var me *metricsEvent
+	var enableHTTP bool
 	if *MetricsAddr != "" {
 		initMetrics()
-		go metricHTTP()
 		me = &metricsEvent{}
+		enableHTTP = true
 	}
 
 	server := rtr.NewServer(sc, me, deh)
@@ -473,6 +563,31 @@ func main() {
 		verify:       *Verify,
 		checktime:    *TimeCheck,
 		userAgent:    *UserAgent,
+		lockJson:     &sync.RWMutex{},
+	}
+
+	if *ExportSign != "" {
+		keyFile, err := os.Open(*ExportSign)
+		if err != nil {
+			log.Fatal(err)
+		}
+		keyBytes, err := ioutil.ReadAll(keyFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		keyFile.Close()
+		keyDec, err := ReadKey(keyBytes, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+		s.key = keyDec
+	}
+
+	if enableHTTP {
+		if *ExportPath != "" {
+			http.HandleFunc(*ExportPath, s.exporter)
+		}
+		go metricHTTP()
 	}
 
 	if *Bind == "" && *BindTLS == "" && *BindSSH == "" {
@@ -589,6 +704,17 @@ func main() {
 		}()
 	}
 
+	slurmFile := *Slurm
+	if slurmFile != "" {
+		err := s.updateSlurm(slurmFile)
+		if err != nil {
+			log.Errorf("Slurm: %v", err)
+		}
+		if !*SlurmRefresh {
+			slurmFile = ""
+		}
+	}
+
 	err := s.updateFile(*CacheBin)
 	if err != nil {
 		switch err.(type) {
@@ -598,6 +724,6 @@ func main() {
 			log.Errorf("Error updating: %v", err)
 		}
 	}
-	s.routineUpdate(*CacheBin, *RefreshInterval)
+	s.routineUpdate(*CacheBin, *RefreshInterval, slurmFile)
 
 }

prefixfile/prefixfile.go

@@ -145,3 +145,11 @@ func (roa *ROAJson) GetMaxLen() int {
 func (roa *ROAJson) String() string {
 	return fmt.Sprintf("%v/%v/%v", roa.Prefix, roa.Length, roa.ASN)
 }
+
+func GetIPBroadcast(ipnet net.IPNet) net.IP {
+	br := make([]byte, len(ipnet.IP))
+	for i := 0; i < len(ipnet.IP); i++ {
+		br[i] = ipnet.IP[i] | (^ipnet.Mask[i])
+	}
+	return net.IP(br)
+}