Merge pull request #32 from ngrok-oss/euan/auto-fd-cleanup

Close all old file descriptors on update

README.md

@@ -49,7 +49,7 @@ import (
 	"time"
 
 	"github.com/inconshreveable/log15"
-	"github.com/ngrok-oss/tableroll/v2"
+	"github.com/ngrok-oss/tableroll/v3"
 )
 
 func main() {

fds.go

@@ -2,14 +2,14 @@ package tableroll
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"os"
 	"sync"
 	"syscall"
 
 	"github.com/inconshreveable/log15"
-	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -88,6 +88,11 @@ type fd struct {
 	// The underlying file object
 	file *file
 
+	// used is set to true the first time this file is used.
+	// It's here to close inherited FDs which are unused in
+	// the new process automatically.
+	used bool
+
 	Kind fdKind `json:"kind"`
 	// ID is the id of this file, stored just for pretty-printing
 	ID string `json:"id"`
@@ -185,13 +190,13 @@ func (f *Fds) Listen(ctx context.Context, id string, cfg *net.ListenConfig, netw
 
 	ln, err = cfg.Listen(ctx, network, addr)
 	if err != nil {
-		return nil, errors.Wrap(err, "can't create new listener")
+		return nil, fmt.Errorf("can't create new listener: %w", err)
 	}
 
 	fdLn, ok := ln.(Listener)
 	if !ok {
 		ln.Close()
-		return nil, errors.Errorf("%T doesn't implement tableroll.Listener", ln)
+		return nil, fmt.Errorf("%T doesn't implement tableroll.Listener", ln)
 	}
 
 	err = f.addListenerLocked(id, network, addr, fdLn)
@@ -231,7 +236,7 @@ func (f *Fds) ListenWith(id, network, addr string, listenerFunc func(network, ad
 	}
 	if _, ok := ln.(Listener); !ok {
 		ln.Close()
-		return nil, errors.Errorf("%T doesn't implement tableroll.Listener", ln)
+		return nil, fmt.Errorf("%T doesn't implement tableroll.Listener", ln)
 	}
 	if err := f.addListenerLocked(id, network, addr, ln.(Listener)); err != nil {
 		ln.Close()
@@ -248,7 +253,8 @@ func (f *Fds) Listener(id string) (net.Listener, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 
-	return f.listenerLocked(id)
+	ln, err := f.listenerLocked(id)
+	return ln, err
 }
 
 func (f *Fds) listenerLocked(id string) (net.Listener, error) {
@@ -259,8 +265,10 @@ func (f *Fds) listenerLocked(id string) (net.Listener, error) {
 
 	ln, err := net.FileListener(file.file.File)
 	if err != nil {
-		return nil, errors.Wrapf(err, "can't inherit listener %s", file.file)
+		return nil, fmt.Errorf("can't inherit listener %s: %w", file.file, err)
 	}
+	// now that we've converted this into a go listener, assume it's used
+	file.used = true
 	return ln, nil
 }
 
@@ -302,7 +310,7 @@ func (f *Fds) DialWith(id, network, address string, dialFn func(network, address
 	fdConn, ok := newConn.(Conn)
 	if !ok {
 		newConn.Close()
-		return nil, errors.Errorf("%T doesn't implement tableroll.Conn", newConn)
+		return nil, fmt.Errorf("%T doesn't implement tableroll.Conn", newConn)
 	}
 
 	if err := f.addConnLocked(id, fdKindConn, network, address, fdConn); err != nil {
@@ -331,21 +339,22 @@ func (f *Fds) connLocked(id string) (net.Conn, error) {
 
 	conn, err := net.FileConn(file.file.File)
 	if err != nil {
-		return nil, errors.Wrapf(err, "can't inherit connection %s", file.file)
+		return nil, fmt.Errorf("can't inherit connection %s: %w", file.file, err)
 	}
 	return conn, nil
 }
 
 func (f *Fds) addConnLocked(id string, kind fdKind, network, addr string, conn syscall.Conn) error {
 	fdObj := &fd{
+		used:    true,
 		Kind:    kind,
 		ID:      id,
 		Network: network,
 		Addr:    addr,
 	}
 	file, err := dupConn(conn, fdObj.String())
 	if err != nil {
-		return errors.Wrapf(err, "can't dup listener %s %s", network, addr)
+		return fmt.Errorf("can't dup listener %s %s: %w", network, addr, err)
 	}
 	fdObj.file = file
 	f.fds[id] = fdObj
@@ -381,6 +390,7 @@ func (f *Fds) OpenFileWith(id string, name string, openFunc func(name string) (*
 	}
 
 	newFd := &fd{
+		used: true,
 		ID:   id,
 		Kind: fdKindFile,
 		file: dup,
@@ -413,7 +423,7 @@ func (f *Fds) Remove(id string) error {
 
 	item, ok := f.fds[id]
 	if !ok {
-		return errors.Errorf("no element in map with id %v", id)
+		return fmt.Errorf("no element in map with id %v", id)
 	}
 	delete(f.fds, id)
 	if item.file != nil {
@@ -449,6 +459,22 @@ func (f *Fds) copy() map[string]*fd {
 	return files
 }
 
+// closeUnused closes unused FDs. It should be called
+// while Fds is locked.
+func (f *Fds) closeUnused() error {
+	var errs []error
+	for name, fd := range f.fds {
+		if !fd.used {
+			err := fd.file.Close()
+			if err != nil {
+				errs = append(errs, fmt.Errorf("error closing %v: %w", name, err))
+			}
+			delete(f.fds, name)
+		}
+	}
+	return errors.Join(errs...)
+}
+
 func dupConn(conn syscall.Conn, name string) (*file, error) {
 	// Use SyscallConn instead of File to avoid making the original
 	// fd non-blocking.
@@ -463,15 +489,15 @@ func dupConn(conn syscall.Conn, name string) (*file, error) {
 		dup, duperr = dupFd(fd, name)
 	})
 	if err != nil {
-		return nil, errors.Wrap(err, "can't access fd")
+		return nil, fmt.Errorf("can't access fd: %w", err)
 	}
 	return dup, duperr
 }
 
 func dupFd(fd uintptr, name string) (*file, error) {
 	dupfd, _, errno := unix.Syscall(unix.SYS_FCNTL, fd, unix.F_DUPFD_CLOEXEC, 0)
 	if errno != 0 {
-		return nil, errors.Wrap(errno, "can't dup fd using fcntl")
+		return nil, fmt.Errorf("can't dup fd using fcntl: %v", errno)
 	}
 
 	return newFile(dupfd, name), nil

go.mod

@@ -1,4 +1,4 @@
-module github.com/ngrok-oss/tableroll/v2
+module github.com/ngrok-oss/tableroll/v3
 
 require (
 	github.com/euank/filelock v0.0.0-20200318073246-6ea232a62104

sibling.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/inconshreveable/log15"
-	"github.com/ngrok-oss/tableroll/v2/internal/proto"
+	"github.com/ngrok-oss/tableroll/v3/internal/proto"
 	"github.com/pkg/errors"
 )
 

transfer_owner.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 
 	"github.com/inconshreveable/log15"
-	"github.com/ngrok-oss/tableroll/v2/internal/proto"
+	"github.com/ngrok-oss/tableroll/v3/internal/proto"
 	"github.com/pkg/errors"
 )
 

upgrader.go

@@ -238,6 +238,12 @@ func (u *Upgrader) Ready() error {
 	if err := u.state.transitionTo(upgraderStateOwner); err != nil {
 		return err
 	}
+
+	// Now cleanup all old FDs while holding the lock
+	u.Fds.lockMutations(errors.New("closing old listeners"))
+	defer u.Fds.unlockMutations()
+	_ = u.Fds.closeUnused()
+
 	return nil
 }