From a61e6a85f695d198a2ac2e8c39479c1edc547c6c Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Sun, 5 May 2024 07:52:51 -0400 Subject: [PATCH 1/7] Added an archive mode toggle to Geth --- config/geth-config.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/config/geth-config.go b/config/geth-config.go index 5b40a1d..633ad57 100644 --- a/config/geth-config.go +++ b/config/geth-config.go @@ -22,6 +22,9 @@ type GethConfig struct { // Number of seconds EVM calls can run before timing out EvmTimeout Parameter[uint64] + // The archive mode flag + ArchiveMode Parameter[bool] + // The Docker Hub tag for Geth ContainerTag Parameter[string] @@ -41,7 +44,9 @@ func NewGethConfig() *GethConfig { CanBeBlank: false, OverwriteOnUpgrade: false, }, - Default: map[Network]uint16{Network_All: calculateGethPeers()}, + Default: map[Network]uint16{ + Network_All: calculateGethPeers(), + }, }, EvmTimeout: Parameter[uint64]{ @@ -53,7 +58,23 @@ func NewGethConfig() *GethConfig { CanBeBlank: false, OverwriteOnUpgrade: false, }, - Default: map[Network]uint64{Network_All: 5}, + Default: map[Network]uint64{ + Network_All: 5, + }, + }, + + ArchiveMode: Parameter[bool]{ + ParameterCommon: &ParameterCommon{ + ID: "archiveMode", + Name: "Enable Archive Mode", + Description: "When enabled, Geth will run in \"archive\" mode which means it can recreate the state of the chain for a previous block. This is required for manually generating the Merkle rewards tree.\n\nArchive mode takes several TB of disk space, so only enable it if you need it and can support it.", + AffectsContainers: []ContainerID{ContainerID_ExecutionClient}, + CanBeBlank: false, + OverwriteOnUpgrade: false, + }, + Default: map[Network]bool{ + Network_All: false, + }, }, ContainerTag: Parameter[string]{ @@ -97,6 +118,7 @@ func (cfg *GethConfig) GetParameters() []IParameter { return []IParameter{ &cfg.MaxPeers, &cfg.EvmTimeout, + &cfg.ArchiveMode, &cfg.ContainerTag, &cfg.AdditionalFlags, } From caca047fa19e5d08bb6d34a8b0a34447d0a7fead Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Sun, 5 May 2024 09:45:22 -0400 Subject: [PATCH 2/7] Added InvalidArgCountError --- utils/input/validation.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/utils/input/validation.go b/utils/input/validation.go index 3ac3d56..0b03dec 100644 --- a/utils/input/validation.go +++ b/utils/input/validation.go @@ -21,14 +21,34 @@ const ( MinPasswordLength int = 12 ) -// -// General types -// +// ============== +// === Errors === +// ============== + +type InvalidArgCountError struct { + ExpectedCount int + ActualCount int +} + +func (e *InvalidArgCountError) Error() string { + return fmt.Sprintf("Incorrect argument count - expected %d but have %d", e.ExpectedCount, e.ActualCount) +} + +func NewInvalidArgCountError(expectedCount int, actualCount int) *InvalidArgCountError { + return &InvalidArgCountError{ + ExpectedCount: expectedCount, + ActualCount: actualCount, + } +} + +// ================== +// === Validation === +// ================== // Validate command argument count -func ValidateArgCount(argCount int, expectedCount int) error { +func ValidateArgCount(argCount int, expectedCount int) *InvalidArgCountError { if argCount != expectedCount { - return fmt.Errorf("Incorrect argument count; expected %d but have %d", expectedCount, argCount) + return NewInvalidArgCountError(expectedCount, argCount) } return nil } From 5b9a3ae77d47fd23a288100ce8ce8afe21606da4 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Sun, 5 May 2024 10:42:32 -0400 Subject: [PATCH 3/7] Made a CLI package, imported arg count validation and index selection --- cli/input/validation.go | 35 ++++++++++ cli/utils/selection.go | 139 ++++++++++++++++++++++++++++++++++++++ utils/input/validation.go | 32 --------- 3 files changed, 174 insertions(+), 32 deletions(-) create mode 100644 cli/input/validation.go create mode 100644 cli/utils/selection.go diff --git a/cli/input/validation.go b/cli/input/validation.go new file mode 100644 index 0000000..fe8f7e6 --- /dev/null +++ b/cli/input/validation.go @@ -0,0 +1,35 @@ +package input + +import "fmt" + +// ============== +// === Errors === +// ============== + +type InvalidArgCountError struct { + ExpectedCount int + ActualCount int +} + +func (e *InvalidArgCountError) Error() string { + return fmt.Sprintf("Incorrect argument count - expected %d but have %d", e.ExpectedCount, e.ActualCount) +} + +func NewInvalidArgCountError(expectedCount int, actualCount int) *InvalidArgCountError { + return &InvalidArgCountError{ + ExpectedCount: expectedCount, + ActualCount: actualCount, + } +} + +// ================== +// === Validation === +// ================== + +// Validate command argument count +func ValidateArgCount(argCount int, expectedCount int) *InvalidArgCountError { + if argCount != expectedCount { + return NewInvalidArgCountError(expectedCount, argCount) + } + return nil +} diff --git a/cli/utils/selection.go b/cli/utils/selection.go new file mode 100644 index 0000000..c2d182d --- /dev/null +++ b/cli/utils/selection.go @@ -0,0 +1,139 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" +) + +// An option that can be selected from a list of choices in the CLI +type SelectionOption[DataType any] struct { + // The underlying element this option represents + Element *DataType + + // The human-readable ID of the option, used for non-interactive selection + ID string + + // The text to display for this option when listing the available options + Display string +} + +// Parse a comma-separated list of indices to select in a multi-index operation +func ParseIndexSelection[DataType any](selectionString string, options []SelectionOption[DataType]) ([]*DataType, error) { + // Select all + if selectionString == "" { + selectedElements := make([]*DataType, len(options)) + for i, option := range options { + selectedElements[i] = option.Element + } + return selectedElements, nil + } + + // Trim spaces + elements := strings.Split(selectionString, ",") + trimmedElements := make([]string, len(elements)) + for i, element := range elements { + trimmedElements[i] = strings.TrimSpace(element) + } + + // Process elements + optionLength := uint64(len(options)) + seenIndices := map[uint64]bool{} + selectedElements := []*DataType{} + for _, element := range trimmedElements { + before, after, found := strings.Cut(element, "-") + if !found { + // Handle non-ranges + index, err := strconv.ParseUint(element, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing index '%s': %w", element, err) + } + index-- + + // Make sure it's in the list of options + if index >= optionLength { + return nil, fmt.Errorf("selection '%s' is too large", element) + } + + // Add it if it's new + _, exists := seenIndices[index] + if !exists { + seenIndices[index] = true + selectedElements = append(selectedElements, options[index].Element) + } + } else { + // Handle ranges + start, err := strconv.ParseUint(before, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing range start in '%s': %w", element, err) + } + end, err := strconv.ParseUint(after, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing range end in '%s': %w", element, err) + } + start-- + end-- + + // Make sure the start and end are in the list of options + if end <= start { + return nil, fmt.Errorf("range end for '%s' is not greater than the start", element) + } + if start >= optionLength { + return nil, fmt.Errorf("range start for '%s' is too large", element) + } + if end >= optionLength { + return nil, fmt.Errorf("range end for '%s' is too large", element) + } + + // Add each index if it's new + for index := start; index <= end; index++ { + _, exists := seenIndices[index] + if !exists { + seenIndices[index] = true + selectedElements = append(selectedElements, options[index].Element) + } + } + } + } + return selectedElements, nil +} + +// Parse a comma-separated list of option IDs to select in a multi-index operation +func ParseOptionIDs[DataType any](selectionString string, options []SelectionOption[DataType]) ([]*DataType, error) { + elements := strings.Split(selectionString, ",") + + // Trim spaces + trimmedElements := make([]string, len(elements)) + for i, element := range elements { + trimmedElements[i] = strings.TrimSpace(element) + } + + // Remove duplicates + uniqueElements := make([]string, 0, len(elements)) + seenIndices := map[string]bool{} + for _, element := range trimmedElements { + _, exists := seenIndices[element] + if !exists { + uniqueElements = append(uniqueElements, element) + seenIndices[element] = true + } + } + + // Validate + selectedElements := make([]*DataType, len(uniqueElements)) + for i, element := range uniqueElements { + // Make sure it's in the list of options + found := false + for _, option := range options { + if option.ID == element { + found = true + selectedElements[i] = option.Element + break + } + } + if !found { + return nil, fmt.Errorf("element '%s' is not a valid option", element) + } + } + return selectedElements, nil +} diff --git a/utils/input/validation.go b/utils/input/validation.go index 0b03dec..c550c23 100644 --- a/utils/input/validation.go +++ b/utils/input/validation.go @@ -21,38 +21,6 @@ const ( MinPasswordLength int = 12 ) -// ============== -// === Errors === -// ============== - -type InvalidArgCountError struct { - ExpectedCount int - ActualCount int -} - -func (e *InvalidArgCountError) Error() string { - return fmt.Sprintf("Incorrect argument count - expected %d but have %d", e.ExpectedCount, e.ActualCount) -} - -func NewInvalidArgCountError(expectedCount int, actualCount int) *InvalidArgCountError { - return &InvalidArgCountError{ - ExpectedCount: expectedCount, - ActualCount: actualCount, - } -} - -// ================== -// === Validation === -// ================== - -// Validate command argument count -func ValidateArgCount(argCount int, expectedCount int) *InvalidArgCountError { - if argCount != expectedCount { - return NewInvalidArgCountError(expectedCount, argCount) - } - return nil -} - // Validate a comma-delimited batch of inputs func ValidateBatch[ReturnType any](name string, value string, validate func(string, string) (ReturnType, error)) ([]ReturnType, error) { elements := strings.Split(value, ",") From 48262707ff32f3d8a35ec94204e94bbbd546fc2f Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 6 May 2024 09:55:40 -0400 Subject: [PATCH 4/7] Made archiveMode a var --- config/geth-config.go | 2 +- config/ids/ids.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/geth-config.go b/config/geth-config.go index 633ad57..ad8dc5d 100644 --- a/config/geth-config.go +++ b/config/geth-config.go @@ -65,7 +65,7 @@ func NewGethConfig() *GethConfig { ArchiveMode: Parameter[bool]{ ParameterCommon: &ParameterCommon{ - ID: "archiveMode", + ID: ids.GethArchiveModeID, Name: "Enable Archive Mode", Description: "When enabled, Geth will run in \"archive\" mode which means it can recreate the state of the chain for a previous block. This is required for manually generating the Merkle rewards tree.\n\nArchive mode takes several TB of disk space, so only enable it if you need it and can support it.", AffectsContainers: []ContainerID{ContainerID_ExecutionClient}, diff --git a/config/ids/ids.go b/config/ids/ids.go index abc9086..c3200c7 100644 --- a/config/ids/ids.go +++ b/config/ids/ids.go @@ -50,7 +50,8 @@ const ( FallbackBnHttpUrlID string = "bnHttpUrl" // Geth - GethEvmTimeoutID string = "evmTimeout" + GethEvmTimeoutID string = "evmTimeout" + GethArchiveModeID string = "archiveMode" // Lighthouse LighthouseQuicPortID string = "p2pQuicPort" From 0d945a66e61571d81b0d201f64d338f25afcc5fa Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 6 May 2024 09:56:08 -0400 Subject: [PATCH 5/7] Created the network port client/server --- api/client/client.go | 60 ++++++------------ api/client/context.go | 46 -------------- api/client/net-context.go | 57 +++++++++++++++++ api/client/settings.go | 5 ++ api/client/types.go | 33 ++++++++++ api/client/unix-context.go | 71 +++++++++++++++++++++ api/server/net-server.go | 78 ++++++++++++++++++++++++ api/server/settings.go | 7 +++ api/server/types.go | 4 ++ api/server/{server.go => unix-server.go} | 19 ++---- 10 files changed, 279 insertions(+), 101 deletions(-) delete mode 100644 api/client/context.go create mode 100644 api/client/net-context.go create mode 100644 api/client/settings.go create mode 100644 api/client/types.go create mode 100644 api/client/unix-context.go create mode 100644 api/server/net-server.go create mode 100644 api/server/settings.go rename api/server/{server.go => unix-server.go} (82%) diff --git a/api/client/client.go b/api/client/client.go index 2a9cf55..a7c6a65 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -1,14 +1,11 @@ package client import ( - "errors" "fmt" "io" - "io/fs" "log/slog" "net/http" "net/url" - "os" "strconv" "strings" @@ -20,22 +17,6 @@ import ( "github.com/ethereum/go-ethereum/common" ) -const ( - jsonContentType string = "application/json" -) - -// IRequester is an interface for making HTTP requests to a specific subroute on the NMC server -type IRequester interface { - // The human-readable requester name (for logging) - GetName() string - - // The name of the subroute to send requests to - GetRoute() string - - // Context to hold settings and utilities the requester should use - GetContext() *RequesterContext -} - // Submit a GET request to the API server func SendGetRequest[DataType any](r IRequester, method string, requestName string, args map[string]string) (*types.ApiResponse[DataType], error) { if args == nil { @@ -49,15 +30,9 @@ func SendGetRequest[DataType any](r IRequester, method string, requestName strin } // Submit a GET request to the API server -func RawGetRequest[DataType any](context *RequesterContext, path string, params map[string]string) (*types.ApiResponse[DataType], error) { - // Make sure the socket exists - _, err := os.Stat(context.socketPath) - if errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("the socket at [%s] does not exist - please start the service and try again", context.socketPath) - } - +func RawGetRequest[DataType any](context IRequesterContext, path string, params map[string]string) (*types.ApiResponse[DataType], error) { // Create the request - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/%s", context.base, path), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", context.GetAddressBase(), path), nil) if err != nil { return nil, fmt.Errorf("error creating HTTP request: %w", err) } @@ -70,10 +45,10 @@ func RawGetRequest[DataType any](context *RequesterContext, path string, params req.URL.RawQuery = values.Encode() // Debug log - context.logger.Debug("API Request", slog.String(log.MethodKey, http.MethodGet), slog.String(log.QueryKey, req.URL.String())) + context.GetLogger().Debug("API Request", slog.String(log.MethodKey, http.MethodGet), slog.String(log.QueryKey, req.URL.String())) // Run the request - resp, err := context.client.Do(req) + resp, err := context.SendRequest(req) return HandleResponse[DataType](context, resp, path, err) } @@ -93,25 +68,28 @@ func SendPostRequest[DataType any](r IRequester, method string, requestName stri } // Submit a POST request to the API server -func RawPostRequest[DataType any](context *RequesterContext, path string, body string) (*types.ApiResponse[DataType], error) { - // Make sure the socket exists - _, err := os.Stat(context.socketPath) - if errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("the socket at [%s] does not exist - please start the service and try again", context.socketPath) +func RawPostRequest[DataType any](context IRequesterContext, path string, body string) (*types.ApiResponse[DataType], error) { + // Create the request + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", context.GetAddressBase(), path), strings.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("error creating HTTP request: %w", err) } + req.Header.Set("Content-Type", jsonContentType) // Debug log - context.logger.Debug("API Request", slog.String(log.MethodKey, http.MethodPost), slog.String(log.PathKey, path), slog.String(log.BodyKey, body)) + context.GetLogger().Debug("API Request", slog.String(log.MethodKey, http.MethodPost), slog.String(log.PathKey, path), slog.String(log.BodyKey, body)) - resp, err := context.client.Post(fmt.Sprintf("http://%s/%s", context.base, path), jsonContentType, strings.NewReader(body)) + // Run the request + resp, err := context.SendRequest(req) return HandleResponse[DataType](context, resp, path, err) } // Processes a response to a request -func HandleResponse[DataType any](context *RequesterContext, resp *http.Response, path string, err error) (*types.ApiResponse[DataType], error) { +func HandleResponse[DataType any](context IRequesterContext, resp *http.Response, path string, err error) (*types.ApiResponse[DataType], error) { if err != nil { return nil, fmt.Errorf("error requesting %s: %w", path, err) } + logger := context.GetLogger() // Read the body defer resp.Body.Close() @@ -122,7 +100,7 @@ func HandleResponse[DataType any](context *RequesterContext, resp *http.Response // Handle 404s specially since they won't have a JSON body if resp.StatusCode == http.StatusNotFound { - context.logger.Debug("API Response (raw)", slog.String(log.CodeKey, resp.Status), slog.String(log.BodyKey, string(bytes))) + logger.Debug("API Response (raw)", slog.String(log.CodeKey, resp.Status), slog.String(log.BodyKey, string(bytes))) return nil, fmt.Errorf("route '%s' not found", path) } @@ -130,18 +108,18 @@ func HandleResponse[DataType any](context *RequesterContext, resp *http.Response var parsedResponse types.ApiResponse[DataType] err = json.Unmarshal(bytes, &parsedResponse) if err != nil { - context.logger.Debug("API Response (raw)", slog.String(log.CodeKey, resp.Status), slog.String(log.BodyKey, string(bytes))) + logger.Debug("API Response (raw)", slog.String(log.CodeKey, resp.Status), slog.String(log.BodyKey, string(bytes))) return nil, fmt.Errorf("error deserializing response to %s: %w", path, err) } // Check if the request failed if resp.StatusCode != http.StatusOK { - context.logger.Debug("API Response", slog.String(log.PathKey, path), slog.String(log.CodeKey, resp.Status), slog.String("err", parsedResponse.Error)) + logger.Debug("API Response", slog.String(log.PathKey, path), slog.String(log.CodeKey, resp.Status), slog.String("err", parsedResponse.Error)) return nil, fmt.Errorf(parsedResponse.Error) } // Debug log - context.logger.Debug("API Response", slog.String(log.BodyKey, string(bytes))) + logger.Debug("API Response", slog.String(log.BodyKey, string(bytes))) return &parsedResponse, nil } diff --git a/api/client/context.go b/api/client/context.go deleted file mode 100644 index 3e9bba0..0000000 --- a/api/client/context.go +++ /dev/null @@ -1,46 +0,0 @@ -package client - -import ( - "context" - "log/slog" - "net" - "net/http" -) - -// The context passed into a requester -type RequesterContext struct { - // The path to the socket to send requests to - socketPath string - - // An HTTP client for sending requests - client *http.Client - - // Logger to print debug messages to - logger *slog.Logger - - // The base route for the client to send requests to (//) - base string -} - -// Creates a new API client requester context -func NewRequesterContext(baseRoute string, socketPath string, log *slog.Logger) *RequesterContext { - requesterContext := &RequesterContext{ - socketPath: socketPath, - base: baseRoute, - logger: log, - client: &http.Client{ - Transport: &http.Transport{ - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return net.Dial("unix", socketPath) - }, - }, - }, - } - - return requesterContext -} - -// Set the logger for the context -func (r *RequesterContext) SetLogger(logger *slog.Logger) { - r.logger = logger -} diff --git a/api/client/net-context.go b/api/client/net-context.go new file mode 100644 index 0000000..5a9e7b7 --- /dev/null +++ b/api/client/net-context.go @@ -0,0 +1,57 @@ +package client + +import ( + "context" + "log/slog" + "net" + "net/http" +) + +// The context passed into a requester +type NetworkRequesterContext struct { + // The base address of the server + address string + + // An HTTP client for sending requests + client *http.Client + + // Logger to print debug messages to + logger *slog.Logger +} + +// Creates a new API client requester context for network-based servers +func NewNetworkRequesterContext(address string, log *slog.Logger) *NetworkRequesterContext { + requesterContext := &NetworkRequesterContext{ + address: address, + logger: log, + client: &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("tcp", address) + }, + }, + }, + } + + return requesterContext +} + +// Get the base of the address used for submitting server requests +func (r *NetworkRequesterContext) GetAddressBase() string { + return r.address +} + +// Get the logger for the context +func (r *NetworkRequesterContext) GetLogger() *slog.Logger { + return r.logger +} + +// Set the logger for the context +func (r *NetworkRequesterContext) SetLogger(logger *slog.Logger) { + r.logger = logger +} + +// Send an HTTP request to the server +func (r *NetworkRequesterContext) SendRequest(request *http.Request) (*http.Response, error) { + return r.client.Do(request) +} diff --git a/api/client/settings.go b/api/client/settings.go new file mode 100644 index 0000000..925aecc --- /dev/null +++ b/api/client/settings.go @@ -0,0 +1,5 @@ +package client + +const ( + jsonContentType string = "application/json" +) diff --git a/api/client/types.go b/api/client/types.go new file mode 100644 index 0000000..b738d46 --- /dev/null +++ b/api/client/types.go @@ -0,0 +1,33 @@ +package client + +import ( + "log/slog" + "net/http" +) + +// IRequester is an interface for making HTTP requests to a specific subroute on the NMC server +type IRequester interface { + // The human-readable requester name (for logging) + GetName() string + + // The name of the subroute to send requests to + GetRoute() string + + // Context to hold settings and utilities the requester should use + GetContext() IRequesterContext +} + +// IRequester is an interface for making HTTP requests to a specific subroute on the NMC server +type IRequesterContext interface { + // Get the base of the address used for submitting server requests + GetAddressBase() string + + // Get the logger for the context + GetLogger() *slog.Logger + + // Set the logger for the context + SetLogger(*slog.Logger) + + // Send an HTTP request to the server + SendRequest(request *http.Request) (*http.Response, error) +} diff --git a/api/client/unix-context.go b/api/client/unix-context.go new file mode 100644 index 0000000..ebe8438 --- /dev/null +++ b/api/client/unix-context.go @@ -0,0 +1,71 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "net" + "net/http" + "os" +) + +// The context passed into a requester +type UnixRequesterContext struct { + // The path to the socket to send requests to + socketPath string + + // An HTTP client for sending requests + client *http.Client + + // Logger to print debug messages to + logger *slog.Logger + + // The base route for the client to send requests to (//) + base string +} + +// Creates a new API client requester context +func NewUnixRequesterContext(baseRoute string, socketPath string, log *slog.Logger) *UnixRequesterContext { + requesterContext := &UnixRequesterContext{ + socketPath: socketPath, + base: baseRoute, + logger: log, + client: &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + }, + }, + } + + return requesterContext +} + +// Get the base of the address used for submitting server requests +func (r *UnixRequesterContext) GetAddressBase() string { + return fmt.Sprintf("http://%s", r.base) +} + +// Get the logger for the context +func (r *UnixRequesterContext) GetLogger() *slog.Logger { + return r.logger +} + +// Set the logger for the context +func (r *UnixRequesterContext) SetLogger(logger *slog.Logger) { + r.logger = logger +} + +// Send an HTTP request to the server +func (r *UnixRequesterContext) SendRequest(request *http.Request) (*http.Response, error) { + // Make sure the socket exists + _, err := os.Stat(r.socketPath) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("the socket at [%s] does not exist - please start the service and try again", r.socketPath) + } + + return r.client.Do(request) +} diff --git a/api/server/net-server.go b/api/server/net-server.go new file mode 100644 index 0000000..088cccd --- /dev/null +++ b/api/server/net-server.go @@ -0,0 +1,78 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "sync" + + "github.com/gorilla/mux" + "github.com/rocket-pool/node-manager-core/log" +) + +type NetworkSocketApiServer struct { + logger *slog.Logger + handlers []IHandler + port uint16 + socket net.Listener + server http.Server + router *mux.Router +} + +func NewNetworkSocketApiServer(logger *slog.Logger, port uint16, handlers []IHandler, baseRoute string, apiVersion string) (*NetworkSocketApiServer, error) { + // Create the router + router := mux.NewRouter() + + // Create the manager + server := &NetworkSocketApiServer{ + logger: logger, + handlers: handlers, + port: port, + router: router, + server: http.Server{ + Handler: router, + }, + } + + // Register each route + nmcRouter := router.Host(baseRoute).PathPrefix("/api/v" + apiVersion).Subrouter() + for _, handler := range server.handlers { + handler.RegisterRoutes(nmcRouter) + } + + return server, nil +} + +// Starts listening for incoming HTTP requests +func (s *NetworkSocketApiServer) Start(wg *sync.WaitGroup) error { + // Create the socket + socket, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", s.port)) + if err != nil { + return fmt.Errorf("error creating socket: %w", err) + } + s.socket = socket + + // Start listening + wg.Add(1) + go func() { + err := s.server.Serve(socket) + if !errors.Is(err, http.ErrServerClosed) { + s.logger.Error("error while listening for HTTP requests", log.Err(err)) + } + wg.Done() + }() + + return nil +} + +// Stops the HTTP listener +func (s *NetworkSocketApiServer) Stop() error { + err := s.server.Shutdown(context.Background()) + if err != nil { + return fmt.Errorf("error stopping listener: %w", err) + } + return nil +} diff --git a/api/server/settings.go b/api/server/settings.go new file mode 100644 index 0000000..1d3cfd2 --- /dev/null +++ b/api/server/settings.go @@ -0,0 +1,7 @@ +package server + +import "github.com/fatih/color" + +const ( + ApiLogColor color.Attribute = color.FgHiBlue +) diff --git a/api/server/types.go b/api/server/types.go index 158f5f4..4789bae 100644 --- a/api/server/types.go +++ b/api/server/types.go @@ -6,3 +6,7 @@ import "github.com/gorilla/mux" type IContextFactory interface { RegisterRoute(router *mux.Router) } + +type IHandler interface { + RegisterRoutes(router *mux.Router) +} diff --git a/api/server/server.go b/api/server/unix-server.go similarity index 82% rename from api/server/server.go rename to api/server/unix-server.go index 020155e..86c2a20 100644 --- a/api/server/server.go +++ b/api/server/unix-server.go @@ -12,20 +12,11 @@ import ( "path/filepath" "sync" - "github.com/fatih/color" "github.com/gorilla/mux" "github.com/rocket-pool/node-manager-core/log" ) -const ( - ApiLogColor color.Attribute = color.FgHiBlue -) - -type IHandler interface { - RegisterRoutes(router *mux.Router) -} - -type ApiServer struct { +type UnixSocketApiServer struct { logger *slog.Logger handlers []IHandler socketPath string @@ -34,12 +25,12 @@ type ApiServer struct { router *mux.Router } -func NewApiServer(logger *slog.Logger, socketPath string, handlers []IHandler, baseRoute string, apiVersion string) (*ApiServer, error) { +func NewUnixSocketApiServer(logger *slog.Logger, socketPath string, handlers []IHandler, baseRoute string, apiVersion string) (*UnixSocketApiServer, error) { // Create the router router := mux.NewRouter() // Create the manager - server := &ApiServer{ + server := &UnixSocketApiServer{ logger: logger, handlers: handlers, socketPath: socketPath, @@ -66,7 +57,7 @@ func NewApiServer(logger *slog.Logger, socketPath string, handlers []IHandler, b } // Starts listening for incoming HTTP requests -func (s *ApiServer) Start(wg *sync.WaitGroup, socketOwnerUid uint32, socketOwnerGid uint32) error { +func (s *UnixSocketApiServer) Start(wg *sync.WaitGroup, socketOwnerUid uint32, socketOwnerGid uint32) error { // Remove the socket if it's already there _, err := os.Stat(s.socketPath) if !errors.Is(err, fs.ErrNotExist) { @@ -109,7 +100,7 @@ func (s *ApiServer) Start(wg *sync.WaitGroup, socketOwnerUid uint32, socketOwner } // Stops the HTTP listener -func (s *ApiServer) Stop() error { +func (s *UnixSocketApiServer) Stop() error { err := s.server.Shutdown(context.Background()) if err != nil { return fmt.Errorf("error stopping listener: %w", err) From c2883a64174bd8673e1e7ffd1325ad022e1ccde5 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 6 May 2024 17:46:39 -0400 Subject: [PATCH 6/7] Cleaned up client and server args, added http trace support --- api/client/net-context.go | 26 ++++++++++++++++++-------- api/server/net-server.go | 9 ++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/api/client/net-context.go b/api/client/net-context.go index 5a9e7b7..e2d3e92 100644 --- a/api/client/net-context.go +++ b/api/client/net-context.go @@ -5,29 +5,36 @@ import ( "log/slog" "net" "net/http" + "net/http/httptrace" + "net/url" ) // The context passed into a requester type NetworkRequesterContext struct { - // The base address of the server - address string + // The base address and route for API calls + apiUrl *url.URL // An HTTP client for sending requests client *http.Client // Logger to print debug messages to logger *slog.Logger + + // Tracer for HTTP requests + tracer *httptrace.ClientTrace } -// Creates a new API client requester context for network-based servers -func NewNetworkRequesterContext(address string, log *slog.Logger) *NetworkRequesterContext { +// Creates a new API client requester context for network-based +// traceOpts is optional. If nil, it will not be used. +func NewNetworkRequesterContext(apiUrl *url.URL, log *slog.Logger, tracer *httptrace.ClientTrace) *NetworkRequesterContext { requesterContext := &NetworkRequesterContext{ - address: address, - logger: log, + apiUrl: apiUrl, + logger: log, + tracer: tracer, client: &http.Client{ Transport: &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return net.Dial("tcp", address) + return net.Dial("tcp", apiUrl.Host) }, }, }, @@ -38,7 +45,7 @@ func NewNetworkRequesterContext(address string, log *slog.Logger) *NetworkReques // Get the base of the address used for submitting server requests func (r *NetworkRequesterContext) GetAddressBase() string { - return r.address + return r.apiUrl.String() } // Get the logger for the context @@ -53,5 +60,8 @@ func (r *NetworkRequesterContext) SetLogger(logger *slog.Logger) { // Send an HTTP request to the server func (r *NetworkRequesterContext) SendRequest(request *http.Request) (*http.Response, error) { + if r.tracer != nil { + request = request.WithContext(httptrace.WithClientTrace(request.Context(), r.tracer)) + } return r.client.Do(request) } diff --git a/api/server/net-server.go b/api/server/net-server.go index 088cccd..1e83c44 100644 --- a/api/server/net-server.go +++ b/api/server/net-server.go @@ -16,13 +16,14 @@ import ( type NetworkSocketApiServer struct { logger *slog.Logger handlers []IHandler + ip string port uint16 socket net.Listener server http.Server router *mux.Router } -func NewNetworkSocketApiServer(logger *slog.Logger, port uint16, handlers []IHandler, baseRoute string, apiVersion string) (*NetworkSocketApiServer, error) { +func NewNetworkSocketApiServer(logger *slog.Logger, ip string, port uint16, handlers []IHandler, baseRoute string, apiVersion string) (*NetworkSocketApiServer, error) { // Create the router router := mux.NewRouter() @@ -30,6 +31,7 @@ func NewNetworkSocketApiServer(logger *slog.Logger, port uint16, handlers []IHan server := &NetworkSocketApiServer{ logger: logger, handlers: handlers, + ip: ip, port: port, router: router, server: http.Server{ @@ -38,7 +40,8 @@ func NewNetworkSocketApiServer(logger *slog.Logger, port uint16, handlers []IHan } // Register each route - nmcRouter := router.Host(baseRoute).PathPrefix("/api/v" + apiVersion).Subrouter() + //router.GetRoute().Host() + nmcRouter := router.PathPrefix("/" + baseRoute + "/api/v" + apiVersion).Subrouter() for _, handler := range server.handlers { handler.RegisterRoutes(nmcRouter) } @@ -49,7 +52,7 @@ func NewNetworkSocketApiServer(logger *slog.Logger, port uint16, handlers []IHan // Starts listening for incoming HTTP requests func (s *NetworkSocketApiServer) Start(wg *sync.WaitGroup) error { // Create the socket - socket, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", s.port)) + socket, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.ip, s.port)) if err != nil { return fmt.Errorf("error creating socket: %w", err) } From 5d763793fe04199576e79685677c11b277446890 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 7 May 2024 12:26:19 -0400 Subject: [PATCH 7/7] Migrated CI from HD and SN --- .github/workflows/commits.yml | 14 ++++++++++ .github/workflows/lint.yml | 46 ++++++++++++++++++++++++++++++++ .github/workflows/unit-tests.yml | 19 +++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 .github/workflows/commits.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/commits.yml b/.github/workflows/commits.yml new file mode 100644 index 0000000..1c9df19 --- /dev/null +++ b/.github/workflows/commits.yml @@ -0,0 +1,14 @@ +# Taken from https://github.com/marketplace/actions/block-fixup-commit-merge?version=v2.0.0 +# Updated to use newer ubuntu and checkout action +name: Git Checks + +on: [pull_request] + +jobs: + block-fixup: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Block Fixup Commit Merge + uses: 13rac1/block-fixup-merge-action@v2.0.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7f7a6e0 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,46 @@ +# Taken from https://github.com/golangci/golangci-lint-action +name: golangci-lint +on: + push: + tags: + - v* + branches: + - main + pull_request: +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v4 + with: + go-version: 1.21.8 + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: latest + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # args: --issues-exit-code=0 + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + # skip-cache: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..1ee6cbc --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,19 @@ +name: NMC Unit Tests +on: + push: + tags: + - v* + branches: + - main + pull_request: +permissions: + contents: read +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: 1.21.8 + - run: go test ./...