Proxy record

README.md

@@ -26,13 +26,13 @@ Killgrave is a simulator for HTTP-based APIs, in simple words a **Mock Server**,
     * [Compile by yourself](#compile-by-yourself)
     * [Other](#other)
 - [Getting Started](#getting-started)
-    * [Using Killgrave by command line](#using-killgrave-by-command-line)
+    * [Using Killgrave from the command line](#using-killgrave-from-the-command-line)
     * [Using Killgrave by config file](#using-killgrave-by-config-file)
     * [Configure CORS](#configure-cors)
-    * [Prepare Killgrave for Proxy Mode](#prepare-killgrave-for-proxy-mode)
-    * [Create an Imposter](#create-an-imposter)
+    * [Preparing Killgrave for Proxy Mode](#preparing-killgrave-for-proxy-mode)
+    * [Creating an Imposter](#creating-an-imposter)
     * [Imposters structure](#imposters-structure)
-    * [Create an Imposter using regex](#create-an-imposter-using-regex)
+    * [Regex in the headers](#regex-in-the-headers)
     * [Create an imposter using JSON Schema](#create-an-imposter-using-json-schema)
     * [Create an imposter with delay](#create-an-imposter-with-delay)
     * [Create an imposter with dynamic responses](#create-an-imposter-with-dynamic-responses)
@@ -159,6 +159,8 @@ $ killgrave -h
         proxy mode you can choose between (all, missing or none) (default "none")
   -proxy-url string
         proxy url, you need to choose a proxy-mode
+  - record-file-path string
+      the file path where the imposters will be recorded if you selected proxy-mode record
   -version
         show the _version of the application
   -watcher
@@ -179,8 +181,8 @@ port: 3000
 host: "localhost"
 proxy:
   url: https://example.com
-  mode: missing
-watcher: true
+  record_file_path: "imposters/record.imp.json"
+  mode: record
 cors:
   methods: ["GET"]
   headers: ["Content-Type"]
@@ -256,6 +258,7 @@ In the CORS section of the file you can find the following options:
 You can use Killgrave in proxy mode using the flags `proxy-mode` and `proxy-url` or their equivalent fields in the configuration file. The following three proxy modes are available:
 * `none`: Default. Killgrave will not behave as a proxy and the mock server will only use the configured imposters.
 * `missing`: With this mode the mock server will try to match the request with a configured imposter, but if no matching endpoint was found, the mock server will call to the real server, declared in the `proxy-url` configuration variable.
+* `record`: This mode works as the same of the `missing` mode but with one exception, when `Killgrave` not match with some `URL` will record into a `record imposter`. You will need to declare `record-file-path` to indicate the path where you want to record the imposters (i.e. `imposters/record.imp.json`)
 * `all`: The mock server will always call to the real server, declared in the `proxy-url` configuration variable.
 
 The `proxy-url` must be the root path of the proxied server. For example, if we have an API running on `http://example.com/things`, the `proxy-url` will be `http://example.com`.

internal/app/cmd/http/http.go

@@ -19,18 +19,17 @@ import (
 )
 
 const (
-	_defaultHost      = "localhost"
-	_defaultPort      = 3000
-	_defaultProxyMode = killgrave.ProxyNone
-	_defaultStrictSlash   = true
+	_defaultHost        = "localhost"
+	_defaultPort        = 3000
+	_defaultProxyMode   = killgrave.ProxyNone
+	_defaultStrictSlash = true
 )
 
 var (
 	errGetDataFromImpostersFlag = errors.New("error trying to get data from imposters flag")
 	errGetDataFromHostFlag      = errors.New("error trying to get data from host flag")
 	errGetDataFromPortFlag      = errors.New("error trying to get data from port flag")
 	errGetDataFromSecureFlag    = errors.New("error trying to get data from secure flag")
-	errMandatoryURL             = errors.New("the field url is mandatory if you selected a proxy mode")
 )
 
 // NewHTTPCmd returns cobra.Command to run http sub command, this command will be used to run the mock server
@@ -59,8 +58,9 @@ func NewHTTPCmd() *cobra.Command {
 	cmd.PersistentFlags().IntP("port", "P", _defaultPort, "Port to run the server")
 	cmd.PersistentFlags().BoolP("watcher", "w", false, "File watcher will reload the server on each file change")
 	cmd.PersistentFlags().BoolP("secure", "s", false, "Run mock server using TLS (https)")
-	cmd.Flags().StringP("proxy", "p", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none")
+	cmd.Flags().StringP("proxy", "p", _defaultProxyMode.String(), "Proxy mode, the options are all, missing, record or none")
 	cmd.Flags().StringP("url", "u", "", "The url where the proxy will redirect to")
+	cmd.Flags().StringP("record-file-path", "o", "", "The record file path when the proxy is on record mode")
 
 	return cmd
 }
@@ -101,7 +101,8 @@ func runServer(cfg killgrave.Config) server.Server {
 		Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router),
 	}
 
-	proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.Proxy.Mode)
+	recorder := server.NewRecorder(cfg.Proxy.RecordFilePath)
+	proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.ImpostersPath, cfg.Proxy.Mode, recorder)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -183,17 +184,8 @@ func configureProxyMode(cmd *cobra.Command, cfg *killgrave.Config) error {
 		return err
 	}
 
-	var url string
-	if mode != killgrave.ProxyNone.String() {
-		url, err = cmd.Flags().GetString("url")
-		if err != nil {
-			return err
-		}
+	url, _ := cmd.Flags().GetString("url")
+	recordFilePath, _ := cmd.Flags().GetString("record-file-path")
 
-		if url == "" {
-			return errMandatoryURL
-		}
-	}
-	cfg.ConfigureProxy(pMode, url)
-	return nil
+	return cfg.ConfigureProxy(pMode, url, recordFilePath)
 }

internal/config.go

@@ -10,6 +10,11 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+var (
+	errMandatoryURL            = errors.New("the field url is mandatory if you selected a proxy mode")
+	errMandatoryRecordFilePath = errors.New("the field outputRecordFile is mandatory if you selected a proxy record")
+)
+
 // Config representation of config file yaml
 type Config struct {
 	ImpostersPath string      `yaml:"imposters_path"`
@@ -32,8 +37,9 @@ type ConfigCORS struct {
 
 // ConfigProxy is a representation of section proxy of the yaml
 type ConfigProxy struct {
-	Url  string    `yaml:"url"`
-	Mode ProxyMode `yaml:"mode"`
+	Url            string    `yaml:"url"`
+	Mode           ProxyMode `yaml:"mode"`
+	RecordFilePath string    `yaml:"record_file_path"`
 }
 
 // ProxyMode is enumeration of proxy server modes
@@ -44,6 +50,8 @@ const (
 	ProxyNone ProxyMode = iota
 	// ProxyMissing handle only missing requests are proxied
 	ProxyMissing
+	// ProxyRecord proxy the missing requests and record them into imposter
+	ProxyRecord
 	// ProxyAll all requests are proxied
 	ProxyAll
 )
@@ -55,10 +63,15 @@ var (
 	errInvalidPort        = errors.New("invalid port")
 )
 
+func (p ProxyMode) Is(mode ProxyMode) bool {
+	return p == mode
+}
+
 func (p ProxyMode) String() string {
 	m := map[ProxyMode]string{
 		ProxyNone:    "none",
 		ProxyMissing: "missing",
+		ProxyRecord:  "record",
 		ProxyAll:     "all",
 	}
 
@@ -74,6 +87,7 @@ func StringToProxyMode(t string) (ProxyMode, error) {
 	m := map[string]ProxyMode{
 		"none":    ProxyNone,
 		"missing": ProxyMissing,
+		"record":  ProxyRecord,
 		"all":     ProxyAll,
 	}
 
@@ -102,9 +116,24 @@ func (p *ProxyMode) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // ConfigureProxy preparing the server with the proxy configuration that the user has indicated
-func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) {
+func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL, recordFilePath string) error {
+	if proxyMode.Is(ProxyNone) {
+		return nil
+	}
+
+	if proxyURL == "" {
+		return errMandatoryURL
+	}
+
+	if proxyMode.Is(ProxyRecord) && recordFilePath == "" {
+		return errMandatoryRecordFilePath
+	}
+
 	cfg.Proxy.Mode = proxyMode
 	cfg.Proxy.Url = proxyURL
+	cfg.Proxy.RecordFilePath = recordFilePath
+
+	return nil
 }
 
 // ConfigOpt function to encapsulate optional parameters
@@ -151,6 +180,11 @@ func NewConfigFromFile(cfgPath string) (Config, error) {
 		return Config{}, fmt.Errorf("%w: error while unmarshalling configFile file %s, using default configuration instead", err, cfgPath)
 	}
 
+	recordFilePath := path.Join(path.Dir(cfgPath), cfg.Proxy.RecordFilePath)
+	if err := cfg.ConfigureProxy(cfg.Proxy.Mode, cfg.Proxy.Url, recordFilePath); err != nil {
+		return Config{}, err
+	}
+
 	cfg.ImpostersPath = path.Join(path.Dir(cfgPath), cfg.ImpostersPath)
 
 	return cfg, nil

internal/config_test.go

@@ -64,6 +64,7 @@ func TestProxyModeUnmarshal(t *testing.T) {
 	}{
 		"valid mode all":     {"all", ProxyAll, false},
 		"valid mode missing": {"missing", ProxyMissing, false},
+		"valid mode record":  {"record", ProxyRecord, false},
 		"valid mode none":    {"none", ProxyNone, false},
 		"empty mode":         {"", ProxyNone, true},
 		"invalid mode":       {"nonsens23e", ProxyNone, true},
@@ -122,10 +123,15 @@ func TestProxyMode_String(t *testing.T) {
 			"none",
 		},
 		{
-			"ProxyNone must be return missing string",
+			"ProxyMissing must be return missing string",
 			ProxyMissing,
 			"missing",
 		},
+		{
+			"ProxyRecord must be return record string",
+			ProxyRecord,
+			"record",
+		},
 		{
 			"ProxyNone must be return all string",
 			ProxyAll,
@@ -211,19 +217,43 @@ func TestNewConfig(t *testing.T) {
 }
 
 func TestConfig_ConfigureProxy(t *testing.T) {
-	expected := Config{
-		ImpostersPath: "imposters",
-		Port:          80,
-		Host:          "localhost",
-		Proxy: ConfigProxy{
-			Url:  "https://friendsofgo.tech",
-			Mode: ProxyAll,
-		},
+	tests := []struct{
+		name string
+		url string
+		mode ProxyMode
+		err error
+	}{
+		{"valid proxy configuration", "https://friendsofgo.tech", ProxyAll, nil},
+		{"none proxy configuration", "", ProxyNone, nil},
+		{"invalid proxy configuration", "", ProxyAll, errMandatoryURL},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T){
+			cfg, err := NewConfig("imposters", "localhost", 80, false)
+			assert.NoError(t, err)
+
+			err = cfg.ConfigureProxy(tt.mode, tt.url, "")
+			assert.Equal(t, tt.err, err)
+		})
 	}
+}
 
-	got, err := NewConfig("imposters", "localhost", 80, false)
-	assert.NoError(t, err)
+func TestConfig_ConfigureProxyRecord(t *testing.T) {
+	tests := []struct {
+		name           string
+		recordFilePath string
+		err            error
+	}{
+		{"valid configuration for Proxy Record", "some_file.imp.json", nil},
+		{"invalid configuration for Proxy Record", "", errMandatoryRecordFilePath},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, err := NewConfig("imposters", "localhost", 80, false)
+			assert.NoError(t, err)
 
-	got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech")
-	assert.Equal(t, expected, got)
+			err = cfg.ConfigureProxy(ProxyRecord, "https://friendsofgo.tech", tt.recordFilePath)
+			assert.Equal(t, tt.err, err)
+		})
+	}
 }

internal/server/http/imposter.go

@@ -58,18 +58,18 @@ func (i *Imposter) CalculateFilePath(filePath string) string {
 type Request struct {
 	Method     string             `json:"method"`
 	Endpoint   string             `json:"endpoint"`
-	SchemaFile *string            `json:"schemaFile"`
-	Params     *map[string]string `json:"params"`
-	Headers    *map[string]string `json:"headers"`
+	SchemaFile *string            `json:"schemaFile,omitempty"`
+	Params     *map[string]string `json:"params,omitempty"`
+	Headers    *map[string]string `json:"headers,omitempty"`
 }
 
 // Response represent the structure of real response
 type Response struct {
 	Status   int                `json:"status"`
 	Body     string             `json:"body"`
-	BodyFile *string            `json:"bodyFile" yaml:"bodyFile"`
-	Headers  *map[string]string `json:"headers"`
-	Delay    ResponseDelay      `json:"delay" yaml:"delay"`
+	BodyFile *string            `json:"bodyFile,omitempty" yaml:"bodyFile,omitempty"`
+	Headers  *map[string]string `json:"headers,omitempty"`
+	Delay    *ResponseDelay     `json:"delay,omitempty" yaml:"delay,omitempty"`
 }
 
 type ImposterFs struct {

internal/server/http/proxy.go

@@ -1,38 +1,74 @@
 package http
 
 import (
+	"bytes"
+	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"strconv"
 
 	killgrave "github.com/friendsofgo/killgrave/internal"
 )
 
 // Proxy represent reverse proxy server.
 type Proxy struct {
-	server *httputil.ReverseProxy
-	mode   killgrave.ProxyMode
-	url    *url.URL
+	server        *httputil.ReverseProxy
+	mode          killgrave.ProxyMode
+	url           *url.URL
+	impostersPath string
+
+	recorder RecorderHTTP
 }
 
 // NewProxy creates new proxy server.
-func NewProxy(rawurl string, mode killgrave.ProxyMode) (*Proxy, error) {
+func NewProxy(rawurl, impostersPath string, mode killgrave.ProxyMode, recorder RecorderHTTP) (*Proxy, error) {
 	u, err := url.Parse(rawurl)
 	if err != nil {
 		return nil, err
 	}
 	reverseProxy := httputil.NewSingleHostReverseProxy(u)
-	return &Proxy{server: reverseProxy, mode: mode, url: u}, nil
+	return &Proxy{
+		server:   reverseProxy,
+		mode:     mode,
+		url:      u,
+		recorder: recorder}, nil
 }
 
 // Handler returns handler that sends request to another server.
-func (p *Proxy) Handler() http.HandlerFunc {
+func (p Proxy) Handler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		r.URL.Host = p.url.Host
 		r.URL.Scheme = p.url.Scheme
 		r.Header.Set("X-Forwarded-Host", r.Header.Get("Host"))
 		r.Host = p.url.Host
+		if p.mode == killgrave.ProxyRecord {
+			p.server.ModifyResponse = p.recordProxy
+		}
 
 		p.server.ServeHTTP(w, r)
 	}
 }
+
+func (p Proxy) recordProxy(resp *http.Response) error {
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	bodyStr := string(b)
+	b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1)
+	body := ioutil.NopCloser(bytes.NewReader(b))
+	resp.Body = body
+	resp.ContentLength = int64(len(b))
+	resp.Header.Set("Content-Length", strconv.Itoa(len(b)))
+
+	responseRecorder := ResponseRecorder{
+		Headers: resp.Header,
+		Status:  resp.StatusCode,
+		Body:    bodyStr,
+	}
+
+	return p.recorder.Record(resp.Request, responseRecorder)
+}