From 7f08b073ba81ccef24b699550811bccdbaa96f74 Mon Sep 17 00:00:00 2001 From: aperezg Date: Sun, 3 Oct 2021 09:02:32 +0200 Subject: [PATCH 01/10] add new mode ProxyRecord --- internal/config.go | 4 ++++ internal/config_test.go | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/config.go b/internal/config.go index e2b1e70..cdf580e 100644 --- a/internal/config.go +++ b/internal/config.go @@ -44,6 +44,8 @@ const ( ProxyNone ProxyMode = iota // ProxyMissing handle only missing requests are proxied ProxyMissing + // ProxyRecord proxy the missing requests and record them into imposter + ProxyRecord // ProxyAll all requests are proxied ProxyAll ) @@ -59,6 +61,7 @@ func (p ProxyMode) String() string { m := map[ProxyMode]string{ ProxyNone: "none", ProxyMissing: "missing", + ProxyRecord: "record", ProxyAll: "all", } @@ -74,6 +77,7 @@ func StringToProxyMode(t string) (ProxyMode, error) { m := map[string]ProxyMode{ "none": ProxyNone, "missing": ProxyMissing, + "record": ProxyRecord, "all": ProxyAll, } diff --git a/internal/config_test.go b/internal/config_test.go index da9ffcf..c868169 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -71,6 +71,7 @@ func TestProxyModeUnmarshal(t *testing.T) { }{ "valid mode all": {"all", ProxyAll, nil}, "valid mode missing": {"missing", ProxyMissing, nil}, + "valid mode record": {"record", ProxyRecord, nil}, "valid mode none": {"none", ProxyNone, nil}, "empty mode": {"", ProxyNone, errors.New("error")}, "invalid mode": {"nonsens23e", ProxyNone, errors.New("error")}, @@ -131,10 +132,15 @@ func TestProxyMode_String(t *testing.T) { "none", }, { - "ProxyNone must be return missing string", + "ProxyMissing must be return missing string", ProxyMissing, "missing", }, + { + "ProxyRecord must be return record string", + ProxyRecord, + "record", + }, { "ProxyNone must be return all string", ProxyAll, From f006d325c911d5868779d4cb9a71b44153c74a8a Mon Sep 17 00:00:00 2001 From: aperezg Date: Sun, 3 Oct 2021 09:14:15 +0200 Subject: [PATCH 02/10] fix the documentation --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aa538fa..89be197 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,13 @@ Killgrave is a simulator for HTTP-based APIs, in simple words a **Mock Server**, * [Docker](#docker) * [Other](#other) - [Getting Started](#getting-started) - * [Using Killgrave by command line](#using-killgrave-by-command-line) + * [Using Killgrave from the command line](#using-killgrave-from-the-command-line) * [Using Killgrave by config file](#using-killgrave-by-config-file) * [Configure CORS](#configure-cors) - * [Prepare Killgrave for Proxy Mode](#prepare-killgrave-for-proxy-mode) - * [Create an Imposter](#create-an-imposter) + * [Preparing Killgrave for Proxy Mode](#preparing-killgrave-for-proxy-mode) + * [Creating an Imposter](#creating-an-imposter) * [Imposters structure](#imposters-structure) - * [Create an Imposter using regex](#create-an-imposter-using-regex) + * [Regex in the headers](#regex-in-the-headers) * [Create an imposter using JSON Schema](#create-an-imposter-using-json-schema) * [Create an imposter with delay](#create-an-imposter-with-delay) * [Create an imposter with dynamic responses](#create-an-imposter-with-dynamic-responses) From 5fda389848c2755a065c0557fdaef3c06aa10945 Mon Sep 17 00:00:00 2001 From: aperezg Date: Wed, 6 Oct 2021 01:56:24 +0200 Subject: [PATCH 03/10] first version of the proxy record, working via Test --- internal/app/cmd/http/http.go | 2 +- internal/config.go | 4 + internal/server/http/imposter.go | 14 +- internal/server/http/proxy.go | 31 +++- internal/server/http/proxy_test.go | 4 +- internal/server/http/recorder.go | 141 ++++++++++++++++++ internal/server/http/recorder_test.go | 28 ++++ internal/server/http/response_delay.go | 5 +- internal/server/http/server.go | 8 +- internal/server/http/server_test.go | 10 +- .../test/testdata/recorder/output.imp.json | 1 + 11 files changed, 227 insertions(+), 21 deletions(-) create mode 100644 internal/server/http/recorder.go create mode 100644 internal/server/http/recorder_test.go create mode 100644 internal/server/http/test/testdata/recorder/output.imp.json diff --git a/internal/app/cmd/http/http.go b/internal/app/cmd/http/http.go index cde41d5..fc224c3 100644 --- a/internal/app/cmd/http/http.go +++ b/internal/app/cmd/http/http.go @@ -100,7 +100,7 @@ func runServer(cfg killgrave.Config) server.Server { Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router), } - proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.Proxy.Mode) + proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.ImpostersPath, cfg.Proxy.Mode) if err != nil { log.Fatal(err) } diff --git a/internal/config.go b/internal/config.go index cdf580e..305038d 100644 --- a/internal/config.go +++ b/internal/config.go @@ -57,6 +57,10 @@ var ( errInvalidPort = errors.New("invalid port") ) +func (p ProxyMode) Is(mode ProxyMode) bool { + return p == mode +} + func (p ProxyMode) String() string { m := map[ProxyMode]string{ ProxyNone: "none", diff --git a/internal/server/http/imposter.go b/internal/server/http/imposter.go index 88c8b1a..52ebf45 100644 --- a/internal/server/http/imposter.go +++ b/internal/server/http/imposter.go @@ -33,7 +33,7 @@ type ImposterConfig struct { // Imposter define an imposter structure type Imposter struct { - BasePath string + BasePath string `json:"-"` Request Request `json:"request"` Response Response `json:"response"` } @@ -52,18 +52,18 @@ func (i *Imposter) CalculateFilePath(filePath string) string { type Request struct { Method string `json:"method"` Endpoint string `json:"endpoint"` - SchemaFile *string `json:"schemaFile"` - Params *map[string]string `json:"params"` - Headers *map[string]string `json:"headers"` + SchemaFile *string `json:"schemaFile,omitempty"` + Params *map[string]string `json:"params,omitempty"` + Headers *map[string]string `json:"headers,omitempty"` } // Response represent the structure of real response type Response struct { Status int `json:"status"` Body string `json:"body"` - BodyFile *string `json:"bodyFile" yaml:"bodyFile"` - Headers *map[string]string `json:"headers"` - Delay ResponseDelay `json:"delay" yaml:"delay"` + BodyFile *string `json:"bodyFile,omitempty" yaml:"bodyFile,omitempty"` + Headers *map[string]string `json:"headers,omitempty"` + Delay *ResponseDelay `json:"delay,omitempty" yaml:"delay,omitempty"` } func findImposters(impostersDirectory string, imposterConfigCh chan ImposterConfig) error { diff --git a/internal/server/http/proxy.go b/internal/server/http/proxy.go index f4c4492..f00f05e 100644 --- a/internal/server/http/proxy.go +++ b/internal/server/http/proxy.go @@ -1,9 +1,13 @@ package http import ( + "bytes" + "errors" + "io/ioutil" "net/http" "net/http/httputil" "net/url" + "strconv" killgrave "github.com/friendsofgo/killgrave/internal" ) @@ -13,20 +17,29 @@ type Proxy struct { server *httputil.ReverseProxy mode killgrave.ProxyMode url *url.URL + impostersPath string } +var ErrImpostersPathEmpty = errors.New("if you want to record the missing request you will need to indicate an imposters path") + // NewProxy creates new proxy server. -func NewProxy(rawurl string, mode killgrave.ProxyMode) (*Proxy, error) { +func NewProxy(rawurl, impostersPath string, mode killgrave.ProxyMode) (*Proxy, error) { u, err := url.Parse(rawurl) if err != nil { return nil, err } reverseProxy := httputil.NewSingleHostReverseProxy(u) + if mode == killgrave.ProxyRecord { + if impostersPath == "" { + return nil, ErrImpostersPathEmpty + } + reverseProxy.ModifyResponse = recordProxy + } return &Proxy{server: reverseProxy, mode: mode, url: u}, nil } // Handler returns handler that sends request to another server. -func (p *Proxy) Handler() http.HandlerFunc { +func (p Proxy) Handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { r.URL.Host = p.url.Host r.URL.Scheme = p.url.Scheme @@ -36,3 +49,17 @@ func (p *Proxy) Handler() http.HandlerFunc { p.server.ServeHTTP(w, r) } } + +func recordProxy(resp *http.Response) error { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + defer resp.Body.Close() + b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1) + body := ioutil.NopCloser(bytes.NewReader(b)) + resp.Body = body + resp.ContentLength = int64(len(b)) + resp.Header.Set("Content-Length", strconv.Itoa(len(b))) + return nil +} \ No newline at end of file diff --git a/internal/server/http/proxy_test.go b/internal/server/http/proxy_test.go index 7ae497b..1239937 100644 --- a/internal/server/http/proxy_test.go +++ b/internal/server/http/proxy_test.go @@ -22,7 +22,7 @@ func TestNewProxy(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - proxy, err := NewProxy(tc.rawURL, tc.mode) + proxy, err := NewProxy(tc.rawURL, "", tc.mode) if err != nil && tc.err == nil { t.Fatalf("not expected any erros and got %v", err) } @@ -47,7 +47,7 @@ func TestProxyHandler(t *testing.T) { })) defer backend.Close() - proxy, err := NewProxy(backend.URL, killgrave.ProxyAll) + proxy, err := NewProxy(backend.URL, "", killgrave.ProxyAll) if err != nil { t.Fatal("NewProxy failed: ", err) } diff --git a/internal/server/http/recorder.go b/internal/server/http/recorder.go new file mode 100644 index 0000000..7b99b11 --- /dev/null +++ b/internal/server/http/recorder.go @@ -0,0 +1,141 @@ +package http + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" +) + +var ( + ErrCreatingRecordDir = errors.New("impossible create record directory") + ErrCreatingRecordFile = errors.New("impossible create record file") + ErrOpenRecordFile = errors.New("impossible open record file") + ErrTryingToReadBody = errors.New("impossible read the body response") + ErrReadingOutputRecordFile = errors.New("error trying to parse the record file") + ErrMarshallingRecordFile = errors.New("error during the marshalling process of the record file") + ErrWritingRecordFile = errors.New("error trying to write on the record file") +) + +type Recorder struct { + outputPathFile string +} + +func NewRecorder(outputPathFile string) Recorder { + return Recorder{ + outputPathFile: outputPathFile, + } +} + +func (r Recorder) Record(req *http.Request, resp *http.Response) error { + f, err := r.prepareOutputFile() + if err != nil { + return err + } + defer f.Close() + + var imposters []Imposter + bytes, _ := ioutil.ReadAll(f) + if err := json.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { + return fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) + } + + imposterRequest := r.prepareImposterRequest(req) + imposterResponse, err := r.prepareImposterResponse(resp) + if err != nil { + return err + } + + imposter := Imposter{ + Request: imposterRequest, + Response: imposterResponse, + } + + //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated + imposters = append(imposters, imposter) + b, err := json.Marshal(imposters) + if err != nil { + return fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + } + + if _, err := f.Write(b); err != nil { + return fmt.Errorf("%v: %w", err, ErrWritingRecordFile) + } + + return nil +} + +func (r Recorder) prepareOutputFile() (*os.File, error) { + dir := filepath.Dir(r.outputPathFile) + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.Mkdir(dir, 0755) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrCreatingRecordDir) + } + } + + var f *os.File + if _, err := os.Stat(r.outputPathFile); os.IsNotExist(err) { + f, err = os.Create(r.outputPathFile) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrCreatingRecordFile) + } + } else { + f, err = os.OpenFile(r.outputPathFile, os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrOpenRecordFile) + } + } + + return f, nil +} + +func (r Recorder) prepareImposterRequest(req *http.Request) Request { + headers := make(map[string]string, len(req.Header)) + for k, v := range req.Header { + for _, val := range v { + headers[k] = val + } + } + + params := make(map[string]string, len(req.URL.Query())) + query := req.URL.Query() + for k, v := range query { + params[k] = v[0] + } + + imposterRequest := Request{ + Method: req.Method, + Endpoint: req.URL.Path, + Headers: &headers, + Params: ¶ms, + } + + return imposterRequest +} + +func (r Recorder) prepareImposterResponse(resp *http.Response) (Response, error) { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return Response{}, fmt.Errorf("%v: %w", err, ErrTryingToReadBody) + } + defer resp.Body.Close() + + headers := make(map[string]string, len(resp.Header)) + for k, v := range resp.Header { + for _, val := range v { + headers[k] = val + } + } + + response := Response{ + Status: resp.StatusCode, + Body: string(b), + Headers: &headers, + } + + return response, nil +} diff --git a/internal/server/http/recorder_test.go b/internal/server/http/recorder_test.go new file mode 100644 index 0000000..41c984b --- /dev/null +++ b/internal/server/http/recorder_test.go @@ -0,0 +1,28 @@ +package http + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestRecorder_Record(t *testing.T) { + recorder := NewRecorder("test/testdata/recorder/output.imp.json") + req, err := http.NewRequest(http.MethodGet, "http://localhost/pokemon?limit=100&offset=200", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Trainer", "Ash Ketchum") + req.Header.Set("Trainer-Key", "25") + + bodyStr := `{"id": 25, name": "Pikachu"}` + + resp := httptest.NewRecorder() + resp.Body.Write([]byte(bodyStr)) + resp.WriteHeader(http.StatusOK) + err = recorder.Record(req, resp.Result()) + + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/server/http/response_delay.go b/internal/server/http/response_delay.go index b837291..7f6a010 100644 --- a/internal/server/http/response_delay.go +++ b/internal/server/http/response_delay.go @@ -16,6 +16,10 @@ type ResponseDelay struct { // Delay return random time.Duration with respect to specified time range. func (d *ResponseDelay) Delay() time.Duration { + if d == nil { + return 0 + } + offset := d.offset if offset > 0 { offset = rand.Int63n(d.offset) @@ -46,7 +50,6 @@ func (d *ResponseDelay) UnmarshalJSON(data []byte) error { return d.parseDelay(input) } - func (d *ResponseDelay) parseDelay(input string) error { const delimiter = ":" diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 56774a3..54ad856 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -86,18 +86,18 @@ func PrepareAccessControl(config killgrave.ConfigCORS) (h []handlers.CORSOption) // Build read all the files on the impostersPath and add different // handlers for each imposter func (s *Server) Build() error { - if s.proxy.mode == killgrave.ProxyAll { + if s.proxy.mode.Is(killgrave.ProxyAll) { s.handleAll(s.proxy.Handler()) } if _, err := os.Stat(s.impostersPath); os.IsNotExist(err) { return fmt.Errorf("%w: the directory %s doesn't exists", err, s.impostersPath) } var imposterConfigCh = make(chan ImposterConfig) - var done = make(chan bool) + var done = make(chan struct{}) go func() { findImposters(s.impostersPath, imposterConfigCh) - done <- true + done <- struct{}{} }() loop: for { @@ -117,7 +117,7 @@ loop: break loop } } - if s.proxy.mode == killgrave.ProxyMissing { + if s.proxy.mode.Is(killgrave.ProxyMissing) || s.proxy.mode.Is(killgrave.ProxyRecord){ s.handleAll(s.proxy.Handler()) } return nil diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index e417d75..8900940 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -58,13 +58,14 @@ func TestBuildProxyMode(t *testing.T) { })) defer proxyServer.Close() makeServer := func(mode killgrave.ProxyMode) (*Server, func()) { + impostersPath := "test/testdata/imposters" router := mux.NewRouter() httpServer := &http.Server{Handler: router} - proxyServer, err := NewProxy(proxyServer.URL, mode) + proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode) if err != nil { t.Fatal("NewProxy failed: ", err) } - server := NewServer("test/testdata/imposters", router, httpServer, proxyServer, false) + server := NewServer(impostersPath, router, httpServer, proxyServer, false) return &server, func() { httpServer.Close() } @@ -136,16 +137,17 @@ func TestBuildSecureMode(t *testing.T) { defer proxyServer.Close() makeServer := func(mode killgrave.ProxyMode) (*Server, func()) { + impostersPath := "test/testdata/imposters_secure" router := mux.NewRouter() cert, _ := tls.X509KeyPair(serverCert, serverKey) httpServer := &http.Server{Handler: router, Addr: ":4430", TLSConfig: &tls.Config{ Certificates: []tls.Certificate{cert}, }} - proxyServer, err := NewProxy(proxyServer.URL, mode) + proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode) if err != nil { t.Fatal("NewProxy failed: ", err) } - server := NewServer("test/testdata/imposters_secure", router, httpServer, proxyServer, true) + server := NewServer(impostersPath, router, httpServer, proxyServer, true) return &server, func() { httpServer.Close() } diff --git a/internal/server/http/test/testdata/recorder/output.imp.json b/internal/server/http/test/testdata/recorder/output.imp.json new file mode 100644 index 0000000..de27e99 --- /dev/null +++ b/internal/server/http/test/testdata/recorder/output.imp.json @@ -0,0 +1 @@ +[{"request":{"method":"GET","endpoint":"/pokemon","params":{"limit":"100","offset":"200"},"headers":{"Trainer":"Ash Ketchum","Trainer-Key":"25"}},"response":{"status":200,"body":"{\"id\": 25, name\": \"Pikachu\"}","headers":{}}}] \ No newline at end of file From 7e82794665941b2cc33f6ab6d2da04d5c8025834 Mon Sep 17 00:00:00 2001 From: aperezg Date: Fri, 8 Oct 2021 01:33:34 +0200 Subject: [PATCH 04/10] add recorder on the notfound requests --- internal/app/cmd/http/http.go | 3 +- internal/server/http/proxy.go | 29 +++++++++------- internal/server/http/proxy_test.go | 4 +-- internal/server/http/recorder.go | 33 ++++++++++++------- internal/server/http/server.go | 15 +++++---- internal/server/http/server_test.go | 4 +-- .../test/testdata/recorder/output.imp.json | 1 - 7 files changed, 53 insertions(+), 36 deletions(-) delete mode 100644 internal/server/http/test/testdata/recorder/output.imp.json diff --git a/internal/app/cmd/http/http.go b/internal/app/cmd/http/http.go index fc224c3..0ff53ee 100644 --- a/internal/app/cmd/http/http.go +++ b/internal/app/cmd/http/http.go @@ -100,7 +100,8 @@ func runServer(cfg killgrave.Config) server.Server { Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router), } - proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.ImpostersPath, cfg.Proxy.Mode) + recorder := server.NewRecorder("imposters/out.imp.json") + proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.ImpostersPath, cfg.Proxy.Mode, recorder) if err != nil { log.Fatal(err) } diff --git a/internal/server/http/proxy.go b/internal/server/http/proxy.go index f00f05e..ed59bee 100644 --- a/internal/server/http/proxy.go +++ b/internal/server/http/proxy.go @@ -2,7 +2,6 @@ package http import ( "bytes" - "errors" "io/ioutil" "net/http" "net/http/httputil" @@ -18,24 +17,26 @@ type Proxy struct { mode killgrave.ProxyMode url *url.URL impostersPath string -} -var ErrImpostersPathEmpty = errors.New("if you want to record the missing request you will need to indicate an imposters path") + recorder RecorderHTTP +} // NewProxy creates new proxy server. -func NewProxy(rawurl, impostersPath string, mode killgrave.ProxyMode) (*Proxy, error) { +func NewProxy(rawurl, impostersPath string, mode killgrave.ProxyMode, recorder RecorderHTTP) (*Proxy, error) { u, err := url.Parse(rawurl) if err != nil { return nil, err } reverseProxy := httputil.NewSingleHostReverseProxy(u) - if mode == killgrave.ProxyRecord { - if impostersPath == "" { - return nil, ErrImpostersPathEmpty - } - reverseProxy.ModifyResponse = recordProxy + + proxy := &Proxy{ + server: reverseProxy, + mode: mode, + url: u, + recorder: recorder, } - return &Proxy{server: reverseProxy, mode: mode, url: u}, nil + + return proxy, nil } // Handler returns handler that sends request to another server. @@ -45,12 +46,15 @@ func (p Proxy) Handler() http.HandlerFunc { r.URL.Scheme = p.url.Scheme r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) r.Host = p.url.Host + if p.mode == killgrave.ProxyRecord { + p.server.ModifyResponse = p.recordProxy + } p.server.ServeHTTP(w, r) } } -func recordProxy(resp *http.Response) error { +func (p Proxy) recordProxy(resp *http.Response) error { b, err := ioutil.ReadAll(resp.Body) if err != nil { return err @@ -61,5 +65,6 @@ func recordProxy(resp *http.Response) error { resp.Body = body resp.ContentLength = int64(len(b)) resp.Header.Set("Content-Length", strconv.Itoa(len(b))) - return nil + + return p.recorder.Record(resp.Request, resp) } \ No newline at end of file diff --git a/internal/server/http/proxy_test.go b/internal/server/http/proxy_test.go index 1239937..7652059 100644 --- a/internal/server/http/proxy_test.go +++ b/internal/server/http/proxy_test.go @@ -22,7 +22,7 @@ func TestNewProxy(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - proxy, err := NewProxy(tc.rawURL, "", tc.mode) + proxy, err := NewProxy(tc.rawURL, "", tc.mode, RecorderNoop{}) if err != nil && tc.err == nil { t.Fatalf("not expected any erros and got %v", err) } @@ -47,7 +47,7 @@ func TestProxyHandler(t *testing.T) { })) defer backend.Close() - proxy, err := NewProxy(backend.URL, "", killgrave.ProxyAll) + proxy, err := NewProxy(backend.URL, "", killgrave.ProxyAll, RecorderNoop{}) if err != nil { t.Fatal("NewProxy failed: ", err) } diff --git a/internal/server/http/recorder.go b/internal/server/http/recorder.go index 7b99b11..810a230 100644 --- a/internal/server/http/recorder.go +++ b/internal/server/http/recorder.go @@ -12,7 +12,6 @@ import ( var ( ErrCreatingRecordDir = errors.New("impossible create record directory") - ErrCreatingRecordFile = errors.New("impossible create record file") ErrOpenRecordFile = errors.New("impossible open record file") ErrTryingToReadBody = errors.New("impossible read the body response") ErrReadingOutputRecordFile = errors.New("error trying to parse the record file") @@ -20,10 +19,18 @@ var ( ErrWritingRecordFile = errors.New("error trying to write on the record file") ) +// RecorderHTTP service to Record the return output of the request +type RecorderHTTP interface { + // Record save the return output from the missing request on the imposters + Record(req *http.Request, resp *http.Response) error +} + +// Recorder implementation of the RecorderHTTP type Recorder struct { outputPathFile string } +// NewRecorder initialise the Recorder func NewRecorder(outputPathFile string) Recorder { return Recorder{ outputPathFile: outputPathFile, @@ -61,6 +68,9 @@ func (r Recorder) Record(req *http.Request, resp *http.Response) error { return fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) } + _ = f.Truncate(0) + _, _ = f.Seek(0,0) + if _, err := f.Write(b); err != nil { return fmt.Errorf("%v: %w", err, ErrWritingRecordFile) } @@ -68,6 +78,13 @@ func (r Recorder) Record(req *http.Request, resp *http.Response) error { return nil } +// RecorderNoop an implementation of the RecorderHTTP without any functionality +type RecorderNoop struct {} + +func (r RecorderNoop) Record(req *http.Request, resp *http.Response) error { + return nil +} + func (r Recorder) prepareOutputFile() (*os.File, error) { dir := filepath.Dir(r.outputPathFile) if _, err := os.Stat(dir); os.IsNotExist(err) { @@ -77,17 +94,9 @@ func (r Recorder) prepareOutputFile() (*os.File, error) { } } - var f *os.File - if _, err := os.Stat(r.outputPathFile); os.IsNotExist(err) { - f, err = os.Create(r.outputPathFile) - if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrCreatingRecordFile) - } - } else { - f, err = os.OpenFile(r.outputPathFile, os.O_APPEND|os.O_WRONLY, os.ModeAppend) - if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrOpenRecordFile) - } + f, err := os.OpenFile(r.outputPathFile, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrOpenRecordFile) } return f, nil diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 54ad856..ad5e621 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -87,8 +87,11 @@ func PrepareAccessControl(config killgrave.ConfigCORS) (h []handlers.CORSOption) // handlers for each imposter func (s *Server) Build() error { if s.proxy.mode.Is(killgrave.ProxyAll) { + // not necessary load the imposters if you will use the tool as a proxy s.handleAll(s.proxy.Handler()) + return nil } + if _, err := os.Stat(s.impostersPath); os.IsNotExist(err) { return fmt.Errorf("%w: the directory %s doesn't exists", err, s.impostersPath) } @@ -104,13 +107,13 @@ loop: select { case imposterConfig := <-imposterConfigCh: var imposters []Imposter - err := s.unmarshalImposters(imposterConfig, &imposters) - if err != nil { + if err := s.unmarshalImposters(imposterConfig, &imposters); err != nil { log.Printf("error trying to load %s imposter: %v", imposterConfig.FilePath, err) - } else { - s.addImposterHandler(imposters, imposterConfig) - log.Printf("imposter %s loaded\n", imposterConfig.FilePath) + continue } + + s.addImposterHandler(imposters, imposterConfig) + log.Printf("imposter %s loaded\n", imposterConfig.FilePath) case <-done: close(imposterConfigCh) close(done) @@ -118,7 +121,7 @@ loop: } } if s.proxy.mode.Is(killgrave.ProxyMissing) || s.proxy.mode.Is(killgrave.ProxyRecord){ - s.handleAll(s.proxy.Handler()) + s.router.NotFoundHandler = s.proxy.Handler() } return nil } diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 8900940..df5c065 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -61,7 +61,7 @@ func TestBuildProxyMode(t *testing.T) { impostersPath := "test/testdata/imposters" router := mux.NewRouter() httpServer := &http.Server{Handler: router} - proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode) + proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode, RecorderNoop{}) if err != nil { t.Fatal("NewProxy failed: ", err) } @@ -143,7 +143,7 @@ func TestBuildSecureMode(t *testing.T) { httpServer := &http.Server{Handler: router, Addr: ":4430", TLSConfig: &tls.Config{ Certificates: []tls.Certificate{cert}, }} - proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode) + proxyServer, err := NewProxy(proxyServer.URL, impostersPath, mode, RecorderNoop{}) if err != nil { t.Fatal("NewProxy failed: ", err) } diff --git a/internal/server/http/test/testdata/recorder/output.imp.json b/internal/server/http/test/testdata/recorder/output.imp.json deleted file mode 100644 index de27e99..0000000 --- a/internal/server/http/test/testdata/recorder/output.imp.json +++ /dev/null @@ -1 +0,0 @@ -[{"request":{"method":"GET","endpoint":"/pokemon","params":{"limit":"100","offset":"200"},"headers":{"Trainer":"Ash Ketchum","Trainer-Key":"25"}},"response":{"status":200,"body":"{\"id\": 25, name\": \"Pikachu\"}","headers":{}}}] \ No newline at end of file From 49aa99d326fe998588ff2ca3af13989e63cea6a5 Mon Sep 17 00:00:00 2001 From: aperezg Date: Fri, 8 Oct 2021 01:34:05 +0200 Subject: [PATCH 05/10] add recorder on the notfound requests --- internal/server/http/proxy.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/internal/server/http/proxy.go b/internal/server/http/proxy.go index ed59bee..1622829 100644 --- a/internal/server/http/proxy.go +++ b/internal/server/http/proxy.go @@ -13,9 +13,9 @@ import ( // Proxy represent reverse proxy server. type Proxy struct { - server *httputil.ReverseProxy - mode killgrave.ProxyMode - url *url.URL + server *httputil.ReverseProxy + mode killgrave.ProxyMode + url *url.URL impostersPath string recorder RecorderHTTP @@ -28,15 +28,11 @@ func NewProxy(rawurl, impostersPath string, mode killgrave.ProxyMode, recorder R return nil, err } reverseProxy := httputil.NewSingleHostReverseProxy(u) - - proxy := &Proxy{ - server: reverseProxy, - mode: mode, - url: u, - recorder: recorder, - } - - return proxy, nil + return &Proxy{ + server: reverseProxy, + mode: mode, + url: u, + recorder: recorder}, nil } // Handler returns handler that sends request to another server. @@ -67,4 +63,4 @@ func (p Proxy) recordProxy(resp *http.Response) error { resp.Header.Set("Content-Length", strconv.Itoa(len(b))) return p.recorder.Record(resp.Request, resp) -} \ No newline at end of file +} From 405c24c3c64f9c58b59f54ec21c14cd0297f4a4b Mon Sep 17 00:00:00 2001 From: aperezg Date: Sat, 9 Oct 2021 02:14:29 +0200 Subject: [PATCH 06/10] fix test to remove the recorder file, and fix store the response body on the imposters and show the result --- internal/server/http/proxy.go | 10 +++++++- internal/server/http/recorder.go | 35 ++++++++++++++------------- internal/server/http/recorder_test.go | 35 +++++++++++++++++++-------- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/internal/server/http/proxy.go b/internal/server/http/proxy.go index 1622829..16dc957 100644 --- a/internal/server/http/proxy.go +++ b/internal/server/http/proxy.go @@ -56,11 +56,19 @@ func (p Proxy) recordProxy(resp *http.Response) error { return err } defer resp.Body.Close() + + bodyStr := string(b) b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1) body := ioutil.NopCloser(bytes.NewReader(b)) resp.Body = body resp.ContentLength = int64(len(b)) resp.Header.Set("Content-Length", strconv.Itoa(len(b))) - return p.recorder.Record(resp.Request, resp) + responseRecorder := ResponseRecorder{ + Headers: resp.Header, + Status: resp.StatusCode, + Body: bodyStr, + } + + return p.recorder.Record(resp.Request, responseRecorder) } diff --git a/internal/server/http/recorder.go b/internal/server/http/recorder.go index 810a230..3bbe7ec 100644 --- a/internal/server/http/recorder.go +++ b/internal/server/http/recorder.go @@ -13,7 +13,6 @@ import ( var ( ErrCreatingRecordDir = errors.New("impossible create record directory") ErrOpenRecordFile = errors.New("impossible open record file") - ErrTryingToReadBody = errors.New("impossible read the body response") ErrReadingOutputRecordFile = errors.New("error trying to parse the record file") ErrMarshallingRecordFile = errors.New("error during the marshalling process of the record file") ErrWritingRecordFile = errors.New("error trying to write on the record file") @@ -22,7 +21,7 @@ var ( // RecorderHTTP service to Record the return output of the request type RecorderHTTP interface { // Record save the return output from the missing request on the imposters - Record(req *http.Request, resp *http.Response) error + Record(req *http.Request, resp ResponseRecorder) error } // Recorder implementation of the RecorderHTTP @@ -30,6 +29,13 @@ type Recorder struct { outputPathFile string } +// ResponseRecorder response data transfer object +type ResponseRecorder struct { + Status int + Headers http.Header + Body string +} + // NewRecorder initialise the Recorder func NewRecorder(outputPathFile string) Recorder { return Recorder{ @@ -37,7 +43,7 @@ func NewRecorder(outputPathFile string) Recorder { } } -func (r Recorder) Record(req *http.Request, resp *http.Response) error { +func (r Recorder) Record(req *http.Request, resp ResponseRecorder) error { f, err := r.prepareOutputFile() if err != nil { return err @@ -69,7 +75,7 @@ func (r Recorder) Record(req *http.Request, resp *http.Response) error { } _ = f.Truncate(0) - _, _ = f.Seek(0,0) + _, _ = f.Seek(0, 0) if _, err := f.Write(b); err != nil { return fmt.Errorf("%v: %w", err, ErrWritingRecordFile) @@ -79,9 +85,9 @@ func (r Recorder) Record(req *http.Request, resp *http.Response) error { } // RecorderNoop an implementation of the RecorderHTTP without any functionality -type RecorderNoop struct {} +type RecorderNoop struct{} -func (r RecorderNoop) Record(req *http.Request, resp *http.Response) error { +func (r RecorderNoop) Record(req *http.Request, resp ResponseRecorder) error { return nil } @@ -106,6 +112,7 @@ func (r Recorder) prepareImposterRequest(req *http.Request) Request { headers := make(map[string]string, len(req.Header)) for k, v := range req.Header { for _, val := range v { + // TODO: configure which headers don't you want to store or more commons like Postman?? headers[k] = val } } @@ -126,24 +133,18 @@ func (r Recorder) prepareImposterRequest(req *http.Request) Request { return imposterRequest } -func (r Recorder) prepareImposterResponse(resp *http.Response) (Response, error) { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return Response{}, fmt.Errorf("%v: %w", err, ErrTryingToReadBody) - } - defer resp.Body.Close() - - headers := make(map[string]string, len(resp.Header)) - for k, v := range resp.Header { +func (r Recorder) prepareImposterResponse(resp ResponseRecorder) (Response, error) { + headers := make(map[string]string, len(resp.Headers)) + for k, v := range resp.Headers { for _, val := range v { headers[k] = val } } response := Response{ - Status: resp.StatusCode, - Body: string(b), + Status: resp.Status, Headers: &headers, + Body: resp.Body, } return response, nil diff --git a/internal/server/http/recorder_test.go b/internal/server/http/recorder_test.go index 41c984b..692a9ae 100644 --- a/internal/server/http/recorder_test.go +++ b/internal/server/http/recorder_test.go @@ -1,28 +1,43 @@ package http import ( + "errors" "net/http" - "net/http/httptest" + "os" + "path/filepath" "testing" ) func TestRecorder_Record(t *testing.T) { - recorder := NewRecorder("test/testdata/recorder/output.imp.json") - req, err := http.NewRequest(http.MethodGet, "http://localhost/pokemon?limit=100&offset=200", nil) + outputPath := "test/testdata/recorder/output.imp.json" + recorder := NewRecorder(outputPath) + req, err := http.NewRequest(http.MethodGet, "http://localhost/items?limit=100&offset=200", nil) if err != nil { t.Fatal(err) } - req.Header.Set("Trainer", "Ash Ketchum") - req.Header.Set("Trainer-Key", "25") + req.Header.Set("ItemUser", "Conan") + req.Header.Set("Item-Key", "25") - bodyStr := `{"id": 25, name": "Pikachu"}` + bodyStr := `{"id": 25, name": "Umbrella"}` - resp := httptest.NewRecorder() - resp.Body.Write([]byte(bodyStr)) - resp.WriteHeader(http.StatusOK) - err = recorder.Record(req, resp.Result()) + resp := ResponseRecorder{ + Status: http.StatusOK, + Body: bodyStr, + } + + err = recorder.Record(req, resp) if err != nil { t.Fatal(err) } + + f, err := os.Stat(outputPath) + if os.IsNotExist(err) { + t.Fatal(err) + } + if f.Size() <= 0 { + t.Fatal(errors.New("empty file")) + } + + os.RemoveAll(filepath.Dir(outputPath)) } From 1c65608a8a507edf17fa4842e3a10d2661d73a5e Mon Sep 17 00:00:00 2001 From: aperezg Date: Sun, 10 Oct 2021 01:38:45 +0200 Subject: [PATCH 07/10] adding the option to indicate the record path on config or flag --- go.sum | 1 + internal/app/cmd/http/http.go | 29 ++++++--------- internal/config.go | 34 +++++++++++++++--- internal/config_test.go | 51 +++++++++++++++++++-------- internal/server/http/recorder_test.go | 4 ++- 5 files changed, 81 insertions(+), 38 deletions(-) diff --git a/go.sum b/go.sum index 14e27e7..a7d0669 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/internal/app/cmd/http/http.go b/internal/app/cmd/http/http.go index 0ff53ee..90fbee2 100644 --- a/internal/app/cmd/http/http.go +++ b/internal/app/cmd/http/http.go @@ -18,10 +18,10 @@ import ( ) const ( - _defaultHost = "localhost" - _defaultPort = 3000 - _defaultProxyMode = killgrave.ProxyNone - _defaultStrictSlash = true + _defaultHost = "localhost" + _defaultPort = 3000 + _defaultProxyMode = killgrave.ProxyNone + _defaultStrictSlash = true ) var ( @@ -29,7 +29,6 @@ var ( errGetDataFromHostFlag = errors.New("error trying to get data from host flag") errGetDataFromPortFlag = errors.New("error trying to get data from port flag") errGetDataFromSecureFlag = errors.New("error trying to get data from secure flag") - errMandatoryURL = errors.New("the field url is mandatory if you selected a proxy mode") ) // NewHTTPCmd returns cobra.Command to run http sub command, this command will be used to run the mock server @@ -58,8 +57,9 @@ func NewHTTPCmd() *cobra.Command { cmd.PersistentFlags().IntP("port", "P", _defaultPort, "Port to run the server") cmd.PersistentFlags().BoolP("watcher", "w", false, "File watcher will reload the server on each file change") cmd.PersistentFlags().BoolP("secure", "s", false, "Run mock server using TLS (https)") - cmd.Flags().StringP("proxy", "p", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") + cmd.Flags().StringP("proxy", "p", _defaultProxyMode.String(), "Proxy mode, the options are all, missing, record or none") cmd.Flags().StringP("url", "u", "", "The url where the proxy will redirect to") + cmd.Flags().StringP("outputRecordFile", "o", "", "The record file path when the proxy is on record mode") return cmd } @@ -100,7 +100,7 @@ func runServer(cfg killgrave.Config) server.Server { Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router), } - recorder := server.NewRecorder("imposters/out.imp.json") + recorder := server.NewRecorder(cfg.Proxy.RecordFilePath) proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.ImpostersPath, cfg.Proxy.Mode, recorder) if err != nil { log.Fatal(err) @@ -181,17 +181,8 @@ func configureProxyMode(cmd *cobra.Command, cfg *killgrave.Config) error { return err } - var url string - if mode != killgrave.ProxyNone.String() { - url, err = cmd.Flags().GetString("url") - if err != nil { - return err - } + url, _ := cmd.Flags().GetString("url") + recordFilePath, _ := cmd.Flags().GetString("outputFileRecord") - if url == "" { - return errMandatoryURL - } - } - cfg.ConfigureProxy(pMode, url) - return nil + return cfg.ConfigureProxy(pMode, url, recordFilePath) } diff --git a/internal/config.go b/internal/config.go index 305038d..0b72160 100644 --- a/internal/config.go +++ b/internal/config.go @@ -10,6 +10,11 @@ import ( "gopkg.in/yaml.v2" ) +var ( + errMandatoryURL = errors.New("the field url is mandatory if you selected a proxy mode") + errMandatoryRecordFilePath = errors.New("the field outputRecordFile is mandatory if you selected a proxy record") +) + // Config representation of config file yaml type Config struct { ImpostersPath string `yaml:"imposters_path"` @@ -32,8 +37,9 @@ type ConfigCORS struct { // ConfigProxy is a representation of section proxy of the yaml type ConfigProxy struct { - Url string `yaml:"url"` - Mode ProxyMode `yaml:"mode"` + Url string `yaml:"url"` + Mode ProxyMode `yaml:"mode"` + RecordFilePath string `yaml:"record_file_path"` } // ProxyMode is enumeration of proxy server modes @@ -81,7 +87,7 @@ func StringToProxyMode(t string) (ProxyMode, error) { m := map[string]ProxyMode{ "none": ProxyNone, "missing": ProxyMissing, - "record": ProxyRecord, + "record": ProxyRecord, "all": ProxyAll, } @@ -110,9 +116,24 @@ func (p *ProxyMode) UnmarshalYAML(unmarshal func(interface{}) error) error { } // ConfigureProxy preparing the server with the proxy configuration that the user has indicated -func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) { +func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL, recordFilePath string) error { + if proxyMode.Is(ProxyNone) { + return nil + } + + if proxyURL == "" { + return errMandatoryURL + } + + if proxyMode.Is(ProxyRecord) && recordFilePath == "" { + return errMandatoryRecordFilePath + } + cfg.Proxy.Mode = proxyMode cfg.Proxy.Url = proxyURL + cfg.Proxy.RecordFilePath = recordFilePath + + return nil } // ConfigOpt function to encapsulate optional parameters @@ -159,6 +180,11 @@ func NewConfigFromFile(cfgPath string) (Config, error) { return Config{}, fmt.Errorf("%w: error while unmarshalling configFile file %s, using default configuration instead", err, cfgPath) } + recordFilePath := path.Join(path.Dir(cfgPath), cfg.Proxy.RecordFilePath) + if err := cfg.ConfigureProxy(cfg.Proxy.Mode, cfg.Proxy.Url, recordFilePath); err != nil { + return Config{}, err + } + cfg.ImpostersPath = path.Join(path.Dir(cfgPath), cfg.ImpostersPath) return cfg, nil diff --git a/internal/config_test.go b/internal/config_test.go index c868169..e779c97 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -2,6 +2,7 @@ package killgrave import ( "errors" + "github.com/stretchr/testify/assert" "reflect" "testing" ) @@ -233,23 +234,45 @@ func TestNewConfig(t *testing.T) { } func TestConfig_ConfigureProxy(t *testing.T) { - expected := Config{ - ImpostersPath: "imposters", - Port: 80, - Host: "localhost", - Proxy: ConfigProxy{ - Url: "https://friendsofgo.tech", - Mode: ProxyAll, - }, + tests := []struct{ + name string + url string + mode ProxyMode + err error + }{ + {"valid proxy configuration", "https://friendsofgo.tech", ProxyAll, nil}, + {"none proxy configuration", "", ProxyNone, nil}, + {"invalid proxy configuration", "", ProxyAll, errMandatoryURL}, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T){ + cfg, err := NewConfig("imposters", "localhost", 80, false) + if err != nil { + t.Fatalf("error not expected: %v", err) + } - got, err := NewConfig("imposters", "localhost", 80, false) - if err != nil { - t.Fatalf("error not expected: %v", err) + err = cfg.ConfigureProxy(tt.mode, tt.url, "") + assert.Equal(t, tt.err, err) + }) } - got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech") +} - if !reflect.DeepEqual(expected, got) { - t.Fatalf("got = %v, want %v", expected, got) +func TestConfig_ConfigureProxyRecord(t *testing.T) { + tests := []struct { + name string + recordFilePath string + err error + }{ + {"valid configuration for Proxy Record", "some_file.imp.json", nil}, + {"invalid configuration for Proxy Record", "", errMandatoryRecordFilePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg, err := NewConfig("imposters", "localhost", 80, false) + assert.NoError(t, err) + + err = cfg.ConfigureProxy(ProxyRecord, "https://friendsofgo.tech", tt.recordFilePath) + assert.Equal(t, tt.err, err) + }) } } diff --git a/internal/server/http/recorder_test.go b/internal/server/http/recorder_test.go index 692a9ae..ac3ea9d 100644 --- a/internal/server/http/recorder_test.go +++ b/internal/server/http/recorder_test.go @@ -39,5 +39,7 @@ func TestRecorder_Record(t *testing.T) { t.Fatal(errors.New("empty file")) } - os.RemoveAll(filepath.Dir(outputPath)) + dir := filepath.Dir(outputPath) + os.Remove(outputPath) + os.RemoveAll(dir) } From 32b1f61763dc7b62e4f4f27d636fd73ef7abecf1 Mon Sep 17 00:00:00 2001 From: aperezg Date: Sun, 10 Oct 2021 02:10:27 +0200 Subject: [PATCH 08/10] compatiblity record with yaml files --- internal/server/http/recorder.go | 72 +++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/internal/server/http/recorder.go b/internal/server/http/recorder.go index 3bbe7ec..9d88f29 100644 --- a/internal/server/http/recorder.go +++ b/internal/server/http/recorder.go @@ -4,10 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "gopkg.in/yaml.v2" "io/ioutil" "net/http" "os" "path/filepath" + "strings" ) var ( @@ -44,18 +46,6 @@ func NewRecorder(outputPathFile string) Recorder { } func (r Recorder) Record(req *http.Request, resp ResponseRecorder) error { - f, err := r.prepareOutputFile() - if err != nil { - return err - } - defer f.Close() - - var imposters []Imposter - bytes, _ := ioutil.ReadAll(f) - if err := json.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { - return fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) - } - imposterRequest := r.prepareImposterRequest(req) imposterResponse, err := r.prepareImposterResponse(resp) if err != nil { @@ -67,11 +57,26 @@ func (r Recorder) Record(req *http.Request, resp ResponseRecorder) error { Response: imposterResponse, } - //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated - imposters = append(imposters, imposter) - b, err := json.Marshal(imposters) + f, err := r.prepareOutputFile() if err != nil { - return fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + return err + } + defer f.Close() + + var b []byte + switch { + case strings.HasSuffix(r.outputPathFile, jsonImposterExtension): + b, err = r.recordOnJSON(f, imposter) + if err != nil { + return err + } + case strings.HasSuffix(r.outputPathFile, yamlImposterExtension) || strings.HasSuffix(r.outputPathFile, ymlImposterExtension) : + b, err = r.recordOnYAML(f, imposter) + if err != nil { + return err + } + default: + return errors.New("file extension not recognized") } _ = f.Truncate(0) @@ -149,3 +154,38 @@ func (r Recorder) prepareImposterResponse(resp ResponseRecorder) (Response, erro return response, nil } + +func (r Recorder) recordOnJSON(file *os.File, imposter Imposter) ([]byte,error) { + var imposters []Imposter + bytes, _ := ioutil.ReadAll(file) + if err := json.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { + return nil, fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) + } + + //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated + imposters = append(imposters, imposter) + b, err := json.Marshal(imposters) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + } + + return b, nil +} + +func (r Recorder) recordOnYAML(file *os.File, imposter Imposter) ([]byte,error) { + var imposters []Imposter + bytes, _ := ioutil.ReadAll(file) + if err := yaml.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { + return nil, fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) + } + + //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated + imposters = append(imposters, imposter) + b, err := yaml.Marshal(imposters) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + } + + return b, nil +} + From a6e7fca8fc0f577887bc26e6de5419402be308e4 Mon Sep 17 00:00:00 2001 From: aperezg Date: Sun, 10 Oct 2021 02:35:10 +0200 Subject: [PATCH 09/10] testing for the record process --- internal/server/http/recorder.go | 55 +++++++++---------- internal/server/http/recorder_test.go | 79 ++++++++++++++++++--------- 2 files changed, 80 insertions(+), 54 deletions(-) diff --git a/internal/server/http/recorder.go b/internal/server/http/recorder.go index 9d88f29..3e37dde 100644 --- a/internal/server/http/recorder.go +++ b/internal/server/http/recorder.go @@ -4,20 +4,22 @@ import ( "encoding/json" "errors" "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "net/http" "os" "path/filepath" "strings" + + "gopkg.in/yaml.v2" ) var ( - ErrCreatingRecordDir = errors.New("impossible create record directory") - ErrOpenRecordFile = errors.New("impossible open record file") - ErrReadingOutputRecordFile = errors.New("error trying to parse the record file") - ErrMarshallingRecordFile = errors.New("error during the marshalling process of the record file") - ErrWritingRecordFile = errors.New("error trying to write on the record file") + errCreatingRecordDir = errors.New("impossible create record directory") + errOpenRecordFile = errors.New("impossible open record file") + errReadingOutputRecordFile = errors.New("error trying to parse the record file") + errMarshallingRecordFile = errors.New("error during the marshalling process of the record file") + errWritingRecordFile = errors.New("error trying to write on the record file") + errUnrecognizedExtension = errors.New("file extension not recognized") ) // RecorderHTTP service to Record the return output of the request @@ -47,10 +49,7 @@ func NewRecorder(outputPathFile string) Recorder { func (r Recorder) Record(req *http.Request, resp ResponseRecorder) error { imposterRequest := r.prepareImposterRequest(req) - imposterResponse, err := r.prepareImposterResponse(resp) - if err != nil { - return err - } + imposterResponse := r.prepareImposterResponse(resp) imposter := Imposter{ Request: imposterRequest, @@ -70,20 +69,18 @@ func (r Recorder) Record(req *http.Request, resp ResponseRecorder) error { if err != nil { return err } - case strings.HasSuffix(r.outputPathFile, yamlImposterExtension) || strings.HasSuffix(r.outputPathFile, ymlImposterExtension) : + case strings.HasSuffix(r.outputPathFile, yamlImposterExtension) || strings.HasSuffix(r.outputPathFile, ymlImposterExtension): b, err = r.recordOnYAML(f, imposter) if err != nil { return err } - default: - return errors.New("file extension not recognized") } _ = f.Truncate(0) _, _ = f.Seek(0, 0) if _, err := f.Write(b); err != nil { - return fmt.Errorf("%v: %w", err, ErrWritingRecordFile) + return fmt.Errorf("%v: %w", err, errWritingRecordFile) } return nil @@ -97,17 +94,22 @@ func (r RecorderNoop) Record(req *http.Request, resp ResponseRecorder) error { } func (r Recorder) prepareOutputFile() (*os.File, error) { + if !strings.HasSuffix(r.outputPathFile, jsonImposterExtension) && + !strings.HasSuffix(r.outputPathFile, yamlImposterExtension) && !strings.HasSuffix(r.outputPathFile, ymlImposterExtension) { + return nil, errUnrecognizedExtension + } + dir := filepath.Dir(r.outputPathFile) if _, err := os.Stat(dir); os.IsNotExist(err) { err := os.Mkdir(dir, 0755) if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrCreatingRecordDir) + return nil, fmt.Errorf("%v: %w", err, errCreatingRecordDir) } } f, err := os.OpenFile(r.outputPathFile, os.O_RDWR|os.O_CREATE, 0755) if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrOpenRecordFile) + return nil, fmt.Errorf("%v: %w", err, errOpenRecordFile) } return f, nil @@ -138,7 +140,7 @@ func (r Recorder) prepareImposterRequest(req *http.Request) Request { return imposterRequest } -func (r Recorder) prepareImposterResponse(resp ResponseRecorder) (Response, error) { +func (r Recorder) prepareImposterResponse(resp ResponseRecorder) Response { headers := make(map[string]string, len(resp.Headers)) for k, v := range resp.Headers { for _, val := range v { @@ -149,43 +151,40 @@ func (r Recorder) prepareImposterResponse(resp ResponseRecorder) (Response, erro response := Response{ Status: resp.Status, Headers: &headers, - Body: resp.Body, + Body: resp.Body, } - return response, nil + return response } -func (r Recorder) recordOnJSON(file *os.File, imposter Imposter) ([]byte,error) { +func (r Recorder) recordOnJSON(file *os.File, imposter Imposter) ([]byte, error) { var imposters []Imposter bytes, _ := ioutil.ReadAll(file) if err := json.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { - return nil, fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) + return nil, fmt.Errorf("%v: %w", err, errReadingOutputRecordFile) } - //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated imposters = append(imposters, imposter) b, err := json.Marshal(imposters) if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + return nil, fmt.Errorf("%v: %w", err, errMarshallingRecordFile) } return b, nil } -func (r Recorder) recordOnYAML(file *os.File, imposter Imposter) ([]byte,error) { +func (r Recorder) recordOnYAML(file *os.File, imposter Imposter) ([]byte, error) { var imposters []Imposter bytes, _ := ioutil.ReadAll(file) if err := yaml.Unmarshal(bytes, &imposters); err != nil && len(bytes) > 0 { - return nil, fmt.Errorf("%v: %w", err, ErrReadingOutputRecordFile) + return nil, fmt.Errorf("%v: %w", err, errReadingOutputRecordFile) } - //TODO: create an inMemory to store which imposters are saved during this session to avoid duplicated imposters = append(imposters, imposter) b, err := yaml.Marshal(imposters) if err != nil { - return nil, fmt.Errorf("%v: %w", err, ErrMarshallingRecordFile) + return nil, fmt.Errorf("%v: %w", err, errMarshallingRecordFile) } return b, nil } - diff --git a/internal/server/http/recorder_test.go b/internal/server/http/recorder_test.go index ac3ea9d..8eec7f4 100644 --- a/internal/server/http/recorder_test.go +++ b/internal/server/http/recorder_test.go @@ -1,45 +1,72 @@ package http import ( - "errors" "net/http" "os" "path/filepath" "testing" + + "github.com/stretchr/testify/assert" ) func TestRecorder_Record(t *testing.T) { - outputPath := "test/testdata/recorder/output.imp.json" - recorder := NewRecorder(outputPath) - req, err := http.NewRequest(http.MethodGet, "http://localhost/items?limit=100&offset=200", nil) - if err != nil { - t.Fatal(err) + tests := []struct { + name string + recordPath string + }{ + { + name: "JSON record", + recordPath: "test/testdata/recorder/output.imp.json", + }, + { + name: "YAML record", + recordPath: "test/testdata/recorder/output.imp.yaml", + }, } - req.Header.Set("ItemUser", "Conan") - req.Header.Set("Item-Key", "25") - bodyStr := `{"id": 25, name": "Umbrella"}` + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T){ + recorder := NewRecorder(tt.recordPath) + req, err := http.NewRequest(http.MethodGet, "http://localhost/items?limit=100&offset=200", nil) + assert.NoError(t, err) - resp := ResponseRecorder{ - Status: http.StatusOK, - Body: bodyStr, - } + req.Header.Set("ItemUser", "Conan") + req.Header.Set("Item-Key", "25") - err = recorder.Record(req, resp) + bodyStr := `{"id": 25, name": "Umbrella"}` - if err != nil { - t.Fatal(err) - } + resp := ResponseRecorder{ + Status: http.StatusOK, + Body: bodyStr, + } + + err = recorder.Record(req, resp) + assert.NoError(t, err) + + f, err := os.Stat(tt.recordPath) + assert.NoError(t, err) + + assert.Greater(t, f.Size(), int64(0), "empty file") - f, err := os.Stat(outputPath) - if os.IsNotExist(err) { - t.Fatal(err) + dir := filepath.Dir(tt.recordPath) + _ = os.Remove(tt.recordPath) + _ = os.RemoveAll(dir) + }) } - if f.Size() <= 0 { - t.Fatal(errors.New("empty file")) +} + +func TestRecorder_RecordWithInvalidExtension(t *testing.T) { + recorder := NewRecorder("test/testdata/recorder/output.imp.dist") + + req, err := http.NewRequest(http.MethodGet, "http://localhost/items?limit=100&offset=200", nil) + assert.NoError(t, err) + + resp := ResponseRecorder{ + Status: http.StatusOK, + Body: "", } - dir := filepath.Dir(outputPath) - os.Remove(outputPath) - os.RemoveAll(dir) -} + err = recorder.Record(req, resp) + assert.Error(t, err) + assert.Equal(t, err, errUnrecognizedExtension) +} \ No newline at end of file From d03826f9e896d7d257b677965987328b83dab13f Mon Sep 17 00:00:00 2001 From: aperezg Date: Mon, 11 Oct 2021 23:27:08 +0200 Subject: [PATCH 10/10] documentation about proxy record --- README.md | 7 +++++-- internal/app/cmd/http/http.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 89be197..c85cd0b 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,8 @@ $ killgrave -h proxy mode you can choose between (all, missing or none) (default "none") -proxy-url string proxy url, you need to choose a proxy-mode + - record-file-path string + the file path where the imposters will be recorded if you selected proxy-mode record -version show the _version of the application -watcher @@ -172,8 +174,8 @@ port: 3000 host: "localhost" proxy: url: https://example.com - mode: missing -watcher: true + record_file_path: "imposters/record.imp.json" + mode: record cors: methods: ["GET"] headers: ["Content-Type"] @@ -249,6 +251,7 @@ In the CORS section of the file you can find the following options: You can use Killgrave in proxy mode using the flags `proxy-mode` and `proxy-url` or their equivalent fields in the configuration file. The following three proxy modes are available: * `none`: Default. Killgrave will not behave as a proxy and the mock server will only use the configured imposters. * `missing`: With this mode the mock server will try to match the request with a configured imposter, but if no matching endpoint was found, the mock server will call to the real server, declared in the `proxy-url` configuration variable. +* `record`: This mode works as the same of the `missing` mode but with one exception, when `Killgrave` not match with some `URL` will record into a `record imposter`. You will need to declare `record-file-path` to indicate the path where you want to record the imposters (i.e. `imposters/record.imp.json`) * `all`: The mock server will always call to the real server, declared in the `proxy-url` configuration variable. The `proxy-url` must be the root path of the proxied server. For example, if we have an API running on `http://example.com/things`, the `proxy-url` will be `http://example.com`. diff --git a/internal/app/cmd/http/http.go b/internal/app/cmd/http/http.go index 90fbee2..cfe8da0 100644 --- a/internal/app/cmd/http/http.go +++ b/internal/app/cmd/http/http.go @@ -59,7 +59,7 @@ func NewHTTPCmd() *cobra.Command { cmd.PersistentFlags().BoolP("secure", "s", false, "Run mock server using TLS (https)") cmd.Flags().StringP("proxy", "p", _defaultProxyMode.String(), "Proxy mode, the options are all, missing, record or none") cmd.Flags().StringP("url", "u", "", "The url where the proxy will redirect to") - cmd.Flags().StringP("outputRecordFile", "o", "", "The record file path when the proxy is on record mode") + cmd.Flags().StringP("record-file-path", "o", "", "The record file path when the proxy is on record mode") return cmd } @@ -182,7 +182,7 @@ func configureProxyMode(cmd *cobra.Command, cfg *killgrave.Config) error { } url, _ := cmd.Flags().GetString("url") - recordFilePath, _ := cmd.Flags().GetString("outputFileRecord") + recordFilePath, _ := cmd.Flags().GetString("record-file-path") return cfg.ConfigureProxy(pMode, url, recordFilePath) }