From 25dee18b9c438c716b8b90a9d1bbc6b2d2de6a92 Mon Sep 17 00:00:00 2001 From: Rafal Chrabaszcz Date: Tue, 22 Oct 2024 18:08:51 +0200 Subject: [PATCH] Add pagination support to get (#12) --- client.go | 82 +++++++++++++++++++++++++++++++++++++++++-- client_pages_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 client_pages_test.go diff --git a/client.go b/client.go index f213db0..bb09ca4 100644 --- a/client.go +++ b/client.go @@ -16,6 +16,7 @@ import ( "time" "github.com/tidwall/gjson" + "github.com/tidwall/sjson" "github.com/juju/ratelimit" ) @@ -25,6 +26,9 @@ const DefaultBackoffMinDelay int = 2 const DefaultBackoffMaxDelay int = 60 const DefaultBackoffDelayFactor float64 = 3 +// maximum number of Items retrieved in a single GET request +var maxItems = 1000 + // Client is an HTTP FMC client. // Use fmc.NewClient to initiate a client. // This will ensure proper cookie handling and processing of modifiers. @@ -302,9 +306,73 @@ func (client *Client) do(req Req, body []byte) (*http.Response, error) { return client.HttpClient.Do(req.HttpReq) } -// Get makes a GET request and returns a GJSON result. -// Results will be the raw data structure as returned by FMC +// Get makes a GET requests and returns a GJSON result. +// It handles pagination and returns all items in a single response. func (client *Client) Get(path string, mods ...func(*Req)) (Res, error) { + // Check if path contains words 'limit' or 'offset' + // If so, assume user is doing a paginated request and return the raw data + if strings.Contains(path, "limit") || strings.Contains(path, "offset") { + return client.get(path, mods...) + } + + // Execute query as provided by user + raw, err := client.get(path, mods...) + if err != nil { + return raw, err + } + + // If there are no more pages, return the response + if !raw.Get("paging.next.0").Exists() { + return raw, nil + } + + log.Printf("[DEBUG] Paginated response detected") + + // Otherwise discard previous response and get all pages + offset := 0 + fullOutput := `{"items":[]}` + + // Lock writing mutex to make sure the pages are not changed during reading + client.writingMutex.Lock() + defer client.writingMutex.Unlock() + + for { + // Get URL path with offset and limit set + urlPath := pathWithOffset(path, offset, maxItems) + + // Execute query + raw, err := client.get(urlPath, mods...) + if err != nil { + return raw, err + } + + // Check if there are any items in the response + items := raw.Get("items") + if !items.Exists() { + return gjson.Parse("null"), fmt.Errorf("no items found in response") + } + + // Remove first and last character (square brackets) from the output + // If resItems is not empty, attach it to full output + if resItems := items.String()[1 : len(items.String())-1]; resItems != "" { + fullOutput, _ = sjson.SetRaw(fullOutput, "items.-1", resItems) + } + + // If there are no more pages, break the loop + if !raw.Get("paging.next.0").Exists() { + // Create new response with all the items + return gjson.Parse(fullOutput), nil + } + + // Increase offset to get next bulk of data + offset += maxItems + } +} + +// get makes a GET request and returns a GJSON result. +// It does the exact request it is told to do. +// Results will be the raw data structure as returned by FMC +func (client *Client) get(path string, mods ...func(*Req)) (Res, error) { err := client.Authenticate() if err != nil { return Res{}, err @@ -499,3 +567,13 @@ func (client *Client) GetFMCVersion() error { return nil } + +// Create URL path with offset and limit +func pathWithOffset(path string, offset, limit int) string { + sep := "?" + if strings.Contains(path, sep) { + sep = "&" + } + + return fmt.Sprintf("%s%soffset=%d&limit=%d", path, sep, offset, limit) +} diff --git a/client_pages_test.go b/client_pages_test.go new file mode 100644 index 0000000..191032f --- /dev/null +++ b/client_pages_test.go @@ -0,0 +1,83 @@ +package fmc + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/h2non/gock.v1" +) + +// TestClientGet_PagesBasic tests the Client::Get method with pagination. +func TestClientGet_PagesBasic(t *testing.T) { + defer gock.Off() + client := authenticatedTestClient() + + // For pagination tests to be readable, we use dummy page size of 3 instead of 500. + // Since we are changing a package-level var, this test cannot be run on t.Parallel(). + maxItems = 3 + + // First request will be without offset to detect if output is paginated. + gock.New(testURL).Get("/url"). + Reply(200). + BodyString(`{"items":[{"this_should_be_ignored":"by_the_client"}],"paging":{"next":["link_to_next_page"]}}`) + // Following requests will be with offset to get all pages. + gock.New(testURL).Get("/url").MatchParam("offset", "0"). + Reply(200). + BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"}],"paging":{"next":["link_to_next_page"]}}`) + gock.New(testURL).Get("/url").MatchParam("offset", "3"). + Reply(200). + BodyString(`{"items":[{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}],"paging":{"next":["link_to_next_page"]}}`) + gock.New(testURL).Get("/url").MatchParam("offset", "6"). + Reply(200). + BodyString(`{"items":[{"name":"object_7","value":"value_7"},{"name":"object_8","value":"value_8"}]}`) + + res, err := client.Get("/url") + assert.NoError(t, err) + assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"},{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"},{"name":"object_7","value":"value_7"},{"name":"object_8","value":"value_8"}]}`, res.Raw) +} + +// TestClientGet_PagesBasic tests the Client::Get method with pagination, where last page is empty. +func TestClientGet_LastPageEmpty(t *testing.T) { + defer gock.Off() + client := authenticatedTestClient() + + // For pagination tests to be readable, we use dummy page size of 3 instead of 500. + // Since we are changing a package-level var, this test cannot be run on t.Parallel(). + maxItems = 3 + + // First request will be without offset to detect if output is paginated. + gock.New(testURL).Get("/url"). + Reply(200). + BodyString(`{"items":[{"this_should_be_ignored":"by_the_client"}],"paging":{"next":["link_to_next_page"]}}`) + // Following requests will be with offset to get all pages. + gock.New(testURL).Get("/url").MatchParam("offset", "0"). + Reply(200). + BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"}],"paging":{"next":["link_to_next_page"]}}`) + gock.New(testURL).Get("/url").MatchParam("offset", "3"). + Reply(200). + BodyString(`{"items":[{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}],"paging":{"next":["link_to_next_page"]}}`) + gock.New(testURL).Get("/url").MatchParam("offset", "6"). + Reply(200). + BodyString(`{"items":[]}`) + + res, err := client.Get("/url") + assert.NoError(t, err) + assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"},{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}]}`, res.Raw) +} + +// TestClientGet_NotPaginatedSite tests the Client::Get method with a non-paginated response. +func TestClientGet_NotPaginatedSite(t *testing.T) { + defer gock.Off() + client := authenticatedTestClient() + + gock.New(testURL).Get("/url"). + Reply(200). + BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"}]}`) + // Deny all further queries. + gock.New(testURL).Get("/url"). + Reply(400) + + res, err := client.Get("/url") + assert.NoError(t, err) + assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"}]}`, res.Raw) +}