Add pagination support to get (#12)

client.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 
 	"github.com/juju/ratelimit"
 )
@@ -25,6 +26,9 @@ const DefaultBackoffMinDelay int = 2
 const DefaultBackoffMaxDelay int = 60
 const DefaultBackoffDelayFactor float64 = 3
 
+// maximum number of Items retrieved in a single GET request
+var maxItems = 1000
+
 // Client is an HTTP FMC client.
 // Use fmc.NewClient to initiate a client.
 // This will ensure proper cookie handling and processing of modifiers.
@@ -302,9 +306,73 @@ func (client *Client) do(req Req, body []byte) (*http.Response, error) {
 	return client.HttpClient.Do(req.HttpReq)
 }
 
-// Get makes a GET request and returns a GJSON result.
-// Results will be the raw data structure as returned by FMC
+// Get makes a GET requests and returns a GJSON result.
+// It handles pagination and returns all items in a single response.
 func (client *Client) Get(path string, mods ...func(*Req)) (Res, error) {
+	// Check if path contains words 'limit' or 'offset'
+	// If so, assume user is doing a paginated request and return the raw data
+	if strings.Contains(path, "limit") || strings.Contains(path, "offset") {
+		return client.get(path, mods...)
+	}
+
+	// Execute query as provided by user
+	raw, err := client.get(path, mods...)
+	if err != nil {
+		return raw, err
+	}
+
+	// If there are no more pages, return the response
+	if !raw.Get("paging.next.0").Exists() {
+		return raw, nil
+	}
+
+	log.Printf("[DEBUG] Paginated response detected")
+
+	// Otherwise discard previous response and get all pages
+	offset := 0
+	fullOutput := `{"items":[]}`
+
+	// Lock writing mutex to make sure the pages are not changed during reading
+	client.writingMutex.Lock()
+	defer client.writingMutex.Unlock()
+
+	for {
+		// Get URL path with offset and limit set
+		urlPath := pathWithOffset(path, offset, maxItems)
+
+		// Execute query
+		raw, err := client.get(urlPath, mods...)
+		if err != nil {
+			return raw, err
+		}
+
+		// Check if there are any items in the response
+		items := raw.Get("items")
+		if !items.Exists() {
+			return gjson.Parse("null"), fmt.Errorf("no items found in response")
+		}
+
+		// Remove first and last character (square brackets) from the output
+		// If resItems is not empty, attach it to full output
+		if resItems := items.String()[1 : len(items.String())-1]; resItems != "" {
+			fullOutput, _ = sjson.SetRaw(fullOutput, "items.-1", resItems)
+		}
+
+		// If there are no more pages, break the loop
+		if !raw.Get("paging.next.0").Exists() {
+			// Create new response with all the items
+			return gjson.Parse(fullOutput), nil
+		}
+
+		// Increase offset to get next bulk of data
+		offset += maxItems
+	}
+}
+
+// get makes a GET request and returns a GJSON result.
+// It does the exact request it is told to do.
+// Results will be the raw data structure as returned by FMC
+func (client *Client) get(path string, mods ...func(*Req)) (Res, error) {
 	err := client.Authenticate()
 	if err != nil {
 		return Res{}, err
@@ -499,3 +567,13 @@ func (client *Client) GetFMCVersion() error {
 
 	return nil
 }
+
+// Create URL path with offset and limit
+func pathWithOffset(path string, offset, limit int) string {
+	sep := "?"
+	if strings.Contains(path, sep) {
+		sep = "&"
+	}
+
+	return fmt.Sprintf("%s%soffset=%d&limit=%d", path, sep, offset, limit)
+}

client_pages_test.go

@@ -0,0 +1,83 @@
+package fmc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/h2non/gock.v1"
+)
+
+// TestClientGet_PagesBasic tests the Client::Get method with pagination.
+func TestClientGet_PagesBasic(t *testing.T) {
+	defer gock.Off()
+	client := authenticatedTestClient()
+
+	// For pagination tests to be readable, we use dummy page size of 3 instead of 500.
+	// Since we are changing a package-level var, this test cannot be run on t.Parallel().
+	maxItems = 3
+
+	// First request will be without offset to detect if output is paginated.
+	gock.New(testURL).Get("/url").
+		Reply(200).
+		BodyString(`{"items":[{"this_should_be_ignored":"by_the_client"}],"paging":{"next":["link_to_next_page"]}}`)
+	// Following requests will be with offset to get all pages.
+	gock.New(testURL).Get("/url").MatchParam("offset", "0").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"}],"paging":{"next":["link_to_next_page"]}}`)
+	gock.New(testURL).Get("/url").MatchParam("offset", "3").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}],"paging":{"next":["link_to_next_page"]}}`)
+	gock.New(testURL).Get("/url").MatchParam("offset", "6").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_7","value":"value_7"},{"name":"object_8","value":"value_8"}]}`)
+
+	res, err := client.Get("/url")
+	assert.NoError(t, err)
+	assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"},{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"},{"name":"object_7","value":"value_7"},{"name":"object_8","value":"value_8"}]}`, res.Raw)
+}
+
+// TestClientGet_PagesBasic tests the Client::Get method with pagination, where last page is empty.
+func TestClientGet_LastPageEmpty(t *testing.T) {
+	defer gock.Off()
+	client := authenticatedTestClient()
+
+	// For pagination tests to be readable, we use dummy page size of 3 instead of 500.
+	// Since we are changing a package-level var, this test cannot be run on t.Parallel().
+	maxItems = 3
+
+	// First request will be without offset to detect if output is paginated.
+	gock.New(testURL).Get("/url").
+		Reply(200).
+		BodyString(`{"items":[{"this_should_be_ignored":"by_the_client"}],"paging":{"next":["link_to_next_page"]}}`)
+	// Following requests will be with offset to get all pages.
+	gock.New(testURL).Get("/url").MatchParam("offset", "0").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"}],"paging":{"next":["link_to_next_page"]}}`)
+	gock.New(testURL).Get("/url").MatchParam("offset", "3").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}],"paging":{"next":["link_to_next_page"]}}`)
+	gock.New(testURL).Get("/url").MatchParam("offset", "6").
+		Reply(200).
+		BodyString(`{"items":[]}`)
+
+	res, err := client.Get("/url")
+	assert.NoError(t, err)
+	assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"},{"name":"object_3","value":"value_3"},{"name":"object_4","value":"value_4"},{"name":"object_5","value":"value_5"},{"name":"object_6","value":"value_6"}]}`, res.Raw)
+}
+
+// TestClientGet_NotPaginatedSite tests the Client::Get method with a non-paginated response.
+func TestClientGet_NotPaginatedSite(t *testing.T) {
+	defer gock.Off()
+	client := authenticatedTestClient()
+
+	gock.New(testURL).Get("/url").
+		Reply(200).
+		BodyString(`{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"}]}`)
+	// Deny all further queries.
+	gock.New(testURL).Get("/url").
+		Reply(400)
+
+	res, err := client.Get("/url")
+	assert.NoError(t, err)
+	assert.Equal(t, `{"items":[{"name":"object_1","value":"value_1"},{"name":"object_2","value":"value_2"}]}`, res.Raw)
+}