inanzzz | Elasticsearch pagination query in Golang

In this example we are going to use Elasticsearch's Search API to query the index and paginate the result set in Golang. We will be using the classic from and size parameters because from+size will always be less than or equal to 10000. However, if you cannot guarantee that, you should stick with Search after feature instead. The best thing about this feature is that it uses Point in Time API (PIT). When a refresh occurs between pagination requests, the order of the result may change which causes inconsistency across pages. The Point in Time API preserves the current index state to prevent such issue.

Files

This is follow up to the previous post A simple Elasticsearch CRUD example in Golang so I am not duplication all the files here.

elasticsearch.go

package elasticsearch

...

// postsResponse represents list of posts in Search API response body.
type postsResponse struct {
	Hits struct {
		Total struct {
			Value int `json:"value"`
		} `json:"total"`
		Hits []struct {
			Source *storage.Post `json:"_source"`
		} `json:"hits"`
	} `json:"hits"`
}

post_storer.go

package storage

...

type PostStorer interface {
	...
	// The returned `int` represents all the found docs, not the ones in the current page!
	ListAll(ctx context.Context, from, size int) (int, []*Post, error)
}

elasticsearch.go

package elasticsearch

...

// Validation
// from >= 0
// size >= 0
// from+size <= 10000
func (p PostStorage) ListAll(ctx context.Context, from, size int) (int, []*storage.Post, error) {
	// res, err := p.elastic.client.Search()
	req := esapi.SearchRequest{
		Index: []string{p.elastic.alias},
		From:  &from,
		Size:  &size,
	}

	ctx, cancel := context.WithTimeout(ctx, p.timeout)
	defer cancel()

	res, err := req.Do(ctx, p.elastic.client)
	if err != nil {
		return 0, nil, fmt.Errorf("list all: request: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		return 0, nil, fmt.Errorf("list all: response: %s", res.String())
	}

	var body postsResponse
	if err := json.NewDecoder(res.Body).Decode(&body); err != nil {
		return 0, nil, fmt.Errorf("list all: decode: %w", err)
	}

	posts := make([]*storage.Post, len(body.Hits.Hits))
	for i, v := range body.Hits.Hits {
		posts[i] = v.Source
	}

	return body.Hits.Total.Value, posts, nil
}