Hello everyone!

We have been investing plenty of personal time and energy for many years to share our knowledge with you all. However, we now need your help to keep this blog running. All you have to do is just click one of the adverts on the site, otherwise it will sadly be taken down due to hosting etc. costs. Thank you.

In this example we are going to use Elasticsearch's Search API to query the index and paginate the result set in Golang. We will be using the classic from and size parameters because from+size will always be less than or equal to 10000. However, if you cannot guarantee that, you should stick with Search after feature instead. The best thing about this feature is that it uses Point in Time API (PIT). When a refresh occurs between pagination requests, the order of the result may change which causes inconsistency across pages. The Point in Time API preserves the current index state to prevent such issue.


Files


This is follow up to the previous post A simple Elasticsearch CRUD example in Golang so I am not duplication all the files here.


elasticsearch.go


package elasticsearch

...

// postsResponse represents list of posts in Search API response body.
type postsResponse struct {
Hits struct {
Total struct {
Value int `json:"value"`
} `json:"total"`
Hits []struct {
Source *storage.Post `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}

post_storer.go


package storage

...

type PostStorer interface {
...
// The returned `int` represents all the found docs, not the ones in the current page!
ListAll(ctx context.Context, from, size int) (int, []*Post, error)
}

elasticsearch.go


package elasticsearch

...

// Validation
// from >= 0
// size >= 0
// from+size <= 10000
func (p PostStorage) ListAll(ctx context.Context, from, size int) (int, []*storage.Post, error) {
// res, err := p.elastic.client.Search()
req := esapi.SearchRequest{
Index: []string{p.elastic.alias},
From: &from,
Size: &size,
}

ctx, cancel := context.WithTimeout(ctx, p.timeout)
defer cancel()

res, err := req.Do(ctx, p.elastic.client)
if err != nil {
return 0, nil, fmt.Errorf("list all: request: %w", err)
}
defer res.Body.Close()

if res.IsError() {
return 0, nil, fmt.Errorf("list all: response: %s", res.String())
}

var body postsResponse
if err := json.NewDecoder(res.Body).Decode(&body); err != nil {
return 0, nil, fmt.Errorf("list all: decode: %w", err)
}

posts := make([]*storage.Post, len(body.Hits.Hits))
for i, v := range body.Hits.Hits {
posts[i] = v.Source
}

return body.Hits.Total.Value, posts, nil
}