In this example we are going to use Localstack and Golang to work with AWS Simple Storage Service (S3). We will create a new bucket, upload an object to a bucket, download an object from a bucket, delete an object from a bucket and list objects in a bucket.


Structure


├── assets
│   ├── id.txt
│   └── logo.png
├── internal
│   ├── bucket
│   │   └── bucket.go
│   └── pkg
│   └── cloud
│   ├── aws
│   │   ├── aws.go
│   │   └── s3.go
│   ├── client.go
│   └── model.go
├── main.go
└── tmp

Files


main.go


package main

import (
"log"
"time"

"github.com/you/aws/internal/bucket"
"github.com/you/aws/internal/pkg/cloud/aws"
)

func main() {
// Create a session instance.
ses, err := aws.New(aws.Config{
Address: "http://localhost:4566",
Region: "eu-west-1",
Profile: "localstack",
ID: "test",
Secret: "test",
})
if err != nil {
log.Fatalln(err)
}

// Test bucket
bucket.Bucket(aws.NewS3(ses, time.Second*5))
}

client.go


package cloud

import (
"context"
"io"
)

type BucketClient interface {
// Creates a new bucket.
Create(ctx context.Context, bucket string) error
// Upload a new object to a bucket and returns its URL to view/download.
UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error)
// Downloads an existing object from a bucket.
DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error
// Deletes an existing object from a bucket.
DeleteObject(ctx context.Context, bucket, fileName string) error
// Lists all objects in a bucket.
ListObjects(ctx context.Context, bucket string) ([]*Object, error)
// Returns an object from bucket for reading.
FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error)
}

model.go


package cloud

import "time"

type Object struct {
Key string
Size int64
ModifiedAt time.Time
}

aws.go


package aws

import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
)

type Config struct {
Address string
Region string
Profile string
ID string
Secret string
}

func New(config Config) (*session.Session, error) {
return session.NewSessionWithOptions(
session.Options{
Config: aws.Config{
Credentials: credentials.NewStaticCredentials(config.ID, config.Secret, ""),
Region: aws.String(config.Region),
Endpoint: aws.String(config.Address),
S3ForcePathStyle: aws.Bool(true),
},
Profile: config.Profile,
},
)
}

s3.go


package aws

import (
"context"
"fmt"
"io"
"time"

"github.com/you/aws/internal/pkg/cloud"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

var _ cloud.BucketClient = S3{}

type S3 struct {
timeout time.Duration
client *s3.S3
uploader *s3manager.Uploader
downloader *s3manager.Downloader
}

func NewS3(session *session.Session, timeout time.Duration) S3 {
s3manager.NewUploader(session)
return S3{
timeout: timeout,
client: s3.New(session),
uploader: s3manager.NewUploader(session),
downloader: s3manager.NewDownloader(session),
}
}

func (s S3) Create(ctx context.Context, bucket string) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()

if _, err := s.client.CreateBucketWithContext(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}); err != nil {
return fmt.Errorf("create: %w", err)
}

if err := s.client.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
}); err != nil {
return fmt.Errorf("wait: %w", err)
}

return nil
}

func (s S3) UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()

res, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Body: body,
Bucket: aws.String(bucket),
Key: aws.String(fileName),
})
if err != nil {
return "", fmt.Errorf("upload: %w", err)
}

return res.Location, nil
}

func (s S3) DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error {
if _, err := s.downloader.DownloadWithContext(ctx, body, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("download: %w", err)
}

return nil
}

func (s S3) DeleteObject(ctx context.Context, bucket, fileName string) error {
if _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("delete: %w", err)
}

if err := s.client.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("wait: %w", err)
}

return nil
}

func (s S3) ListObjects(ctx context.Context, bucket string) ([]*cloud.Object, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()

res, err := s.client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
})
if err != nil {
return nil, fmt.Errorf("list: %w", err)
}

objects := make([]*cloud.Object, len(res.Contents))

for i, object := range res.Contents {
objects[i] = &cloud.Object{
Key: *object.Key,
Size: *object.Size,
ModifiedAt: *object.LastModified,
}
}

return objects, nil
}

func (s S3) FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()

res, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
})
if err != nil {
return nil, err
}

return res.Body, nil
}

bucket.go


This is just a "dirty" example of usages!


package bucket

import (
"context"
"fmt"
"log"
"os"

"github.com/you/aws/internal/pkg/cloud"
)

func Bucket(client cloud.BucketClient) {
ctx := context.Background()

create(ctx, client)
uploadObject(ctx, client)
downloadObject(ctx, client)
deleteObject(ctx, client)
listObjects(ctx, client)
}

func create(ctx context.Context, client cloud.BucketClient) {
if err := client.Create(ctx, "aws-test"); err != nil {
log.Fatalln(err)
}
log.Println("create: ok")
}

func uploadObject(ctx context.Context, client cloud.BucketClient) {
file, err := os.Open("./assets/id.txt")
if err != nil {
log.Fatalln(err)
}
defer file.Close()

url, err := client.UploadObject(ctx, "aws-test", "id.txt", file)
if err != nil {
log.Fatalln(err)
}
log.Println("upload object:", url)
}

func downloadObject(ctx context.Context, client cloud.BucketClient) {
file, err := os.Create("./tmp/id.txt")
if err != nil {
log.Fatalln(err)
}
defer file.Close()

if err := client.DownloadObject(ctx, "aws-test", "id.txt", file); err != nil {
log.Fatalln(err)
}
log.Println("download object: ok")
}

func deleteObject(ctx context.Context, client cloud.BucketClient) {
if err := client.DeleteObject(ctx, "aws-test", "id.txt"); err != nil {
log.Fatalln(err)
}
log.Println("delete object: ok")
}

func listObjects(ctx context.Context, client cloud.BucketClient) {
objects, err := client.ListObjects(ctx, "aws-test")
if err != nil {
log.Fatalln(err)
}
log.Println("list objects:")
for _, object := range objects {
fmt.Printf("%+v\n", object)
}
}

Test


Obviously the "list objects" section will be empty because you deleted the file but I am manually adding something there for demonstration purposes.


$ go run --race main.go
2021/01/23 21:35:59 create: ok
2021/01/23 21:35:59 upload object: http://localhost:4566/aws-test/id.txt
2021/01/23 21:35:59 download object: ok
2021/01/23 21:35:59 delete object: ok
2021/01/23 21:35:59 list objects:
&{Key:id.txt Size:23 ModifiedAt:2021-01-23 21:36:30 +0000 UTC}