23/01/2021 - AWS, GO
In this example we are going to use Localstack and Golang to work with AWS Simple Storage Service (S3). We will create a new bucket, upload an object to a bucket, download an object from a bucket, delete an object from a bucket and list objects in a bucket.
├── assets
│ ├── id.txt
│ └── logo.png
├── internal
│ ├── bucket
│ │ └── bucket.go
│ └── pkg
│ └── cloud
│ ├── aws
│ │ ├── aws.go
│ │ └── s3.go
│ ├── client.go
│ └── model.go
├── main.go
└── tmp
package main
import (
"log"
"time"
"github.com/you/aws/internal/bucket"
"github.com/you/aws/internal/pkg/cloud/aws"
)
func main() {
// Create a session instance.
ses, err := aws.New(aws.Config{
Address: "http://localhost:4566",
Region: "eu-west-1",
Profile: "localstack",
ID: "test",
Secret: "test",
})
if err != nil {
log.Fatalln(err)
}
// Test bucket
bucket.Bucket(aws.NewS3(ses, time.Second*5))
}
package cloud
import (
"context"
"io"
)
type BucketClient interface {
// Creates a new bucket.
Create(ctx context.Context, bucket string) error
// Upload a new object to a bucket and returns its URL to view/download.
UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error)
// Downloads an existing object from a bucket.
DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error
// Deletes an existing object from a bucket.
DeleteObject(ctx context.Context, bucket, fileName string) error
// Lists all objects in a bucket.
ListObjects(ctx context.Context, bucket string) ([]*Object, error)
// Returns an object from bucket for reading.
FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error)
}
package cloud
import "time"
type Object struct {
Key string
Size int64
ModifiedAt time.Time
}
package aws
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
)
type Config struct {
Address string
Region string
Profile string
ID string
Secret string
}
func New(config Config) (*session.Session, error) {
return session.NewSessionWithOptions(
session.Options{
Config: aws.Config{
Credentials: credentials.NewStaticCredentials(config.ID, config.Secret, ""),
Region: aws.String(config.Region),
Endpoint: aws.String(config.Address),
S3ForcePathStyle: aws.Bool(true),
},
Profile: config.Profile,
},
)
}
package aws
import (
"context"
"fmt"
"io"
"time"
"github.com/you/aws/internal/pkg/cloud"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
var _ cloud.BucketClient = S3{}
type S3 struct {
timeout time.Duration
client *s3.S3
uploader *s3manager.Uploader
downloader *s3manager.Downloader
}
func NewS3(session *session.Session, timeout time.Duration) S3 {
s3manager.NewUploader(session)
return S3{
timeout: timeout,
client: s3.New(session),
uploader: s3manager.NewUploader(session),
downloader: s3manager.NewDownloader(session),
}
}
func (s S3) Create(ctx context.Context, bucket string) error {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
if _, err := s.client.CreateBucketWithContext(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}); err != nil {
return fmt.Errorf("create: %w", err)
}
if err := s.client.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
}); err != nil {
return fmt.Errorf("wait: %w", err)
}
return nil
}
func (s S3) UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
res, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Body: body,
Bucket: aws.String(bucket),
Key: aws.String(fileName),
})
if err != nil {
return "", fmt.Errorf("upload: %w", err)
}
return res.Location, nil
}
func (s S3) DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error {
if _, err := s.downloader.DownloadWithContext(ctx, body, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("download: %w", err)
}
return nil
}
func (s S3) DeleteObject(ctx context.Context, bucket, fileName string) error {
if _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("delete: %w", err)
}
if err := s.client.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}); err != nil {
return fmt.Errorf("wait: %w", err)
}
return nil
}
func (s S3) ListObjects(ctx context.Context, bucket string) ([]*cloud.Object, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
res, err := s.client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
})
if err != nil {
return nil, fmt.Errorf("list: %w", err)
}
objects := make([]*cloud.Object, len(res.Contents))
for i, object := range res.Contents {
objects[i] = &cloud.Object{
Key: *object.Key,
Size: *object.Size,
ModifiedAt: *object.LastModified,
}
}
return objects, nil
}
func (s S3) FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error) {
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
res, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
})
if err != nil {
return nil, err
}
return res.Body, nil
}
This is just a "dirty" example of usages!
package bucket
import (
"context"
"fmt"
"log"
"os"
"github.com/you/aws/internal/pkg/cloud"
)
func Bucket(client cloud.BucketClient) {
ctx := context.Background()
create(ctx, client)
uploadObject(ctx, client)
downloadObject(ctx, client)
deleteObject(ctx, client)
listObjects(ctx, client)
}
func create(ctx context.Context, client cloud.BucketClient) {
if err := client.Create(ctx, "aws-test"); err != nil {
log.Fatalln(err)
}
log.Println("create: ok")
}
func uploadObject(ctx context.Context, client cloud.BucketClient) {
file, err := os.Open("./assets/id.txt")
if err != nil {
log.Fatalln(err)
}
defer file.Close()
url, err := client.UploadObject(ctx, "aws-test", "id.txt", file)
if err != nil {
log.Fatalln(err)
}
log.Println("upload object:", url)
}
func downloadObject(ctx context.Context, client cloud.BucketClient) {
file, err := os.Create("./tmp/id.txt")
if err != nil {
log.Fatalln(err)
}
defer file.Close()
if err := client.DownloadObject(ctx, "aws-test", "id.txt", file); err != nil {
log.Fatalln(err)
}
log.Println("download object: ok")
}
func deleteObject(ctx context.Context, client cloud.BucketClient) {
if err := client.DeleteObject(ctx, "aws-test", "id.txt"); err != nil {
log.Fatalln(err)
}
log.Println("delete object: ok")
}
func listObjects(ctx context.Context, client cloud.BucketClient) {
objects, err := client.ListObjects(ctx, "aws-test")
if err != nil {
log.Fatalln(err)
}
log.Println("list objects:")
for _, object := range objects {
fmt.Printf("%+v\n", object)
}
}
Obviously the "list objects" section will be empty because you deleted the file but I am manually adding something there for demonstration purposes.
$ go run --race main.go
2021/01/23 21:35:59 create: ok
2021/01/23 21:35:59 upload object: http://localhost:4566/aws-test/id.txt
2021/01/23 21:35:59 download object: ok
2021/01/23 21:35:59 delete object: ok
2021/01/23 21:35:59 list objects:
&{Key:id.txt Size:23 ModifiedAt:2021-01-23 21:36:30 +0000 UTC}