inanzzz | Using yaml for application configs and secrets in Golang

In this example we are going to see how YAML files can be used to manage application configs (config.yaml) and secrets (secret.yaml) in Golang. A few things to note:

Your structs must mirror relevant YAML files.

YAML files use anchor and alias to prevent duplicates.

You must not keep secret.yaml in the VCS or Docker images. It should be used in the local development mode.

Deployment environments use a third party vault provider for the actual secrets for security reasons. This example uses AWS SSM but there are other options like GCP Secret Manager, Azure Key Vault, HashiCorp Vault so on.

Secret values must contain string representation of any data. This includes structured versions such as slice, map and json in order to guarantee cross-platform compatibility between providers as it is common to all. e.g. "blue,yellow,green" or "key-1:val-1,key-2:val-2" or "{"key":true}"

Secret struct fields must not be pointers otherwise they will be as as null for the deployment environments.

Structure

├── Makefile
├── app
│   ├── config.go
│   └── secret.go
├── config.yaml
├── docker-compose.yaml
├── main.go
├── pkg
│   ├── xaws
│   │   ├── config.go
│   │   └── ssm.go
│   └── xconfig
│       ├── file.go
│       ├── object.go
│       ├── plain.go
│       └── secure.go
└── secret.yaml

Makefile

dev:
	@ENV=dev go run -race main.go

stg:
	@ENV=stg go run -race main.go

pro:
	@ENV=pro go run -race main.go

docker:
	docker-compose down
	docker system prune --volumes --force
	docker-compose up

config.yaml

# DEVELOPMENT ------------------------------------------------------------------
dev: &dev
  Service:
    Name: "blog"
  RoutingOutputFile: "rouing.md"
  Logger:
    Level: "debug"
  HTTP: &dev-http
    Server: &dev-http-server
      Host: "0.0.0.0"
      Port: "8080"
      Request:
        Timeout: "5s"
        Size: 1000000
    Client: &dev-http-client
      Retry: 0
  Postgres: &dev-postgres
    Port: "5432"
    Database: "blog"
    SSLMode: "disable"

# STAGING ----------------------------------------------------------------------
stg: &stg
  <<: *dev
  RoutingOutputFile: ""
  Logger:
    Level: "info"
  HTTP:
    <<: *dev-http
    Server:
      <<: *dev-http-server
      Host: "https://stg.blog.com"
    Client:
      <<: *dev-http-client
      Retry: 3
  Postgres:
    <<: *dev-postgres
    SSLMode: "verify-ca"

# PRODUCTION -------------------------------------------------------------------
pro:
  <<: *stg
  RoutingOutputFile: ""
  Logger:
    Level: "info"
  HTTP:
    <<: *dev-http
    Server:
      <<: *dev-http-server
      Host: "https://blog.com"
    Client:
      <<: *dev-http-client
      Retry: 5

secret.yaml

Staging and production blocks are unnecessary because when the application runs in corresponding environment, values will come from the Vault. I am just putting them here if you wish to comment out Vault in the code to see how it works in local environment.

# DEVELOPMENT ------------------------------------------------------------------
dev: &dev
  Postgres: &dev-postgres
    Host: "0.0.0.0"
    Username: "postgres"
    Password: "postgres"

# STAGING ----------------------------------------------------------------------
stg: &stg
  <<: *dev
  Postgres: &stg-postgres
    <<: *dev-postgres
    Host: "blog.staging.eu-west-1.rds.amazonaws.com"
    Username: "blog_staging"
    Password: "blog_staging_pass"

# PRODUCTION -------------------------------------------------------------------
pro:
  <<: *stg
  Postgres:
    <<: *stg-postgres
    Host: "blog.production.eu-west-1.rds.amazonaws.com"
    Username: "blog_production"
    Password: "blog_production_pass"

app/config.go

package app

type Config struct {
	Service struct {
		Name string
	}
	RoutingOutputFile string
	Logger            struct {
		Level string
	}
	HTTP struct {
		Server struct {
			Host    string
			Port    string
			Request struct {
				Timeout string
				Size    int
			}
		}
		Client struct {
			Retry int
		}
	}
	Postgres struct {
		Port     string
		Database string
		SSLMode  string
	}
}

app/secret.go

package app

type Secret struct {
	Postgres struct {
		Host     string
		Username string
		Password string
	}
}

aws/config.go

package xaws

import (
	"context"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
)

func Config(ctx context.Context) (aws.Config, error) {
	return config.LoadDefaultConfig(ctx)
}

aws/ssm/go

package xaws

import (
	"context"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/ssm"
)

type SSM struct {
	client *ssm.Client
	prefix string
}

func NewSSM(cfg aws.Config, prefix string) SSM {
	return SSM{
		client: ssm.NewFromConfig(cfg, func(o *ssm.Options) {
			o.BaseEndpoint = aws.String("http://localhost:4566")
		}),
		prefix: prefix,
	}
}

func (s SSM) ValuesByPaths(ctx context.Context, paths []string) (map[string]any, error) {
	for k, v := range paths {
		paths[k] = s.prefix + v
	}

	res, err := s.client.GetParameters(ctx, &ssm.GetParametersInput{
		Names: paths,
	})
	if err != nil {
		return nil, err
	}

	var (
		params = make(map[string]any)
		index  = len(s.prefix)
	)

	for _, param := range res.Parameters {
		params[(*param.Name)[index:]] = *param.Value
	}

	return params, nil
}

plain.go

package xconfig

type Plain struct {
	Path  string
	Block string
}

func (p Plain) Parse(target any) error {
	return (file{
		path:  p.Path,
		block: p.Block,
	}).parse(target)
}

secure.go

package xconfig

import (
	"context"
	"fmt"
)

type extractor interface {
	ValuesByPaths(ctx context.Context, paths []string) (map[string]any, error)
}

type Secure struct {
	Path     string
	Block    string
	Joiner   string
	UseVault bool
	Vault    extractor
}

func (s Secure) Parse(target any) error {
	if !s.UseVault {
		return (file{
			path:  s.Path,
			block: s.Block,
		}).parse(target)
	}

	source := object{
		joiner: s.Joiner,
	}

	paths := source.readFieldPaths(target)

	values, err := s.Vault.ValuesByPaths(context.Background(), paths)
	if err != nil {
		return fmt.Errorf("find values in vault: %w", err)
	}

	if err := source.writeFieldValues(target, values); err != nil {
		return err
	}

	return nil
}

file.go

package xconfig

import (
	"encoding/json"
	"fmt"
	"os"

	"gopkg.in/yaml.v3"
)

type file struct {
	path  string
	block string
}

func (f file) parse(target any) error {
	file, err := os.ReadFile(f.path)
	if err != nil {
		return fmt.Errorf("os read file: %w", err)
	}

	var content map[string]any
	if err := yaml.Unmarshal(file, &content); err != nil {
		return fmt.Errorf("yaml unmarshal: %w", err)
	}

	var (
		cfg any = content
		ok      = true
	)

	if f.block != "" {
		cfg, ok = content[f.block]
	}

	if !ok {
		return fmt.Errorf("block not found in yaml file: %s", f.block)
	}

	data, err := json.Marshal(cfg)
	if err != nil {
		return fmt.Errorf("json marshal: %w", err)
	}

	if err := json.Unmarshal(data, target); err != nil {
		return fmt.Errorf("json unmarshal: %w", err)
	}

	return nil
}

object.go

package xconfig

import (
	"fmt"
	"reflect"
)

type object struct {
	paths  []string
	joiner string
}

func (o *object) readFieldPaths(target any) []string {
	src := reflect.ValueOf(target)

	if src.Kind() == reflect.Ptr {
		src = reflect.Indirect(src)
	}

	o.read(src, "")

	return o.paths
}

func (o *object) read(target reflect.Value, parent string) {
	for i := 0; i < target.NumField(); i++ {
		field := target.Field(i)
		name := target.Type().Field(i).Name

		if field.Kind() == reflect.Struct {
			o.read(field, parent+name+o.joiner)

			continue
		}

		o.paths = append(o.paths, parent+name)
	}
}

func (o *object) writeFieldValues(target any, values map[string]any) error {
	src := reflect.ValueOf(target)

	if src.Kind() != reflect.Ptr {
		return fmt.Errorf("target is not a pointer: %s", src)
	}

	o.write(src, values, "")

	return nil
}

func (o *object) write(target reflect.Value, values map[string]any, parent string) {
	if target.Kind() == reflect.Ptr {
		target = target.Elem()
	}

	for i := 0; i < target.NumField(); i++ {
		field := target.Field(i)
		name := target.Type().Field(i).Name
		kind := field.Kind()

		if kind == reflect.Struct {
			o.write(field, values, parent+name+o.joiner)

			continue
		}

		if kind == reflect.Ptr {
			continue
		}

		value := target.FieldByName(name)
		if !value.CanSet() {
			continue
		}

		value.SetString(values[parent+name].(string))
	}
}

main.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"

	"config/app"
	"config/pkg/xaws"
	"config/pkg/xconfig"
)

func main() {
	local := "dev"

	env := local
	if val := os.Getenv("ENV"); val != "" {
		env = val
	}

	// DEPENDENCY --------------------------------------------------------------

	aws, err := xaws.Config(context.Background())
	if err != nil {
		log.Fatalln(err)
	}

	// PARSE -------------------------------------------------------------------

	var (
		config app.Config
		secret app.Secret
	)

	plain := xconfig.Plain{
		Path:  "config.yaml",
		Block: env,
	}
	if err := plain.Parse(&config); err != nil {
		log.Fatalln(err)
	}

	secure := xconfig.Secure{
		Path:     "secret.yaml",
		Block:    env,
		Joiner:   "/",
		UseVault: env != local,
		Vault:    xaws.NewSSM(aws, "/"+config.Service.Name+"/"+env+"/"),
	}
	if err := secure.Parse(&secret); err != nil {
		log.Fatalln(err)
	}

	// You can now use `config` and `secret` as you wish!

	// DUMP --------------------------------------------------------------------

	cfg, _ := json.MarshalIndent(config, "", "  ")
	scr, _ := json.MarshalIndent(secret, "", "  ")

	fmt.Println("CONFIG", strings.Repeat("-", 50))
	fmt.Println(string(cfg))

	fmt.Println("SECRET", strings.Repeat("-", 50))
	fmt.Println(string(scr))
}

docker-compose.yaml

version: "3.8"

services:
  localstack:
    image: "localstack/localstack"
    container_name: "localstack"
    ports:
      - "4566:4566"
    environment:
      - DEBUG=1
      - SERVICES=ssm
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"

Setup

Run make docker first then commands below.

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/stg/Postgres/Host" \
    --type String \
    --value "blog.1111111111.eu-west-1.rds.amazonaws.com" \
    --tier Standard \
    --overwrite

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/stg/Postgres/Username" \
    --type String \
    --value "blog_staging" \
    --tier Standard \
    --overwrite

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/stg/Postgres/Password" \
    --type String \
    --value "blog_staging_pass" \
    --tier Standard \
    --overwrite

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/pro/Postgres/Host" \
    --type String \
    --value "blog.2222222222.eu-west-1.rds.amazonaws.com" \
    --tier Standard \
    --overwrite

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/pro/Postgres/Username" \
    --type String \
    --value "blog_production" \
    --tier Standard \
    --overwrite

aws --endpoint-url http://localhost:4566 ssm put-parameter \
    --name "/blog/pro/Postgres/Password" \
    --type String \
    --value "blog_production_pass" \
    --tier Standard \
    --overwrite

Test

$ make dev

CONFIG --------------------------------------------------
{
  "Service": {
    "Name": "blog"
  },
  "RoutingOutputFile": "rouing.md",
  "Logger": {
    "Level": "debug"
  },
  "HTTP": {
    "Server": {
      "Host": "0.0.0.0",
      "Port": "8080",
      "Request": {
        "Timeout": "5s",
        "Size": 1000000
      }
    },
    "Client": {
      "Retry": 0
    }
  },
  "Postgres": {
    "Port": "5432",
    "Database": "blog",
    "SSLMode": "disable"
  }
}
SECRET --------------------------------------------------
{
  "Postgres": {
    "Host": "0.0.0.0",
    "Username": "postgres",
    "Password": "postgres"
  }
}

$make stg

CONFIG --------------------------------------------------
{
  "Service": {
    "Name": "blog"
  },
  "RoutingOutputFile": "",
  "Logger": {
    "Level": "info"
  },
  "HTTP": {
    "Server": {
      "Host": "https://stg.blog.com",
      "Port": "8080",
      "Request": {
        "Timeout": "5s",
        "Size": 1000000
      }
    },
    "Client": {
      "Retry": 3
    }
  },
  "Postgres": {
    "Port": "5432",
    "Database": "blog",
    "SSLMode": "verify-ca"
  }
}
SECRET --------------------------------------------------
{
  "Postgres": {
    "Host": "blog.1111111111.eu-west-1.rds.amazonaws.com",
    "Username": "blog_staging",
    "Password": "blog_staging_pass"
  }
}

$ make pro
CONFIG --------------------------------------------------
{
  "Service": {
    "Name": "blog"
  },
  "RoutingOutputFile": "",
  "Logger": {
    "Level": "info"
  },
  "HTTP": {
    "Server": {
      "Host": "https://blog.com",
      "Port": "8080",
      "Request": {
        "Timeout": "5s",
        "Size": 1000000
      }
    },
    "Client": {
      "Retry": 5
    }
  },
  "Postgres": {
    "Port": "5432",
    "Database": "blog",
    "SSLMode": "verify-ca"
  }
}
SECRET --------------------------------------------------
{
  "Postgres": {
    "Host": "blog.2222222222.eu-west-1.rds.amazonaws.com",
    "Username": "blog_production",
    "Password": "blog_production_pass"
  }
}