Application deployment with Helm and GitHub actions to Kubernetes cluster
03/02/2022 - DOCKER, GIT, GO, HELM, KUBERNETES
In this example we are going to use GitHub actions to deploy our Golang application to a Kubernetes cluster using Helm. I am going to use a local Minikube cluster so there will be a "setup" section below to expose it to public with Ngrok.
I have local Kubernetes setup as listed below but we are just going to work with nonprod cluster and develop namespace.
CLUSTER NAMESPACES
prod default
nonprod develop, test, sandbox
Structure
├── Makefile
├── .dockerignore
├── .gitignore
├── .github
│ └── workflows
│ ├── cd.yaml
│ └── ci.yaml
├── infra
│ ├── docker
│ │ └── Dockerfile
│ └── helm
│ ├── .helmignore
│ ├── Chart.yaml
│ ├── templates
│ │ ├── configmap.yaml
│ │ ├── deployment.yaml
│ │ └── service.yaml
│ └── values.yaml
├── main.go
└── main_test.go
Files
Makefile
This is for local usage only.
TAG := $(shell git rev-parse --short HEAD)
.PHONY: help
help: ## Display available commands.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
# LOCAL ------------------------------------------------------------------------
.PHONY: build
build: ## Build service binary
go build -race -ldflags "-s -w -X main.ver=${TAG}" -o rest main.go
.PHONY: run
run: ## Run service
HTTP_ADDR=:8080 ./rest
# DOCKER -----------------------------------------------------------------------
.PHONY: docker-push
docker-push: ## Build, tag and push service image to registry then clean up local environment
docker build --build-arg VER=${TAG} -t you/rest:${TAG} -f infra/docker/Dockerfile .
docker push you/rest:${TAG}
docker rmi you/rest:${TAG}
docker system prune --volumes --force
# DEPLOY -----------------------------------------------------------------------
.PHONY: helm-deploy
helm-deploy: ## Deploy go applications.
helm upgrade --install --atomic --timeout 1m rest infra/helm/ -f infra/helm/values.yaml \
--kube-context nonprod --namespace develop --create-namespace \
--set image.tag=${TAG}
.dockerignore
.dockerignore
.gitignore
*.md
Makefile
rest
infra/
.git/
.idea/
.DS_Store/
.gitignore
rest
.DS_Store
.idea/
main.go
package main
import (
"log"
"net/http"
"os"
)
var ver string
func main() {
rtr := http.DefaultServeMux
rtr.HandleFunc("/", home{}.handle)
addr := os.Getenv("HTTP_ADDR")
log.Printf("%s: info: http listen and serve: %s", ver, addr)
if err := http.ListenAndServe(addr, rtr); err != nil && err != http.ErrServerClosed {
log.Printf("%s: error: http listen and serve: %s", ver, err)
}
}
type home struct{}
func (h home) handle(w http.ResponseWriter, r *http.Request) {
log.Printf("%s: info: X-Request-ID: %s\n", ver, r.Header.Get("X-Request-ID"))
_, _ = w.Write([]byte(ver))
}
main_test.go
package main
import (
"net/http"
"net/http/httptest"
"testing"
)
func Test_home_handle(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
res := httptest.NewRecorder()
home{}.handle(res, req)
if res.Code != http.StatusOK {
t.Error("expected 200 but got", res.Code)
}
}
Dockerfile
FROM golang:1.17.5-alpine3.15 as build
WORKDIR /source
COPY . .
ARG VER
RUN CGO_ENABLED=0 go build -ldflags "-s -w -X main.ver=${VER}" -o rest main.go
FROM alpine:3.15
COPY --from=build /source/rest /rest
EXPOSE 8080
ENTRYPOINT ["./rest"]
.helmignore
.DS_Store
.git/
.gitignore
.idea/
*.md
Chart.yaml
apiVersion: v2
name: rest
type: application
icon: https://
description: This is an HTTP API
version: 0.0.1
values.yaml
namespace: default
env:
HTTP_ADDR: :8080
image:
name: you/rest
tag: latest
pull: IfNotPresent
deployment:
timestamp: 2006-01-02T15:04:05
replicas: 1
container:
name: go
port: 8080
service:
type: ClusterIP
port: 8080
configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}
namespace: {{ .Release.Namespace | default .Values.namespace }}
data:
HTTP_ADDR: {{ .Values.env.HTTP_ADDR }}
deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}
namespace: {{ .Release.Namespace | default .Values.namespace }}
labels:
app: {{ .Release.Name }}
spec:
replicas: {{ .Values.deployment.replicas }}
selector:
matchLabels:
app: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ .Release.Name }}
annotations:
timestamp: {{ now | date .Values.deployment.timestamp }}
spec:
containers:
- name: {{ .Values.deployment.container.name }}
image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pull }}
ports:
- containerPort: {{ .Values.deployment.container.port }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}
service.yaml
apiVersion: v1
kind: Service
metadata:
name: {{ .Release.Name }}
namespace: {{ .Release.Namespace | default .Values.namespace }}
spec:
type: {{ .Values.service.type }}
selector:
app: {{ .Release.Name }}
ports:
- port: {{ .Values.service.port }}
targetPort: {{ .Values.deployment.container.port }}
cd.yaml
# Trigger the workflow to deploy to "nonprod" cluster using "develop" environment only when:
# - an existing pull request with any name/type is merged to the master or develop branch
# - a commit is directly pushed to the master or develop branch
name: Continuous Deployment
on:
push:
branches:
- master
- develop
jobs:
setup:
runs-on: ubuntu-latest
outputs:
repo: ${{ steps.vars.outputs.repo }}
tag: ${{ steps.vars.outputs.tag }}
steps:
- name: Use repository
uses: actions/checkout@v2
- name: Build variables
id: vars
run: |
echo "::set-output name=repo::$GITHUB_REPOSITORY"
echo "::set-output name=tag::$(git rev-parse --short "$GITHUB_SHA")"
- name: Upload repository
uses: actions/upload-artifact@v2
with:
name: repository
path: |
${{ github.workspace }}/infra
${{ github.workspace }}/.dockerignore
${{ github.workspace }}/main.go
${{ github.workspace }}/main_test.go
${{ github.workspace }}/go.mod
${{ github.workspace }}/go.sum
test:
needs: setup
runs-on: ubuntu-latest
steps:
- name: Use Golang 1.17
uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Download repository
uses: actions/download-artifact@v2
with:
name: repository
- name: Run tests
run: go test -v -race -timeout=180s -count=1 -cover ./...
docker:
needs: [setup, test]
runs-on: ubuntu-latest
steps:
- name: Download repository
uses: actions/download-artifact@v2
with:
name: repository
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push image
uses: docker/build-push-action@v2
with:
push: true
file: ./infra/docker/Dockerfile
tags: ${{ needs.setup.outputs.repo }}:${{ needs.setup.outputs.tag }}
build-args: VER=${{ needs.setup.outputs.tag }}
deploy:
needs: [setup, docker]
runs-on: ubuntu-latest
steps:
- name: Download repository
uses: actions/download-artifact@v2
with:
name: repository
- name: Create kube config
run: |
mkdir -p $HOME/.kube/
echo "${{ secrets.KUBE_CONFIG }}" > $HOME/.kube/config
chmod 600 $HOME/.kube/config
- name: Install helm
run: |
curl -LO https://get.helm.sh/helm-v3.8.0-linux-amd64.tar.gz
tar -zxvf helm-v3.8.0-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/helm
helm version
- name: Lint helm charts
run: helm lint ./infra/helm/
- name: Deploy
run: |
helm upgrade --install --atomic --timeout 1m rest ./infra/helm/ -f ./infra/helm/values.yaml \
--kube-context nonprod --namespace develop --create-namespace \
--set image.tag=${{ needs.setup.outputs.tag }}
ci.yaml
# Trigger the workflow only when:
# - a new pull request with any name/type is opened against the master, develop, hotfix/* or release/* branch
# - a commit is directly pushed to the pull request
name: Continuous Integration
on:
pull_request:
branches:
- master
- develop
- hotfix/*
- release/*
jobs:
setup:
runs-on: ubuntu-latest
steps:
- name: Use repository
uses: actions/checkout@v2
- name: Upload repository
uses: actions/upload-artifact@v2
with:
name: repository
path: |
${{ github.workspace }}/main.go
${{ github.workspace }}/main_test.go
${{ github.workspace }}/go.mod
${{ github.workspace }}/go.sum
test:
needs: setup
runs-on: ubuntu-latest
steps:
- name: Use Golang 1.17
uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Download repository
uses: actions/download-artifact@v2
with:
name: repository
- name: Run tests
run: go test -v -race -timeout=180s -count=1 -cover ./...
Debug
Run command below to see how Helm interprets Kubernetes files.
$ helm template ./infra/helm/
---
# Source: rest/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: release-name
namespace: default
data:
HTTP_ADDR: :8080
---
# Source: rest/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
name: release-name
namespace: default
spec:
type: ClusterIP
selector:
app: release-name
ports:
- port: 8080
targetPort: 8080
---
# Source: rest/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: release-name
namespace: default
labels:
app: release-name
spec:
replicas: 1
selector:
matchLabels:
app: release-name
template:
metadata:
labels:
app: release-name
annotations:
timestamp: 2022-02-03T09:58:35
spec:
containers:
- name: go
image: "you/rest:latest"
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
envFrom:
- configMapRef:
name: release-name
Setup
Add Docker secrets and Kube config to GitHub
This will let your workflow to push images to DockerHub registry.
DOCKERHUB_USERNAME
DOCKERHUB_TOKEN
This will let your workflows to deploy the application to your local K8S clusters.
// Enable routing public requests to K8S API.
$ kubectl proxy --disable-filter=true
Starting to serve on 127.0.0.1:8001
(Optionally $ kubectl proxy --port=8001 --accept-hosts='.*\.ngrok.io')
// Expose kubectl proxy to the Internet with ngrok.
$ ./ngrok http 127.0.0.1:8001
Forwarding http://117b-2a02-c7f-e84f-c900-85c1-38ee-a128-9cec.ngrok.io
Forwarding https://117b-2a02-c7f-e84f-c900-85c1-38ee-a128-9cec.ngrok.io
// Create a modified copy of your local kube config.
$ kubectl config view --flatten > ~/Desktop/kube_config
Remove `certificate-authority-data` line
Add `insecure-skip-tls-verify: true` line
Replace `server` value to `https://117b-2a02-c7f-e84f-c900-85c1-38ee-a128-9cec.ngrok.io`
Finally it should look like something like below.
```
apiVersion: v1
clusters:
- cluster:
insecure-skip-tls-verify: true
server: https://117b-2a02-c7f-e84f-c900-85c1-38ee-a128-9cec.ngrok.io
name: nonprod
- cluster:
insecure-skip-tls-verify: true
server: https://117b-2a02-c7f-e84f-c900-85c1-38ee-a128-9cec.ngrok.io
name: prod
contexts:
- context:
cluster: nonprod
user: nonprod
name: nonprod
- context:
cluster: prod
user: prod
name: prod
current-context: nonprod
kind: Config
preferences: {}
users:
- name: nonprod
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0F...==
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVp...=
- name: prod
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0F...==
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVk...=
```
Copy this content into `KUBE_CONFIG` GitHub secret.
Finally, you should have these secrets in GitHub.
DOCKERHUB_USERNAME
DOCKERHUB_TOKEN
KUBE_CONFIG
Prepare local clusters
$ minikube start -p prod --vm-driver=virtualbox
$ minikube start -p nonprod --vm-driver=virtualbox
$ kubectl config get-contexts
CURRENT NAME CLUSTER AUTHINFO NAMESPACE
* nonprod nonprod nonprod
prod prod prod
Test
Continuous Integration
Continuous Deployment
Verify deployment
Cluster info
$ helm list --kube-context nonprod --all-namespaces
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
rest develop 2 2022-02-02 18:07:38.982227388 +0000 UTC deployed rest-0.0.1
$ kubectl --context nonprod --namespace develop get all
NAME READY STATUS RESTARTS AGE
pod/rest-749fcd6464-54n8r 1/1 Running 0 5h29m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/rest ClusterIP 10.100.20.2078080/TCP 5h43m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/rest 1/1 1 1 5h43m
NAME DESIRED CURRENT READY AGE
replicaset.apps/rest-749fcd6464 1 1 1 5h29m
Accessing the application
$ kubectl --context nonprod --namespace develop port-forward service/rest 8080:8080
Forwarding from 127.0.0.1:8080 -> 8080
Forwarding from [::1]:8080 -> 8080
Example request
$ curl --request GET 'http://localhost:8080/' --header 'X-Request-ID: 123'
f11148a
Logs
$ kubectl --context nonprod --namespace develop logs -f service/rest
2022/02/02 18:08:00 f11148a: info: http listen and serve: :8080
2022/02/03 10:27:51 f11148a: info: X-Request-ID: 123