Using Nginx Ingress controller and internal Nginx proxy to divert traffic in Kubernetes

29/12/2020 - KUBERNETES, NGINX

In this example we are going to design an architecture where Kubernetes Nginx Ingress controller and internal Nginx proxy work together to divert traffic to relevant service/pods. This is handled by reading the domain at Ingress level first then reading the rest of the URL at internal Nginx proxy level. The reason why we are not doing everything at Ingress level is because it would have limitations like background calls, manipulating headers so on.


We are accepting HTTPS request from outside and use HTTP for internal communications. Ingress is responsible for SSL termination. See whole workflow below. A small note for the design below; you would normally put an additional proxy between the client and Ingress in order to prevent your Kubernetes cluster from being exposed to outside completely, but it is out of context at this stage.



  1. GATEWAY: This is our actual Ingress and exposed to external traffic. Accepts HTTPS request externally and forwards HTTP request to internal services.

  2. DOC: This represents our company API documentation and served at doc.my-company.com domain. Ingress directly talks to this service.

  3. PROXY: This is the internal Nginx proxy app. Ingress directly talks to this service. Any api.my-company.com domain specific traffic is handled by this service and diverted to relevant services/pods.

  4. OAUTH: This is our internal OAuth service. Every request could be sent to this service first before being passed on to other services for authentication/authorisation purposes.

  5. API: This is some random internal service.

API


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Files


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: api-deployment
  labels:
    app: api

spec:
  replicas: 1
  selector:
    matchLabels:
      app: api
  template:
    metadata:
      labels:
        app: api
    spec:
      containers:
        - name: golang
          image: you/api:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: api-service

spec:
  type: ClusterIP
  selector:
    app: api
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("api running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/api/v1/clients", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 clients list\n"))
	})
	rtr.HandleFunc("/api/v1/clients/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 single client\n"))
	})
	rtr.HandleFunc("/api/v1/customers/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 customer list\n"))
	})
	rtr.HandleFunc("/api/v1/customers/111/logs", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 single customer's logs\n"))
	})

	// V2
	rtr.HandleFunc("/api/v2/clients", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 clients list\n"))
	})
	rtr.HandleFunc("/api/v2/clients/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 single client\n"))
	})
	rtr.HandleFunc("/api/v2/customers/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 customer list\n"))
	})
	rtr.HandleFunc("/api/v2/customers/111/logs", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 single customer's logs\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/api/v1/clients
	curl --request GET http://localhost:3000/api/v1/clients/111
	curl --request GET http://localhost:3000/api/v1/customers/111
	curl --request GET http://localhost:3000/api/v1/customers/111/logs
	curl --request GET http://localhost:3000/api/v2/clients
	curl --request GET http://localhost:3000/api/v2/clients/111
	curl --request GET http://localhost:3000/api/v2/customers/111
	curl --request GET http://localhost:3000/api/v2/customers/111/logs

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/api:latest -f docker/Dockerfile .
	docker push you/api:latest
	docker rmi you/api:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

OAUTH


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Files


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: oauth-deployment
  labels:
    app: oauth

spec:
  replicas: 1
  selector:
    matchLabels:
      app: oauth
  template:
    metadata:
      labels:
        app: oauth
    spec:
      containers:
        - name: golang
          image: you/oauth:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: oauth-service

spec:
  type: ClusterIP
  selector:
    app: oauth
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("oauth running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/api/v1/authorization", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 authorization code\n"))
	})
	rtr.HandleFunc("/api/v1/oauth/token", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 oauth code\n"))
	})

	// V2
	rtr.HandleFunc("/api/v2/authorization", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 authorization code\n"))
	})
	rtr.HandleFunc("/api/v2/oauth/token", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 oauth code\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/api/v1/authorization
	curl --request GET http://localhost:3000/api/v1/oauth/token
	curl --request GET http://localhost:3000/api/v2/authorization
	curl --request GET http://localhost:3000/api/v2/oauth/token

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/oauth:latest -f docker/Dockerfile .
	docker push you/oauth:latest
	docker rmi you/oauth:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

DOC


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Files


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: doc-deployment
  labels:
    app: doc

spec:
  replicas: 1
  selector:
    matchLabels:
      app: doc
  template:
    metadata:
      labels:
        app: doc
    spec:
      containers:
        - name: golang
          image: you/doc:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: doc-service

spec:
  type: ClusterIP
  selector:
    app: doc
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("doc running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/v1", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 docs\n"))
	})

	// V2
	rtr.HandleFunc("/v2", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 docs\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/v1
	curl --request GET http://localhost:3000/v2

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/doc:latest -f docker/Dockerfile .
	docker push you/doc:latest
	docker rmi you/doc:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

PROXY


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
└── docker
    ├── Dockerfile
    └── default.conf

Files


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: proxy-deployment
  labels:
    app: proxy

spec:
  replicas: 1
  selector:
    matchLabels:
      app: proxy
  template:
    metadata:
      labels:
        app: proxy
    spec:
      containers:
        - name: nginx
          image: you/proxy:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 80

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: proxy-service

spec:
  type: ClusterIP
  selector:
    app: proxy
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 80

Dockerfile

FROM nginx:1.19.5-alpine

COPY docker/default.conf /etc/nginx/conf.d/default.conf

default.conf

server {
    listen 80;
    # 443

    resolver kube-dns.kube-system.svc.cluster.local;

    ## oauth service
    location ~ ^/(?<version>v\d+)/authorization$ {
        proxy_pass http://oauth-service.default.svc.cluster.local:80/api/$version/authorization$is_args$args;
    }
    location ~ ^/(?<version>v\d+)/oauth/token$ {
        proxy_pass http://oauth-service.default.svc.cluster.local:80/api/$version/oauth/token$is_args$args;
    }
    ## end ################

    ## api service
    location ~ ^/(?<version>v\d+)/clients(?<suffix>.*) {
        proxy_pass http://api-service.default.svc.cluster.local:80/api/$version/clients$suffix$is_args$args;
    }
    location ~ ^/(?<version>v\d+)/customers(?<suffix>.*) {
        proxy_pass http://api-service.default.svc.cluster.local:80/api/$version/customers$suffix$is_args$args;
    }
    ## end ################

    ## all unmatched urls
    location / {
        default_type application/json;
        return 404 '{"operation": "client", "code": "route_not_found", "message": "The route cannot be found"}';
    }
    ## end ################
}

Makefile

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/proxy:latest -f docker/Dockerfile .
	docker push you/proxy:latest
	docker rmi you/proxy:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

GATEWAY


├── Makefile
├── cert
│   └── cert.conf
└── deploy
    └── k8s
        └── ingress.yaml

Files


ingress.yaml

apiVersion: networking.k8s.io/v1beta1
kind: Ingress

metadata:
  name: gateway-ingress

spec:
  tls:
    - hosts:
        - doc.my-company.com
        - api.my-company.com
      secretName: gateway-tls-secret
  rules:
    - host: doc.my-company.com
      http:
        paths:
          - path: /
            backend:
              serviceName: doc-service
              servicePort: 80

    - host: api.my-company.com
      http:
        paths:
          - path: /
            backend:
              serviceName: proxy-service
              servicePort: 80

cert.conf

[ req ]
default_bits = 2048
prompt = no
default_md = sha256
req_extensions = req_ext
distinguished_name = dn

[ dn ]
C = UK
ST = London
L = London
O = Your Ltd.
OU = Information Technologies
emailAddress = email@email.com
CN = my-company.com

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = my-company.com
DNS.2 = www.my-company.com
DNS.3 = doc.my-company.com
DNS.4 = api.my-company.com

Makefile

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/ingress.yaml

## Send a dummy request to the exposed service on kubernetes.
.PHONY: k8s-test-doc
k8s-test-doc:
	curl --insecure -I --request GET https://doc.my-company.com/v1
	curl --insecure -I --request GET https://doc.my-company.com/v2

## Send a dummy request to the exposed service on kubernetes.
.PHONY: k8s-test-api
k8s-test-api:
	curl --insecure -I --request GET https://api.my-company.com/v1/authorization
	curl --insecure -I --request GET https://api.my-company.com/v1/oauth/token
	curl --insecure -I --request GET https://api.my-company.com/v2/authorization
	curl --insecure -I --request GET https://api.my-company.com/v2/oauth/token
	curl --insecure -I --request GET https://api.my-company.com/v1/clients
	curl --insecure -I --request GET https://api.my-company.com/v1/clients/111
	curl --insecure -I --request GET https://api.my-company.com/v1/customers/111
	curl --insecure -I --request GET https://api.my-company.com/v1/customers/111/logs
	curl --insecure -I --request GET https://api.my-company.com/v2/clients
	curl --insecure -I --request GET https://api.my-company.com/v2/clients/111
	curl --insecure -I --request GET https://api.my-company.com/v2/customers/111
	curl --insecure -I --request GET https://api.my-company.com/v2/customers/111/logs

Setup


We need to create SSL certificates, Kubernetes secret to go with and add domains to /etc/hosts file. This is required for the GATEWAY service.


Certificates


$ openssl req -nodes -new -x509 -sha256 -days 1825 \
   -config cert/cert.conf -extensions 'req_ext' \
   -keyout ~/Desktop/k8s.key -out ~/Desktop/k8s.crt

Secret


$ kubectl create secret tls gateway-tls-secret \
   --cert=~/Desktop/k8s.crt \
   --key=~/Desktop/k8s.key

Hosts


You would normally do this after deploying the GATEWAY service but I already know it so showing it here. You will need to change the IP below to kubectl get ingress gateway-ingress -o yaml | grep ip command output.


# /etc/hosts
192.168.99.100 doc.my-company.com
192.168.99.100 api.my-company.com

Deployment


First you need to push your service images then deploy.


api$ make push
oauth$ make push
doc$ make push
proxy$ make push

api$ make deploy
oauth$ make deploy
doc$ make deploy
proxy$ make deploy
gateway$ make deploy

Test


gateway$ make k8s-test-doc
curl --insecure --request GET https://doc.my-company.com/v1
get v1 docs
curl --insecure --request GET https://doc.my-company.com/v2
get v2 docs

gateway$ make k8s-test-api
curl --insecure --request GET https://api.my-company.com/v1/authorization
get v1 authorization code
curl --insecure --request GET https://api.my-company.com/v1/oauth/token
get v1 oauth code
curl --insecure --request GET https://api.my-company.com/v2/authorization
get v2 authorization code
curl --insecure --request GET https://api.my-company.com/v2/oauth/token
get v2 oauth code
curl --insecure --request GET https://api.my-company.com/v1/clients
get v1 clients list
curl --insecure --request GET https://api.my-company.com/v1/clients/111
get v1 single client
curl --insecure --request GET https://api.my-company.com/v1/customers/111
get v1 customer list
curl --insecure --request GET https://api.my-company.com/v1/customers/111/logs
get v1 single customers logs
curl --insecure --request GET https://api.my-company.com/v2/clients
get v2 clients list
curl --insecure --request GET https://api.my-company.com/v2/clients/111
get v2 single client
curl --insecure --request GET https://api.my-company.com/v2/customers/111
get v2 customer list
curl --insecure --request GET https://api.my-company.com/v2/customers/111/logs
get v2 single customers logs