Kubernetes'te trafiği yönlendirmek için Nginx Ingress denetleyicisini ve dahili Nginx proxy'sini kullanma

29/12/2020 - KUBERNETES, NGINX

Bu örnekte, Kubernetes Nginx Ingress denetleyicisinin ve dahili Nginx proxy'nin trafiği ilgili Podlara yönlendirmek için birlikte çalıştığı bir mimari tasarlayacağız. Bu, önce domain adresini Ingress düzeyinde okuyarak, ardından URL'nin geri kalanını dahili Nginx proxy düzeyinde okuyarak ele alır. Her şeyi Ingress düzeyinde yapmamamızın nedeni, arka plan çağrıları, başlıkları değiştirme gibi sınırlamalara sahip olmasıdır.


Dışarıdan HTTPS talebini kabul ediyoruz ve dahili iletişim için HTTP kullanıyoruz. Ingress, SSL sonlandırmasından sorumludur. Aşağıdaki iş akışına bakabilirsiniz. Aşağıdaki tasarım için küçük bir not: Kubernetes kümenizin tamamen dışarıya maruz kalmasını önlemek için normalde istemci ile Ingress arasına ek bir proxy koyarsınız, ancak bu konumuzun dışıdır.



  1. GATEWAY: Bu bizim gerçek Ingress ve harici trafiğe maruz kalıyor. Harici olarak HTTPS isteğini kabul eder ve HTTP isteğini dahili hizmetlere iletir.

  2. DOC: Bu, şirket API dokümantasyonunu temsil eder ve doc.my-company.com domain alanında sunulur. Ingress doğrudan bu servisle iletişim kurar.

  3. PROXY: Bu, dahili Nginx proxy uygulamasıdır. Ingress doğrudan bu servisle iletişim kurar. api.my-company.com domaini ile ilgili tüm istekler bu servis tarafından işlenir ve alakalı Podlara iletirilirler.

  4. OAUTH: Bu, dahili OAuth hizmetimizdir. Her istek, kimlik doğrulama/yetkilendirme amacıyla diğer hizmetlere iletilmeden önce bu hizmete gönderilebilir.

  5. API: Bu rastgele bir dahili hizmettir.

API


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Dosyalar


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: api-deployment
  labels:
    app: api

spec:
  replicas: 1
  selector:
    matchLabels:
      app: api
  template:
    metadata:
      labels:
        app: api
    spec:
      containers:
        - name: golang
          image: you/api:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: api-service

spec:
  type: ClusterIP
  selector:
    app: api
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("api running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/api/v1/clients", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 clients list\n"))
	})
	rtr.HandleFunc("/api/v1/clients/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 single client\n"))
	})
	rtr.HandleFunc("/api/v1/customers/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 customer list\n"))
	})
	rtr.HandleFunc("/api/v1/customers/111/logs", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 single customer's logs\n"))
	})

	// V2
	rtr.HandleFunc("/api/v2/clients", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 clients list\n"))
	})
	rtr.HandleFunc("/api/v2/clients/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 single client\n"))
	})
	rtr.HandleFunc("/api/v2/customers/111", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 customer list\n"))
	})
	rtr.HandleFunc("/api/v2/customers/111/logs", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 single customer's logs\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/api/v1/clients
	curl --request GET http://localhost:3000/api/v1/clients/111
	curl --request GET http://localhost:3000/api/v1/customers/111
	curl --request GET http://localhost:3000/api/v1/customers/111/logs
	curl --request GET http://localhost:3000/api/v2/clients
	curl --request GET http://localhost:3000/api/v2/clients/111
	curl --request GET http://localhost:3000/api/v2/customers/111
	curl --request GET http://localhost:3000/api/v2/customers/111/logs

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/api:latest -f docker/Dockerfile .
	docker push you/api:latest
	docker rmi you/api:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

OAUTH


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Dosyalar


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: oauth-deployment
  labels:
    app: oauth

spec:
  replicas: 1
  selector:
    matchLabels:
      app: oauth
  template:
    metadata:
      labels:
        app: oauth
    spec:
      containers:
        - name: golang
          image: you/oauth:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: oauth-service

spec:
  type: ClusterIP
  selector:
    app: oauth
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("oauth running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/api/v1/authorization", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 authorization code\n"))
	})
	rtr.HandleFunc("/api/v1/oauth/token", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 oauth code\n"))
	})

	// V2
	rtr.HandleFunc("/api/v2/authorization", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 authorization code\n"))
	})
	rtr.HandleFunc("/api/v2/oauth/token", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 oauth code\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/api/v1/authorization
	curl --request GET http://localhost:3000/api/v1/oauth/token
	curl --request GET http://localhost:3000/api/v2/authorization
	curl --request GET http://localhost:3000/api/v2/oauth/token

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/oauth:latest -f docker/Dockerfile .
	docker push you/oauth:latest
	docker rmi you/oauth:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

DOC


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
├── docker
│   └── Dockerfile
└── main.go

Dosyalar


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: doc-deployment
  labels:
    app: doc

spec:
  replicas: 1
  selector:
    matchLabels:
      app: doc
  template:
    metadata:
      labels:
        app: doc
    spec:
      containers:
        - name: golang
          image: you/doc:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 3000

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: doc-service

spec:
  type: ClusterIP
  selector:
    app: doc
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 3000

Dockerfile

#
# STAGE 1: build
#
FROM golang:1.15-alpine3.12 as build

WORKDIR /source
COPY . .

RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o main main.go

#
# STAGE 2: run
#
FROM alpine:3.12

COPY --from=build /source/main /main

ENTRYPOINT ["./main"]

main.go

package main

import (
	"log"
	"net/http"
)

func main() {
	log.Println("doc running")

	rtr := http.NewServeMux()

	// V1
	rtr.HandleFunc("/v1", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v1 docs\n"))
	})

	// V2
	rtr.HandleFunc("/v2", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.RequestURI())
		_, _ = w.Write([]byte("get v2 docs\n"))
	})

	if err := http.ListenAndServe(":3000", rtr); err != nil {
		log.Fatalln(err)
	}
}

Makefile

## Build application binary.
.PHONY: build
build:
	go build -race -ldflags "-s -w" -o main main.go

## Build application binary and run it.
.PHONY: run
run: build
	./main

## Send a dummy request to the local server.
.PHONY: local-test
local-test:
	curl --request GET http://localhost:3000/v1
	curl --request GET http://localhost:3000/v2

## -------------------------------------------------------------

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/doc:latest -f docker/Dockerfile .
	docker push you/doc:latest
	docker rmi you/doc:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

PROXY


├── Makefile
├── deploy
│   └── k8s
│       ├── deployment.yaml
│       └── service.yaml
└── docker
    ├── Dockerfile
    └── default.conf

Dosyalar


deployment.yaml

apiVersion: apps/v1
kind: Deployment

metadata:
  name: proxy-deployment
  labels:
    app: proxy

spec:
  replicas: 1
  selector:
    matchLabels:
      app: proxy
  template:
    metadata:
      labels:
        app: proxy
    spec:
      containers:
        - name: nginx
          image: you/proxy:latest
          ports:
            - name: http
              protocol: TCP
              containerPort: 80

service.yaml

apiVersion: v1
kind: Service

metadata:
  name: proxy-service

spec:
  type: ClusterIP
  selector:
    app: proxy
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 80

Dockerfile

FROM nginx:1.19.5-alpine

COPY docker/default.conf /etc/nginx/conf.d/default.conf

default.conf

server {
    listen 80;
    # 443

    resolver kube-dns.kube-system.svc.cluster.local;

    ## oauth service
    location ~ ^/(?<version>v\d+)/authorization$ {
        proxy_pass http://oauth-service.default.svc.cluster.local:80/api/$version/authorization$is_args$args;
    }
    location ~ ^/(?<version>v\d+)/oauth/token$ {
        proxy_pass http://oauth-service.default.svc.cluster.local:80/api/$version/oauth/token$is_args$args;
    }
    ## end ################

    ## api service
    location ~ ^/(?<version>v\d+)/clients(?<suffix>.*) {
        proxy_pass http://api-service.default.svc.cluster.local:80/api/$version/clients$suffix$is_args$args;
    }
    location ~ ^/(?<version>v\d+)/customers(?<suffix>.*) {
        proxy_pass http://api-service.default.svc.cluster.local:80/api/$version/customers$suffix$is_args$args;
    }
    ## end ################

    ## all unmatched urls
    location / {
        default_type application/json;
        return 404 '{"operation": "client", "code": "route_not_found", "message": "The route cannot be found"}';
    }
    ## end ################
}

Makefile

## Build, tag and push application image to registry then clean up.
.PHONY: push
push:
	docker build -t you/proxy:latest -f docker/Dockerfile .
	docker push you/proxy:latest
	docker rmi you/proxy:latest
	docker system prune --volumes --force

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/deployment.yaml
	kubectl apply -f deploy/k8s/service.yaml

GATEWAY


├── Makefile
├── cert
│   └── cert.conf
└── deploy
    └── k8s
        └── ingress.yaml

Dosyalar


ingress.yaml

apiVersion: networking.k8s.io/v1beta1
kind: Ingress

metadata:
  name: gateway-ingress

spec:
  tls:
    - hosts:
        - doc.my-company.com
        - api.my-company.com
      secretName: gateway-tls-secret
  rules:
    - host: doc.my-company.com
      http:
        paths:
          - path: /
            backend:
              serviceName: doc-service
              servicePort: 80

    - host: api.my-company.com
      http:
        paths:
          - path: /
            backend:
              serviceName: proxy-service
              servicePort: 80

cert.conf

[ req ]
default_bits = 2048
prompt = no
default_md = sha256
req_extensions = req_ext
distinguished_name = dn

[ dn ]
C = UK
ST = London
L = London
O = Your Ltd.
OU = Information Technologies
emailAddress = email@email.com
CN = my-company.com

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = my-company.com
DNS.2 = www.my-company.com
DNS.3 = doc.my-company.com
DNS.4 = api.my-company.com

Makefile

## Deploy application to kubernetes cluster.
.PHONY: deploy
deploy:
	kubectl apply -f deploy/k8s/ingress.yaml

## Send a dummy request to the exposed service on kubernetes.
.PHONY: k8s-test-doc
k8s-test-doc:
	curl --insecure -I --request GET https://doc.my-company.com/v1
	curl --insecure -I --request GET https://doc.my-company.com/v2

## Send a dummy request to the exposed service on kubernetes.
.PHONY: k8s-test-api
k8s-test-api:
	curl --insecure -I --request GET https://api.my-company.com/v1/authorization
	curl --insecure -I --request GET https://api.my-company.com/v1/oauth/token
	curl --insecure -I --request GET https://api.my-company.com/v2/authorization
	curl --insecure -I --request GET https://api.my-company.com/v2/oauth/token
	curl --insecure -I --request GET https://api.my-company.com/v1/clients
	curl --insecure -I --request GET https://api.my-company.com/v1/clients/111
	curl --insecure -I --request GET https://api.my-company.com/v1/customers/111
	curl --insecure -I --request GET https://api.my-company.com/v1/customers/111/logs
	curl --insecure -I --request GET https://api.my-company.com/v2/clients
	curl --insecure -I --request GET https://api.my-company.com/v2/clients/111
	curl --insecure -I --request GET https://api.my-company.com/v2/customers/111
	curl --insecure -I --request GET https://api.my-company.com/v2/customers/111/logs

Kurulum


Öncelikle SSL sertifikaları, gizli Kubernetes girdileri oluşturmalı ve /etc/hosts dosyasına etki alanları eklemeliyiz. Bu, GATEWAY hizmeti için gereklidir.


Sertifikalar


$ openssl req -nodes -new -x509 -sha256 -days 1825 \
   -config cert/cert.conf -extensions 'req_ext' \
   -keyout ~/Desktop/k8s.key -out ~/Desktop/k8s.crt

Gizli girdiler


$ kubectl create secret tls gateway-tls-secret \
   --cert=~/Desktop/k8s.crt \
   --key=~/Desktop/k8s.key

Hosts


Bunu normalde GATEWAY hizmetini dağıttıktan sonra yaparsınız ama ben IP adresini bildiğim için burada gösteriyorum. IP adresini kubectl get ingress gateway-ingress -o yaml | grep ip komutunun çıktısına göre değiştirmelisiniz.


# /etc/hosts
192.168.99.100 doc.my-company.com
192.168.99.100 api.my-company.com

Dağıtım


Önce imajları aktarmanız ve ardından dağıtmanız gerekir.


api$ make push
oauth$ make push
doc$ make push
proxy$ make push

api$ make deploy
oauth$ make deploy
doc$ make deploy
proxy$ make deploy
gateway$ make deploy

Test


gateway$ make k8s-test-doc
curl --insecure --request GET https://doc.my-company.com/v1
get v1 docs
curl --insecure --request GET https://doc.my-company.com/v2
get v2 docs

gateway$ make k8s-test-api
curl --insecure --request GET https://api.my-company.com/v1/authorization
get v1 authorization code
curl --insecure --request GET https://api.my-company.com/v1/oauth/token
get v1 oauth code
curl --insecure --request GET https://api.my-company.com/v2/authorization
get v2 authorization code
curl --insecure --request GET https://api.my-company.com/v2/oauth/token
get v2 oauth code
curl --insecure --request GET https://api.my-company.com/v1/clients
get v1 clients list
curl --insecure --request GET https://api.my-company.com/v1/clients/111
get v1 single client
curl --insecure --request GET https://api.my-company.com/v1/customers/111
get v1 customer list
curl --insecure --request GET https://api.my-company.com/v1/customers/111/logs
get v1 single customers logs
curl --insecure --request GET https://api.my-company.com/v2/clients
get v2 clients list
curl --insecure --request GET https://api.my-company.com/v2/clients/111
get v2 single client
curl --insecure --request GET https://api.my-company.com/v2/customers/111
get v2 customer list
curl --insecure --request GET https://api.my-company.com/v2/customers/111/logs
get v2 single customers logs