微服务架构中的NGINX_apache

 

 

 

root@ubuntu:~/kubernetes-ingress/deployments/common# ls
crds  default-server-secret.yaml  ingress-class.yaml  nginx-config.yaml  ns-and-sa.yaml
root@ubuntu:~/kubernetes-ingress/deployments/common# kubectl apply -f ns-and-sa.yaml  
namespace/nginx-ingress created
serviceaccount/nginx-ingress created
root@ubuntu:~/kubernetes-ingress/deployments/common# cd ..
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f rbac/
ap-rbac.yaml  rbac.yaml     
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f rbac/rbac.yaml 
clusterrole.rbac.authorization.k8s.io/nginx-ingress created
clusterrolebinding.rbac.authorization.k8s.io/nginx-ingress created
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f common/default-server-secret.yaml 
secret/default-server-secret created
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f common/n
nginx-config.yaml  ns-and-sa.yaml     
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f common/nginx-config.yaml 
configmap/nginx-config created
root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f common/ingress-class.yaml 
error: unable to recognize "common/ingress-class.yaml": no matches for kind "IngressClass" in version "networking.k8s.io/v1"
root@ubuntu:~/kubernetes-ingress/deployments# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# cat common/ingress-class.yaml
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
  name: nginx
  # annotations:
  #   ingressclass.kubernetes.io/is-default-class: "true"
spec:
  controller: nginx.org/ingress-controller

 

改成 networking.k8s.io/v1beta

root@ubuntu:~/kubernetes-ingress/deployments# kubectl apply -f common/ingress-class.yaml 
ingressclass.networking.k8s.io/nginx created
root@ubuntu:~/kubernetes-ingress/deployments# cat   common/ingress-class.yaml
apiVersion: networking.k8s.io/v1beta1
kind: IngressClass
metadata:
  name: nginx
  # annotations:
  #   ingressclass.kubernetes.io/is-default-class: "true"
spec:
  controller: nginx.org/ingress-controller

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl create  -f deployment/nginx-
nginx-ingress.yaml       nginx-plus-ingress.yaml  
root@ubuntu:~/kubernetes-ingress/deployments# kubectl create  -f deployment/nginx-ingress.yaml 
deployment.apps/nginx-ingress created
root@ubuntu:~/kubernetes-ingress/deployments# kubectl get svc -A
NAMESPACE       NAME                                 TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE
default         apache-svc                           ClusterIP   10.111.63.105    <none>        80/TCP                       20d
default         coffee-svc                           ClusterIP   10.109.121.61    <none>        80/TCP                       8m12s
default         kubernetes                           ClusterIP   10.96.0.1        <none>        443/TCP                      55d
default         nginx-svc                            ClusterIP   10.103.182.145   <none>        80/TCP                       20d
default         tea-svc                              ClusterIP   10.105.105.208   <none>        80/TCP                       8m12s
default         web2                                 ClusterIP   10.99.87.66      <none>        8097/TCP                     20d
default         web3                                 ClusterIP   10.107.70.171    <none>        8097/TCP                     20d
ingress-nginx   ingress-nginx-controller             NodePort    10.105.207.185   <none>        80:31679/TCP,443:32432/TCP   15h
ingress-nginx   ingress-nginx-controller-admission   ClusterIP   10.101.64.30     <none>        443/TCP                      15h
kube-system     kube-dns                             ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP       55d
ns-calico1      nodeport-svc                         NodePort    10.109.58.6      <none>        3000:30090/TCP               33d

 

 

root@ubuntu:~/kubernetes-ingress# kubectl get svc -o wide  -A
NAMESPACE     NAME           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE   SELECTOR
default       apache-svc     ClusterIP   10.111.63.105    <none>        80/TCP                   20d   app=apache-app
default       coffee-svc     ClusterIP   10.109.121.61    <none>        80/TCP                   60m   app=coffee
default       kubernetes     ClusterIP   10.96.0.1        <none>        443/TCP                  55d   <none>
default       nginx-svc      ClusterIP   10.103.182.145   <none>        80/TCP                   20d   app=nginx-app
default       tea-svc        ClusterIP   10.105.105.208   <none>        80/TCP                   60m   app=tea
default       web2           ClusterIP   10.99.87.66      <none>        8097/TCP                 20d   run=web2
default       web3           ClusterIP   10.107.70.171    <none>        8097/TCP                 20d   run=web3
kube-system   kube-dns       ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP   55d   k8s-app=kube-dns
ns-calico1    nodeport-svc   NodePort    10.109.58.6      <none>        3000:30090/TCP           34d   app=calico1-nginx
root@ubuntu:~/kubernetes-ingress# kubectl get pod  -o wide  -n nginx-ingress
NAME                             READY   STATUS    RESTARTS   AGE     IP             NODE    NOMINATED NODE   READINESS GATES
nginx-ingress-85d86d7d6d-nwkcx   0/1     Running   0          5m15s   10.244.41.14   cloud   <none>           <none>
root@ubuntu:~/kubernetes-ingress# 

 

 

 

 


root@ubuntu:~/kubernetes-ingress# cd examples/c
complete-example/   custom-annotations/ customization/      custom-log-format/  custom-templates/   
root@ubuntu:~/kubernetes-ingress# cd examples/complete-example/
root@ubuntu:~/kubernetes-ingress/examples/complete-example# ls
cafe-ingress.yaml  cafe-secret.yaml  cafe.yaml  dashboard.png  README.md
root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl create -f cafe.yaml
deployment.apps/coffee created
service/coffee-svc created
deployment.apps/tea created
service/tea-svc created
root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl create -f cafe-secret.yaml
secret/cafe-secret created
root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl create -f cafe-ingress.yaml
error: unable to recognize "cafe-ingress.yaml": no matches for kind "Ingress" in version "networking.k8s.io/v1"
root@ubuntu:~/kubernetes-ingress/examples/complete-example# vi cafe-ingress.yaml

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl create -f cafe-ingress.yaml
error: error validating "cafe-ingress.yaml": error validating data: [ValidationError(Ingress.spec.rules[0].http.paths[0].backend): unknown field "service" in io.k8s.api.networking.v1beta1.IngressBackend, ValidationError(Ingress.spec.rules[0].http.paths[1].backend): unknown field "service" in io.k8s.api.networking.v1beta1.IngressBackend]; if you choose to ignore these errors, turn validation off with --validate=false
root@ubuntu:~/kubernetes-ingress/examples/complete-example# 

 

 

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl explain Ingress
KIND:     Ingress
VERSION:  extensions/v1beta1

DESCRIPTION:
     Ingress is a collection of rules that allow inbound connections to reach
     the endpoints defined by a backend. An Ingress can be configured to give
     services externally-reachable urls, load balance traffic, terminate SSL,
     offer name based virtual hosting etc. DEPRECATED - This group version of
     Ingress is deprecated by networking.k8s.io/v1beta1 Ingress. See the release
     notes for more information.

FIELDS:
   apiVersion   <string>
     APIVersion defines the versioned schema of this representation of an
     object. Servers should convert recognized schemas to the latest internal
     value, and may reject unrecognized values. More info:
     https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources

   kind <string>
     Kind is a string value representing the REST resource this object
     represents. Servers may infer this from the endpoint the client submits
     requests to. Cannot be updated. In CamelCase. More info:
     https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds

   metadata     <Object>
     Standard object's metadata. More info:
     https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata

   spec <Object>
     Spec is the desired state of the Ingress. More info:
     https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status

   status       <Object>
     Status is the current state of the Ingress. More info:
     https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status

 

 

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example# cat cafe-ingress.yaml 
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: cafe-ingress
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - cafe.example.com
    secretName: cafe-secret
  rules:
  - host: cafe.example.com
    http:
      paths:
      - path: /tea
        pathType: Prefix
        backend:
          serviceName: tea-svc
          servicePort: 80
      - path: /coffee
        pathType: Prefix
        backend:
          serviceName: coffee-svc
          servicePort: 80

 

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl create -f cafe-ingress.yaml
ingress.networking.k8s.io/cafe-ingress created

 

 namesapce 是nginx-ingress

^Croot@ubuntu:~/kubernetes-ingress# kubectl port-forward  nginx-ingress-85d86d7d6d-nwkcx 9999:443 -n nginx-ingress
Forwarding from 127.0.0.1:9999 -> 443
Forwarding from [::1]:9999 -> 443
Handling connection for 9999
Handling connection for 9999
E0826 12:02:46.764087 2648505 portforward.go:400] an error occurred forwarding 9999 -> 443: error forwarding port 443 to pod 240494b7e3176e2d9db2cb5f8266fd16690542d2f4c05a7c95618ebf1547f48b, uid : exit status 1: 2021/08/26 12:02:46 socat[2270844] E write(5, 0xaaaaf4faeec0, 5): Broken pipe
Handling connection for 9999

 

 

微服务架构中的NGINX_apache_02微服务架构中的NGINX_ubuntu_03

 

 微服务架构中的NGINX_nginx_04

 怎么部署两套ngnix controller

  微服务架构中的NGINX_apache_05

 

 微服务架构中的NGINX_ide_06

 

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example#  kubectl get ingressclass
NAME    CONTROLLER                     PARAMETERS   AGE
nginx   nginx.org/ingress-controller   <none>       3h35m
root@ubuntu:~/kubernetes-ingress/examples/complete-example# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments/common# cat ingress-class.yaml 
apiVersion: networking.k8s.io/v1beta1
kind: IngressClass
metadata:
  name: nginx
  # annotations:
  #   ingressclass.kubernetes.io/is-default-class: "true"
spec:
  controller: nginx.org/ingress-controller

微服务架构中的NGINX_apache_07

 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl get ing  -o wide  -A
NAMESPACE   NAME              CLASS    HOSTS                                    ADDRESS          PORTS     AGE
default     cafe-ingress      nginx    cafe.example.com                                          80, 443   162m
default     example-ingress   <none>   ubuntu.com                               10.105.207.185   80        20d
default     micro-ingress     <none>   nginx.mydomain.com,apache.mydomain.com   10.105.207.185   80        20d
default     web-ingress       <none>   web.mydomain.com                         10.105.207.185   80        20d
default     web-ingress-lb    <none>   web3.mydomain.com,web2.mydomain.com      10.105.207.185   80        20d
root@ubuntu:~/kubernetes-ingress/deployments# 

更改名称

root@ubuntu:~/kubernetes-ingress/deployments# kubectl create   -f  common/ingress-class.yaml
ingressclass.networking.k8s.io/nginx-org created
root@ubuntu:~/kubernetes-ingress/deployments#  kubectl get ingressclass
NAME        CONTROLLER                     PARAMETERS   AGE
nginx-org   nginx.org/ingress-controller   <none>       12s
root@ubuntu:~/kubernetes-ingress/deployments# cat  common/ingress-class.yaml
apiVersion: networking.k8s.io/v1beta1
kind: IngressClass
metadata:
  name: nginx-org
  # annotations:
  #   ingressclass.kubernetes.io/is-default-class: "true"
spec:
  controller: nginx.org/ingress-controller

更改

 cafe-ingress 
root@ubuntu:~/kubernetes-ingress/deployments# kubectl get ing  -o wide  -A
NAMESPACE   NAME              CLASS       HOSTS                                    ADDRESS          PORTS     AGE
default     cafe-ingress      nginx-org   cafe.example.com                                          80, 443   167m
default     example-ingress   <none>      ubuntu.com                               10.105.207.185   80        20d
default     micro-ingress     <none>      nginx.mydomain.com,apache.mydomain.com   10.105.207.185   80        20d
default     web-ingress       <none>      web.mydomain.com                         10.105.207.185   80        20d
default     web-ingress-lb    <none>      web3.mydomain.com,web2.mydomain.com      10.105.207.185   80        20d
root@ubuntu:~/kubernetes-ingress/deployments# kubectl edit ing cafe-ingress

 

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl exec -it nginx-ingress-85d86d7d6d-nwkcx -n nginx-ingress -- /bin/bash
nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ cd /etc/n
nginx/         nsswitch.conf  
nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ cd /etc/nginx/conf
conf.d/              config-version.conf  
nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ cd /etc/nginx/conf.d/
nginx@nginx-ingress-85d86d7d6d-nwkcx:/etc/nginx/conf.d$ ls
nginx@nginx-ingress-85d86d7d6d-nwkcx:/etc/nginx/conf.d$ ls
nginx@nginx-ingress-85d86d7d6d-nwkcx:/etc/nginx/conf.d$ 

竟然什么都没有

 

nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ cat /etc/nginx/nginx.conf  | grep include
    include       /etc/nginx/mime.types;
    include /etc/nginx/config-version.conf;
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/stream-conf.d/*.conf;
nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ 

微服务架构中的NGINX_ubuntu_08

 

 

 

微服务架构中的NGINX_nginx_09

 

 

 

微服务架构中的NGINX_f5_10

 

 

root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl get pod,svc,secret,ingress -n nginx-ingress -o wide
NAME                                 READY   STATUS    RESTARTS   AGE     IP             NODE    NOMINATED NODE   READINESS GATES
pod/nginx-ingress-85d86d7d6d-nwkcx   0/1     Running   0          3h16m   10.244.41.14   cloud   <none>           <none>

NAME                               TYPE                                  DATA   AGE
secret/default-server-secret       kubernetes.io/tls                     2      4h23m
secret/default-token-9226g         kubernetes.io/service-account-token   3      4h23m
secret/nginx-ingress-token-xnkwn   kubernetes.io/service-account-token   3      4h23m
root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl get pod | grep coffe
coffee-5f56ff9788-plfcq         1/1     Running   0          3h3m
coffee-5f56ff9788-zs2f7         1/1     Running   0          3h3m
root@ubuntu:~/kubernetes-ingress/examples/complete-example# kubectl get pod | grep tea
tea-69c99ff568-hdcbl            1/1     Running   0          3h3m
tea-69c99ff568-p59d6            1/1     Running   0          3h3m
tea-69c99ff568-tm9q6            1/1     Running   0          3h3m
root@ubuntu:~/kubernetes-ingress/examples/complete-example# 

 

kubectl logs  nginx-ingress-85d86d7d6d-nwkcx -n nginx-ingress 

erverroutes.k8s.nginx.org)
E0826 07:03:43.188312       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.Policy: failed to list *v1.Policy: the server could not find the requested resource (get policies.k8s.nginx.org)
E0826 07:03:48.360767       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1alpha1.TransportServer: failed to list *v1alpha1.TransportServer: the server could not find the requested resource (get transportservers.k8s.nginx.org)
E0826 07:04:14.214771       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.Policy: failed to list *v1.Policy: the server could not find the requested resource (get policies.k8s.nginx.org)
E0826 07:04:17.201091       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.VirtualServer: failed to list *v1.VirtualServer: the server could not find the requested resource (get virtualservers.k8s.nginx.org)
E0826 07:04:33.115149       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.VirtualServerRoute: failed to list *v1.VirtualServerRoute: the server could not find the requested resource (get virtualserverroutes.k8s.nginx.org)
E0826 07:04:41.914552       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1alpha1.TransportServer: failed to list *v1alpha1.TransportServer: the server could not find the requested resource (get transportservers.k8s.nginx.org)
E0826 07:04:54.294521       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.Policy: failed to list *v1.Policy: the server could not find the requested resource (get policies.k8s.nginx.org)
E0826 07:04:54.660443       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.VirtualServer: failed to list *v1.VirtualServer: the server could not find the requested resource (get virtualservers.k8s.nginx.org)
E0826 07:05:25.632495       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1alpha1.TransportServer: failed to list *v1alpha1.TransportServer: the server could not find the requested resource (get transportservers.k8s.nginx.org)
E0826 07:05:32.079676       1 reflector.go:138] pkg/mod/k8s.io/client-go@v0.21.2/tools/cache/reflector.go:167: Failed to watch *v1.VirtualServerRoute: failed to list *v1.VirtualServerRoute: the server could not find the requested resource (get virtual

 

執行

 
kubectl apply -f crds/k8s.nginx.org_virtualservers.yaml
kubectl apply -f crds/k8s.nginx.org_virtualserverroutes.yaml
kubectl apply -f crds/k8s.nginx.org_transportservers.yaml
kubectl apply -f crds/k8s.nginx.org_policies.yaml
kubectl apply -f crds/k8s.nginx.org_globalconfigurations.yaml

規則生成了,但是沒有caffe的

root@ubuntu:~/kubernetes-ingress/deployments# kubectl exec -it nginx-ingress-85d86d7d6d-nwkcx -n nginx-ingress -- ls /etc/nginx/conf.d/
default-example-ingress.conf default-web-ingress-lb.conf
default-micro-ingress.conf default-web-ingress.conf
root@ubuntu:~/kubernetes-ingress/deployments#

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl exec -it nginx-ingress-85d86d7d6d-nwkcx -n nginx-ingress -- cat  /etc/nginx/conf.d/default-micro-ingress.conf
# configuration for default/micro-ingress

upstream default-micro-ingress-apache.mydomain.com-apache-svc-80 {
        zone default-micro-ingress-apache.mydomain.com-apache-svc-80 256k;
        random two least_conn;

        server 10.244.243.197:80 max_fails=1 fail_timeout=10s max_conns=0;
        server 10.244.41.61:80 max_fails=1 fail_timeout=10s max_conns=0;

}
upstream default-micro-ingress-nginx.mydomain.com-nginx-svc-80 {
        zone default-micro-ingress-nginx.mydomain.com-nginx-svc-80 256k;
        random two least_conn;

        server 10.244.243.195:80 max_fails=1 fail_timeout=10s max_conns=0;
        server 10.244.41.58:80 max_fails=1 fail_timeout=10s max_conns=0;

}


server {


        listen 80;








        server_tokens on;

        server_name nginx.mydomain.com;

        set $resource_type "ingress";
        set $resource_name "micro-ingress";
        set $resource_namespace "default";







        location / {
                set $service "nginx-svc"; 


                proxy_http_version 1.1;


                proxy_connect_timeout 60s;
                proxy_read_timeout 60s;
                proxy_send_timeout 60s;
                client_max_body_size 1m;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Host $host;
                proxy_set_header X-Forwarded-Port $server_port;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_buffering on;

                proxy_pass http://default-micro-ingress-nginx.mydomain.com-nginx-svc-80;


        }


}
server {


        listen 80;








        server_tokens on;

        server_name apache.mydomain.com;

        set $resource_type "ingress";
        set $resource_name "micro-ingress";
        set $resource_namespace "default";







        location / {
                set $service "apache-svc"; 


                proxy_http_version 1.1;


                proxy_connect_timeout 60s;
                proxy_read_timeout 60s;
                proxy_send_timeout 60s;
                client_max_body_size 1m;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Host $host;
                proxy_set_header X-Forwarded-Port $server_port;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_buffering on;

                proxy_pass http://default-micro-ingress-apache.mydomain.com-apache-svc-80;


        }


}

 

 

nginx@nginx-ingress-85d86d7d6d-nwkcx:/$ ./nginx-ingress  -h
Usage of ./nginx-ingress:
  -alsologtostderr
        log to standard error as well as files
  -default-server-tls-secret string
        A Secret with a TLS certificate and key for TLS termination of the default server. Format: <namespace>/<name>.
                If not set, than the certificate and key in the file "/etc/nginx/secrets/default" are used.
                If "/etc/nginx/secrets/default" doesn't exist, the Ingress Controller will configure NGINX to reject TLS connections to the default server.
                If a secret is set, but the Ingress controller is not able to fetch it from Kubernetes API or it is not set and the Ingress Controller
                fails to read the file "/etc/nginx/secrets/default", the Ingress controller will fail to start.
  -enable-app-protect
        Enable support for NGINX App Protect. Requires -nginx-plus.
  -enable-custom-resources
        Enable custom resources (default true)
  -enable-internal-routes
        Enable support for internal routes with NGINX Service Mesh. Requires -spire-agent-address and -nginx-plus. Is for use with NGINX Service Mesh only.
  -enable-latency-metrics
        Enable collection of latency metrics for upstreams. Requires -enable-prometheus-metrics
  -enable-leader-election
        Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress, VirtualServer and VirtualServerRoute resources -- only one replica will report status (default true). See -report-ingress-status flag. (default true)
  -enable-preview-policies
        Enable preview policies
  -enable-prometheus-metrics
        Enable exposing NGINX or NGINX Plus metrics in the Prometheus format
  -enable-snippets
        Enable custom NGINX configuration snippets in VirtualServer, VirtualServerRoute and TransportServer resources.
  -enable-tls-passthrough
        Enable TLS Passthrough on port 443. Requires -enable-custom-resources
  -external-service string
        Specifies the name of the service with the type LoadBalancer through which the Ingress controller pods are exposed externally.
                The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. For Ingress resources only: Requires -report-ingress-status.
  -global-configuration string
        The namespace/name of the GlobalConfiguration resource for global configuration of the Ingress Controller. Requires -enable-custom-resources. Format: <namespace>/<name>
  -health-status
        Add a location based on the value of health-status-uri to the default server. The location responds with the 200 status code for any request.
                Useful for external health-checking of the Ingress controller
  -health-status-uri string
        Sets the URI of health status location in the default server. Requires -health-status (default "/nginx-health")
  -ingress-class string
        A class of the Ingress controller.

更改啓動參數

        args:
          - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
          - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
          - -ingress-class=nginx-org

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl exec -it nginx-ingress-6f7d9bdb87-twhxg  -n nginx-ingress -- ls /etc/nginx/conf.d/
default-cafe-ingress.conf
root@ubuntu:~/kubernetes-ingress/deployments# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl get svc -A
NAMESPACE     NAME           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
default       apache-svc     ClusterIP   10.111.63.105    <none>        80/TCP                   20d
default       coffee-svc     ClusterIP   10.101.87.73     <none>        80/TCP                   3h46m
default       kubernetes     ClusterIP   10.96.0.1        <none>        443/TCP                  55d
default       nginx-svc      ClusterIP   10.103.182.145   <none>        80/TCP                   20d
default       tea-svc        ClusterIP   10.103.138.254   <none>        80/TCP                   3h46m
default       web2           ClusterIP   10.99.87.66      <none>        8097/TCP                 20d
default       web3           ClusterIP   10.107.70.171    <none>        8097/TCP                 20d
kube-system   kube-dns       ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP   55d
ns-calico1    nodeport-svc   NodePort    10.109.58.6      <none>        3000:30090/TCP           34d
root@ubuntu:~/kubernetes-ingress/deployments# kubectl get pod -n nginx-ingress -o wide
NAME                             READY   STATUS    RESTARTS   AGE     IP             NODE    NOMINATED NODE   READINESS GATES
nginx-ingress-6f7d9bdb87-twhxg   1/1     Running   0          8m29s   10.244.41.21   cloud   <none>           <none>
root@ubuntu:~/kubernetes-ingress/deployments#  curl  https://10.244.41.21:443/coffee --insecure
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx/1.21.0</center>
</body>
</html>
root@ubuntu:~/kubernetes-ingress/deployments#  curl  https://10.244.41.21:443/tea --insecure
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx/1.21.0</center>
</body>
</html>
root@ubuntu:~/kubernetes-ingress/deployments#  curl  https://10.244.41.21:443--insecure
curl: (3) Port number ended with '-'
root@ubuntu:~/kubernetes-ingress/deployments#  curl  https://10.244.41.21:443 --insecure
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>nginx/1.21.0</center>
</body>
</html>
root@ubuntu:~/kubernetes-ingress/deployments# curl --resolve cafe.example.com:443:10.244.41.21 https://cafe.example.com:443/tea --insecure
Server address: 10.244.41.15:8080
Server name: tea-69c99ff568-p59d6
Date: 26/Aug/2021:07:51:22 +0000
URI: /tea
Request ID: 68a8e864e4cc43b7f3896da25312e68c

 

 

root@ubuntu:~/kubernetes-ingress/deployments# curl --resolve cafe.example.com:443:10.244.41.21 https://cafe.example.com:443/coffee --insecure
Server address: 10.244.243.252:8080
Server name: coffee-5f56ff9788-plfcq
Date: 26/Aug/2021:09:19:50 +0000
URI: /coffee
Request ID: 60e7be5ff2fd71f2b1ec8452a8705242

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl get pod -A -o wide | grep  -E 'coffee|tea'
default          coffee-5f56ff9788-plfcq                    1/1     Running            0          5h20m   10.244.243.252   ubuntu   <none>           <none>
default          coffee-5f56ff9788-zs2f7                    1/1     Running            0          5h20m   10.244.41.19     cloud    <none>           <none>
default          tea-69c99ff568-hdcbl                       1/1     Running            0          5h20m   10.244.41.17     cloud    <none>           <none>
default          tea-69c99ff568-p59d6                       1/1     Running            0          5h20m   10.244.41.15     cloud    <none>           <none>
default          tea-69c99ff568-tm9q6                       1/1     Running            0          5h20m   10.244.41.18     cloud    <none>           <none>

 

root@ubuntu:~/kubernetes-ingress/deployments# curl --resolve cafe.example.com:443:10.244.41.21 
curl: no URL specified!
curl: try 'curl --help' or 'curl --manual' for more information
root@ubuntu:~/kubernetes-ingress/deployments# curl  https://cafe.example.com:443/coffee --insecure
curl: (6) Could not resolve host: cafe.example.com
root@ubuntu:~/kubernetes-ingress/deployments# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# curl --resolve cafe.example.com:443:10.244.41.21 https://cafe.example.com:443/coffee --insecure -v
* Added cafe.example.com:443:10.244.41.21 to DNS cache
* Hostname cafe.example.com was found in DNS cache
*   Trying 10.244.41.21...
* TCP_NODELAY set
* Connected to cafe.example.com (10.244.41.21) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
*   CAfile: /etc/ssl/certs/ca-certificates.crt
  CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES256-GCM-SHA384
* ALPN, server accepted to use http/1.1
* Server certificate:
*  subject: C=US; ST=CA; O=Internet Widgits Pty Ltd; CN=cafe.example.com
*  start date: Sep 12 16:15:35 2018 GMT
*  expire date: Sep 11 16:15:35 2023 GMT
*  issuer: C=US; ST=CA; O=Internet Widgits Pty Ltd; CN=cafe.example.com  
*  SSL certificate verify result: self signed certificate (18), continuing anyway.
> GET /coffee HTTP/1.1
> Host: cafe.example.com
> User-Agent: curl/7.58.0
> Accept: */*
> 
< HTTP/1.1 200 OK
< Server: nginx/1.21.0
< Date: Thu, 26 Aug 2021 09:24:20 GMT
< Content-Type: text/plain
< Content-Length: 164
< Connection: keep-alive
< Expires: Thu, 26 Aug 2021 09:24:19 GMT
< Cache-Control: no-cache
< 
Server address: 10.244.243.252:8080
Server name: coffee-5f56ff9788-plfcq
Date: 26/Aug/2021:09:24:20 +0000
URI: /coffee
Request ID: e1d17cb44eb89b94e8efe178344f5dd8
* Connection #0 to host cafe.example.com left intact
root@ubuntu:~/kubernetes-ingress/deployments# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# kubectl get pod -A -o wide | grep  ingress
nginx-ingress    nginx-ingress-6f7d9bdb87-twhxg             1/1     Running            0          109m    10.244.41.21     cloud    <none>           <none>
root@ubuntu:~/kubernetes-ingress/deployments# 

 

 

root@ubuntu:~/kubernetes-ingress/deployments# curl -H "Host: cafe.example.com" https://10.244.41.21:443/coffee --insecure -v
*   Trying 10.244.41.21...
* TCP_NODELAY set
* Connected to 10.244.41.21 (10.244.41.21) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
*   CAfile: /etc/ssl/certs/ca-certificates.crt
  CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES256-GCM-SHA384
* ALPN, server accepted to use http/1.1
* Server certificate:
*  subject: CN=NGINXIngressController
*  start date: Sep 12 18:03:35 2018 GMT
*  expire date: Sep 11 18:03:35 2023 GMT
*  issuer: CN=NGINXIngressController
*  SSL certificate verify result: self signed certificate (18), continuing anyway.
> GET /coffee HTTP/1.1
> Host: cafe.example.com
> User-Agent: curl/7.58.0
> Accept: */*
> 
< HTTP/1.1 200 OK
< Server: nginx/1.21.0
< Date: Thu, 26 Aug 2021 09:26:59 GMT
< Content-Type: text/plain
< Content-Length: 164
< Connection: keep-alive
< Expires: Thu, 26 Aug 2021 09:26:58 GMT
< Cache-Control: no-cache
< 
Server address: 10.244.243.252:8080
Server name: coffee-5f56ff9788-plfcq
Date: 26/Aug/2021:09:26:59 +0000
URI: /coffee
Request ID: f7b3794d0f1a25443bb4f8d6526dadfd
* Connection #0 to host 10.244.41.21 left intact

 

 

http請求先匹配域名cafe.example.com,再匹配uri/coffee

root@ubuntu:~/kubernetes-ingress/deployments# kubectl exec -it nginx-ingress-6f7d9bdb87-twhxg  -n nginx-ingress -- cat  /etc/nginx/conf.d/default-cafe-ingress.conf
# configuration for default/cafe-ingress

upstream default-cafe-ingress-cafe.example.com-coffee-svc-80 {
        zone default-cafe-ingress-cafe.example.com-coffee-svc-80 256k;
        random two least_conn;

        server 10.244.243.252:8080 max_fails=1 fail_timeout=10s max_conns=0;
        server 10.244.41.19:8080 max_fails=1 fail_timeout=10s max_conns=0;

}
upstream default-cafe-ingress-cafe.example.com-tea-svc-80 {
        zone default-cafe-ingress-cafe.example.com-tea-svc-80 256k;
        random two least_conn;

        server 10.244.41.15:8080 max_fails=1 fail_timeout=10s max_conns=0;
        server 10.244.41.17:8080 max_fails=1 fail_timeout=10s max_conns=0;
        server 10.244.41.18:8080 max_fails=1 fail_timeout=10s max_conns=0;

}


server {


        listen 80;




        listen 443 ssl;


        ssl_certificate /etc/nginx/secrets/default-cafe-secret;
        ssl_certificate_key /etc/nginx/secrets/default-cafe-secret;







        server_tokens on;

        server_name cafe.example.com;

        set $resource_type "ingress";
        set $resource_name "cafe-ingress";
        set $resource_namespace "default";






        if ($scheme = http) {
                return 301 https://$host:443$request_uri;
        }



        location /tea {
                set $service "tea-svc"; 


                proxy_http_version 1.1;


                proxy_connect_timeout 60s;
                proxy_read_timeout 60s;
                proxy_send_timeout 60s;
                client_max_body_size 1m;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Host $host;
                proxy_set_header X-Forwarded-Port $server_port;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_buffering on;

                proxy_pass http://default-cafe-ingress-cafe.example.com-tea-svc-80;


        }
        location /coffee {
                set $service "coffee-svc"; 


                proxy_http_version 1.1;


                proxy_connect_timeout 60s;
                proxy_read_timeout 60s;
                proxy_send_timeout 60s;
                client_max_body_size 1m;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Host $host;
                proxy_set_header X-Forwarded-Port $server_port;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_buffering on;

                proxy_pass http://default-cafe-ingress-cafe.example.com-coffee-svc-80;


        }


}

 

 微服务架构中的NGINX_f5_11

 

 confgimap对应全局配置(绿色)

ingress(包括annotaion):对应局部配置

 confgimap和ingress(都和golang template有关

 

apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: cafe-ingress
spec:
  ingressClassName: nginx-org
  tls:
  - hosts:
    - cafe.example.com
    secretName: cafe-secret
  rules:
  - host: cafe.example.com
    http:
      paths:
      - path: /tea
        pathType: Prefix
        backend:
          serviceName: tea-svc
          servicePort: 80
      - path: /coffee
        pathType: Prefix
        backend:
          serviceName: coffee-svc
          servicePort: 80

 微服务架构中的NGINX_ubuntu_12

 

 

k8s官网

微服务架构中的NGINX_ubuntu_13

 

 微服务架构中的NGINX_ide_14

 

 

deploy/static/provider/baremetal/deploy.yaml 

微服务架构中的NGINX_ide_15

 

 

 

 

 

demo

 https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengsettingupingresscontroller.htm

https://www.digitalocean.com/community/tutorials/how-to-set-up-an-nginx-ingress-on-digitalocean-kubernetes-using-helm

 

在K8S集群中使用NGINX Ingress V1.7

Multiple Ingress controllers