MTL Istio Egress для внешних сервисов

В настоящее время я (и безуспешно) пытаюсь настроить MTL через istio-egressgateway для доступа к внешней службе кластера K8s. Я следую инструкциям, указанным на istio docs, но ничего не работает должным образом, и я не могу понять, в чем я ошибаюсь.

Окружающая среда

• 3 ВМ под VMWare ESXi (1 мастер, 2 узла)
• ОС: CentOS 7 с ядром 5.2.10-1.el7 от elrepo
• Сеть k8s: cilium -1.6.4
• Certificates : cert-manager-0.12 nowebhook
  • ClusterIssuer CA cert = Imported from private PKI
• Внешний балансировщик нагрузки K8s: metallb-0.8.3

kubectl и istio используются в кластере K8s

$ kubectl version
Client Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.3", GitCommit:"b3cbbae08ec52a7fc73d334838e18d17e8512749", GitTreeState:"clean", BuildDate:"2019-11-13T11:23:11Z", GoVersion:"go1.12.12", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.4", GitCommit:"224be7bdce5a9dd0c2fd0d46b83865648e2fe0ba", GitTreeState:"clean", BuildDate:"2019-12-11T12:37:43Z", GoVersion:"go1.12.12", Compiler:"gc", Platform:"linux/amd64"}

$ istioctl version
client version: 1.2.10
citadel version: 1.2.10
egressgateway version: 1.2.10
galley version: 1.2.10
ingressgateway version: 1.2.10
pilot version: 1.2.10
policy version: 1.2.10
sidecar-injector version: 1.2.10
telemetry version: 1.2.10

istio-egressgateway был повторно развернут, как указано в istio docs

Это примененный .yaml почти копипаст из примера, с небольшими изменениями, указывающими на внешнюю службу через ее IP-адрес.

apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: nginx
  namespace: tools-istio
spec:
  hosts:
  - <my.hostname>
  ports:
  - number: 80
    name: http
    protocol: HTTP
  - number: 443
    name: https
    protocol: HTTPS
  resolution: DNS
  endpoints:
  - address: <private_ip_address>
    ports:
      https: 443
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: nginx
  namespace: tools-istio
spec:
  hosts:
  - <my.hostname>
  tls:
  - match:
    - port: 443
      sni_hosts:
      - <my.hostname>
    route:
    - destination:
        host: <my.hostname>
        port:
          number: 443
      weight: 100
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: istio-egressgateway
  namespace: tools-istio
spec:
  selector:
    istio: egressgateway
  servers:
  - port:
      number: 443
      name: https
      protocol: HTTPS
    hosts:
    - <my.hostname>
    tls:
      mode: MUTUAL
      serverCertificate: /etc/certs/cert-chain.pem
      privateKey: /etc/certs/key.pem
      caCertificates: /etc/certs/root-cert.pem
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: egressgateway-for-nginx
  namespace: tools-istio
spec:
  host: istio-egressgateway.istio-system.svc.cluster.local
  subsets:
  - name: nginx
    trafficPolicy:
      loadBalancer:
        simple: ROUND_ROBIN
      portLevelSettings:
      - port:
          number: 443
        tls:
          mode: ISTIO_MUTUAL
          sni: <my.hostname>
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: direct-nginx-through-egress-gateway
  namespace: tools-istio
spec:
  hosts:
  - <my.hostname>
  gateways:
  - istio-egressgateway
  - mesh
  http:
  - match:
    - gateways:
      - mesh
      port: 80
    route:
    - destination:
        host: istio-egressgateway.istio-system.svc.cluster.local
        subset: nginx
        port:
          number: 443
      weight: 100
  - match:
    - gateways:
      - istio-egressgateway
      port: 443
    route:
    - destination:
        host: <my.hostname>
        port:
          number: 443
      weight: 100
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: originate-mtls-for-nginx
  namespace: tools-istio
spec:
  host: <my.hostname>
  trafficPolicy:
    loadBalancer:
      simple: ROUND_ROBIN
    portLevelSettings:
    - port:
        number: 443
      tls:
        mode: ISTIO_MUTUAL
        sni: <my.hostname>
        # I've also tried with
        # mode: MUTUAL
        # clientCertificate: /etc/nginx-external-tls/tls.crt
        # privateKey: /etc/nginx-external-tls/tls.key
        # caCertificates: /etc/nginx-external-tls/ca.crt

При выполнении запроса к внешней службе через curl мы должны получить веб-код HTML, но вместо этого он пытается выполнить запрос по протоколу HTTP, а не HTTPS, почему ??. istio-proxy из модуля отладчика должен переводить HTTP в HTTPS с использованием сертификатов Citadel.

 $ kubectl -n tools-istio exec -it debugger-5844b6d674-4fvt9 -c debugger -- curl http://<my.hostname> -v
*   Trying <my.hostname>:80...
* TCP_NODELAY set
* Connected to <my.hostname> (<private_ip_address>) port 80 (#0)
> GET / HTTP/1.1
> Host: <my.hostname>
> User-Agent: curl/7.66.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 503 Service Unavailable
< content-length: 91
< content-type: text/plain
< date: Wed, 08 Jan 2020 09:29:09 GMT
< server: envoy
<
* Connection #0 to host <my.hostname> intact
upstream connect error or disconnect/reset before headers. reset reason: connection failure

При проверке proxy-status я получаю следующее

$ istioctl proxy-status
NAME                                                   CDS                            LDS        EDS               RDS          PILOT                            VERSION
debugger-5844b6d674-4fvt9.tools-istio                  STALE (Never Acknowledged)     SYNCED     SYNCED (51%)      SYNCED       istio-pilot-5f45f6768c-fmvr7     1.2.10
istio-egressgateway-5ff889c5fd-jtz55.istio-system      SYNCED                         SYNCED     SYNCED (100%)     SYNCED       istio-pilot-5f45f6768c-fmvr7     1.2.10
istio-ingressgateway-779cfdd879-bb7w7.istio-system     STALE (Never Acknowledged)     SYNCED     SYNCED (96%)      NOT SENT     istio-pilot-5f45f6768c-fmvr7     1.2.10

Что, согласно официальным istio docs, может означать либо неправильную конфигурацию в .yaml, либо istio ошибка

При проверке конечной точки все выглядит нормально

$ istioctl proxy-config endpoint debugger-5844b6d674-4fvt9.tools-istio | grep <my.hostname>
<private_ip_address>:80         HEALTHY     outbound|80||<my.hostname>
<private_ip_address>:443        HEALTHY     outbound|443||<my.hostname>

В противном случае журналы модуля отладчика istio-proxy отображают ошибки ниже. Но внешний сертификат находится не на уровне модуля, а в перекомпилированном istio-egressgateway (опять же, ничего забавного, я только что следил за документами istio), так почему istio-proxy ищет внешние сертификаты ??

[2020-01-08 07:56:04.827][17][debug][init] [external/envoy/source/common/init/watcher_impl.cc:27] init manager Cluster outbound|443||afv.test.recouv destroyed
[2020-01-08 07:56:04.827][17][warning][config] [external/envoy/source/common/config/grpc_mux_subscription_impl.cc:73] gRPC config for type.googleapis.com/envoy.api.v2.Cluster rejected: Error adding/updating cluster(s) outbound|443||<my.hostname>: Invalid path: /etc/nginx-external-tls/tls.crt

$ kubectl -n istio-system logs istio-pilot-5f45f6768c-fmvr7 -c discovery | grep <my.hostname>

2020-01-08T08:48:54.536073Z     warn    ads     ADS:CDS: ACK ERROR 127.0.0.1:60012 sidecar~10.27.187.219~debugger-5844b6d674-4fvt9.tools-istio~tools-istio.svc.cluster.local-15433 (debugger-5844b6d674-4fvt9.tools-istio) version_info:"2020-01-08T07:50:00Z/110" node:<id:"sidecar~10.27.187.219~debugger-5844b6d674-4fvt9.tools-istio~tools-istio.svc.cluster.local" cluster:"debugger.tools-istio" metadata:<fields:<key:"CONFIG_NAMESPACE" value:<string_value:"tools-istio" > > fields:<key:"INCLUDE_INBOUND_PORTS" value:<string_value:"8080" > > fields:<key:"INTERCEPTION_MODE" value:<string_value:"REDIRECT" > > fields:<key:"ISTIO_META_INSTANCE_IPS" value:<string_value:"10.27.187.219,10.27.187.219,fe80::a07f:d0ff:feec:5e71" > > fields:<key:"ISTIO_PROXY_SHA" value:<string_value:"istio-proxy:f9c0feb10cc42277e97dd080a2e045f62d1739a9" > > fields:<key:"ISTIO_PROXY_VERSION" value:<string_value:"1.1.3" > > fields:<key:"ISTIO_VERSION" value:<string_value:"1.2.10" > > fields:<key:"POD_NAME" value:<string_value:"debugger-5844b6d674-4fvt9" > > fields:<key:"app" value:<string_value:"debugger" > > fields:<key:"istio" value:<string_value:"sidecar" > > fields:<key:"pod-template-hash" value:<string_value:"5844b6d674" > > fields:<key:"prometheus.io/path" value:<string_value:"/metrics" > > fields:<key:"prometheus.io/port" value:<string_value:"8080" > > fields:<key:"prometheus.io/scrape" value:<string_value:"true" > > fields:<key:"sidecar.istio.io/rewriteAppHTTPProbers" value:<string_value:"true" > > > locality:<> build_version:"f9c0feb10cc42277e97dd080a2e045f62d1739a9/1.11.0-dev/Modified/RELEASE/BoringSSL" > type_url:"type.googleapis.com/envoy.api.v2.Cluster" response_nonce:"59a1bb9a-aca6-47c0-b718-a86d7e392db0" error_detail:<code:13 message:"Error adding/updating cluster(s) outbound|443||afv.test.recouv: Invalid path: /etc/nginx-external-tls/tls.crt" >
  message: "Error adding/updating cluster(s) outbound|443||<my.hostname>: Invalid path: /etc/nginx-external-tls/tls.crt"

$ kubectl -n istio-system logs istio-egressgateway-5ff889c5fd-jtz55
[2020-01-08 08:59:53.737][24][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:527] updating TLS cluster outbound_.80_._.<my.hostname>
[2020-01-08 08:59:53.737][24][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:975] membership update for TLS cluster outbound_.80_._.<my.hostname> added 1 removed 0
[2020-01-08 08:59:53.737][24][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:527] updating TLS cluster outbound_.443_._.<my.hostname>
[2020-01-08 08:59:53.737][24][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:975] membership update for TLS cluster outbound_.443_._.<my.hostname> added 1 removed 0

# External AC
$ kubectl -n istio-system exec -it istio-egressgateway-5ff889c5fd-jtz55 -- ls /etc/cluster-issuer-tls
tls.crt  tls.key

# Client
$ kubectl -n istio-system exec -it istio-egressgateway-5ff889c5fd-jtz55 -- ls /etc/nginx-external-tls
ca.crt  tls.crt  tls.key