Google Container Engine (Kubernetes): Websocket (Socket.io) не работает на нескольких репликах
Я новичок в Google Container Engine (GKE). Когда запустить на localhost все работает нормально, но при развертывании в производство с GKE я получил ошибку websocket.
Мое приложение для узла разрабатывается с Hapi.js а также Socket.io и моя структура показана на изображении ниже.
Я использую клей для создания сервера Hapi. Ниже мой manifest.json
{
...
"connections": [
{
"host": "app",
"address": "0.0.0.0",
"port": 8000,
"labels": ["api"],
"routes": {
"cors": false,
"security": {
"hsts": false,
"xframe": true,
"xss": true,
"noOpen": true,
"noSniff": true
}
},
"router": {
"stripTrailingSlash": true
},
"load": {
"maxHeapUsedBytes": 1073741824,
"maxRssBytes": 1610612736,
"maxEventLoopDelay": 5000
}
},
{
"host": "app",
"address": "0.0.0.0",
"port": 8099,
"labels": ["web"],
"routes": {
"cors": true,
"security": {
"hsts": false,
"xframe": true,
"xss": true,
"noOpen": true,
"noSniff": true
}
},
"router": {
"stripTrailingSlash": true
},
"load": {
"maxHeapUsedBytes": 1073741824,
"maxRssBytes": 1610612736,
"maxEventLoopDelay": 5000
}
},
{
"host": "app",
"address": "0.0.0.0",
"port": 8999,
"labels": ["admin"],
"routes": {
"cors": true,
"security": {
"hsts": false,
"xframe": true,
"xss": true,
"noOpen": true,
"noSniff": true
}
},
"router": {
"stripTrailingSlash": true
},
"load": {
"maxHeapUsedBytes": 1073741824,
"maxRssBytes": 1610612736,
"maxEventLoopDelay": 5000
},
"state": {
"ttl": null,
"isSecure": false,
"isHttpOnly": true,
"path": null,
"domain": null,
"encoding": "none",
"clearInvalid": false,
"strictHeader": true
}
}
],
...
}
И мой nginx.conf
worker_processes 5; ## Default: 1
worker_rlimit_nofile 8192;
error_log /dev/stdout info;
events {
worker_connections 4096; ## Default: 1024
}
http {
access_log /dev/stdout;
server {
listen 80 default_server;
listen [::]:80 default_server;
# Redirect all HTTP requests to HTTPS with a 301 Moved Permanently response.
return 301 https://$host$request_uri;
}
server {
listen 443 ssl default_server;
listen [::]:443 ssl default_server;
server_name _;
# Configure ssl
ssl_certificate /etc/secret/ssl/myapp.com.csr;
ssl_certificate_key /etc/secret/ssl/myapp.com.key;
include /etc/nginx/ssl-params.conf;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name api.myapp.com;
location / {
proxy_pass http://api_app/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Handle Web Socket connections
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name myapp.com;
location / {
proxy_pass http://web_app/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Handle Web Socket connections
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name admin.myapp.com;
location / {
proxy_pass http://admin_app/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Handle Web Socket connections
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
# Define your "upstream" servers - the
# servers request will be sent to
upstream api_app {
server localhost:8000;
}
upstream web_app {
server localhost:8099;
}
upstream admin_app {
server localhost:8999;
}
}
Кубернетес сервис app-service.yaml
apiVersion: v1
kind: Service
metadata:
name: app-nginx
labels:
app: app-nginx
spec:
type: LoadBalancer
ports:
# The port that this service should serve on.
- port: 80
targetPort: 80
protocol: TCP
name: http
- port: 443
targetPort: 443
protocol: TCP
name: https
# Label keys and values that must match in order to receive traffic for this service.
selector:
app: app-nginx
Kubernetes Deployment app-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: app-nginx
spec:
replicas: 3
template:
metadata:
labels:
app: app-nginx
spec:
containers:
- name: nginx
image: us.gcr.io/myproject/nginx
ports:
- containerPort: 80
name: http
- containerPort: 443
name: https
volumeMounts:
# This name must match the volumes.name below.
- name: ssl-secret
readOnly: true
mountPath: /etc/secret/ssl
- name: app
image: us.gcr.io/myproject/bts-server
ports:
- containerPort: 8000
name: api
- containerPort: 8099
name: web
- containerPort: 8999
name: admin
volumeMounts:
# This name must match the volumes.name below.
- name: client-secret
readOnly: true
mountPath: /etc/secret/client
- name: admin-secret
readOnly: true
mountPath: /etc/secret/admin
volumes:
- name: ssl-secret
secret:
secretName: ssl-key-secret
- name: client-secret
secret:
secretName: client-key-secret
- name: admin-secret
secret:
secretName: admin-key-secret
А я пользуюсь Cloudflare SSL full strict,
Ошибка при получении из консоли браузера:
WebSocket connection to 'wss://api.myapp.com/socket.io/?EIO=3&transport=websocket&sid=4Ky-y9K7J0XotrBFAAAQ' failed: WebSocket is closed before the connection is established.
https://api.myapp.com/socket.io/?EIO=3&transport=polling&t=LYByND2&sid=4Ky-y9K7J0XotrBFAAAQ Failed to load resource: the server responded with a status of 400 ()
VM50:35 WebSocket connection to 'wss://api.myapp.com/socket.io/?EIO=3&transport=websocket&sid=FsCGx-UE7ohrsSSqAAAT' failed: Error during WebSocket handshake: Unexpected response code: 502WrappedWebSocket @ VM50:35WS.doOpen @ socket.io.js:6605Transport.open @ socket.io.js:4695Socket.probe @ socket.io.js:3465Socket.onOpen @ socket.io.js:3486Socket.onHandshake @ socket.io.js:3546Socket.onPacket @ socket.io.js:3508(anonymous function) @ socket.io.js:3341Emitter.emit @ socket.io.js:6102Transport.onPacket @ socket.io.js:4760callback @ socket.io.js:4510(anonymous function) @ socket.io.js:5385exports.decodePayloadAsBinary @ socket.io.js:5384exports.decodePayload @ socket.io.js:5152Polling.onData @ socket.io.js:4514(anonymous function) @ socket.io.js:4070Emitter.emit @ socket.io.js:6102Request.onData @ socket.io.js:4231Request.onLoad @ socket.io.js:4312xhr.onreadystatechange @ socket.io.js:4184
socket.io.js:4196 GET https://api.myapp.com/socket.io/?EIO=3&transport=polling&t=LYByNpy&sid=FsCGx-UE7ohrsSSqAAAT 400 ()
А вот логи Nginx:
[22/Nov/2016:12:10:19 +0000] "GET /socket.io/?EIO=3&transport=websocket&sid=MGc--oncQbQI6NOZAAAX HTTP/1.1" 101 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
10.8.0.1 - - [22/Nov/2016:12:10:19 +0000] "POST /socket.io/?EIO=3&transport=polling&t=LYByQBw&sid=MGc--oncQbQI6NOZAAAX HTTP/1.1" 200 2 "https://myapp.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
10.128.0.2 - - [22/Nov/2016:12:10:20 +0000] "GET /socket.io/?EIO=3&transport=polling&t=LYByQKp HTTP/1.1" 200 101 "https://myapp.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
10.8.0.1 - - [22/Nov/2016:12:10:21 +0000] "GET /socket.io/?EIO=3&transport=polling&t=LYByQWo&sid=c5nkusT9fEPRsu2rAAAY HTTP/1.1" 200 24 "https://myapp.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
2016/11/22 12:10:21 [error] 6#6: *157 connect() failed (111: Connection refused) while connecting to upstream, client: 10.8.0.1, server: api.myapp.com, request: "GET /socket.io/?EIO=3&transport=polling&t=LYByQaN&sid=c5nkusT9fEPRsu2rAAAY HTTP/1.1", upstream: "http://[::1]:8000/socket.io/?EIO=3&transport=polling&t=LYByQaN&sid=c5nkusT9fEPRsu2rAAAY", host: "api.myapp.com", referrer: "https://myapp.com/"
2016/11/22 12:10:21 [warn] 6#6: *157 upstream server temporarily disabled while connecting to upstream, client: 10.8.0.1, server: api.myapp.com, request: "GET /socket.io/?EIO=3&transport=polling&t=LYByQaN&sid=c5nkusT9fEPRsu2rAAAY HTTP/1.1", upstream: "http://[::1]:8000/socket.io/?EIO=3&transport=polling&t=LYByQaN&sid=c5nkusT9fEPRsu2rAAAY", host: "api.myapp.com", referrer: "https://myapp.com/"
10.8.0.1 - - [22/Nov/2016:12:10:22 +0000] "GET /socket.io/?EIO=3&transport=polling&t=LYByQaN&sid=c5nkusT9fEPRsu2rAAAY HTTP/1.1" 200 4 "https://myapp.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
ОБНОВИТЬ
Когда я меняюсь replicas в 1 в app-deployment.yaml это работает. Но я думаю, что это не очень хорошее решение. Мне нужно 3 реплики.
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: app-nginx
spec:
replicas: 1
template:
metadata:
labels:
app: app-nginx
Как заставить работать 3 реплики?
1 ответ
Решение
После того как я обновлю шаблон службы Kubernetes для использования sessionAffinity: ClientIP это работает сейчас. Но просто получите ошибку при первом нажатии Ctrl + F5 и при втором нажатии все работает нормально.
Error during WebSocket handshake: Unexpected response code: 400
Тем не менее, я все еще получаю данные с сервера. Так что я думаю, что все в порядке.
Обновленный Сервисный шаблон
apiVersion: v1
kind: Service
metadata:
name: app-nginx
labels:
app: app-nginx
spec:
sessionAffinity: ClientIP
type: LoadBalancer
ports:
# The port that this service should serve on.
- port: 80
targetPort: 80
protocol: TCP
name: http
- port: 443
targetPort: 443
protocol: TCP
name: https
# Label keys and values that must match in order
# to receive traffic for this service.
selector:
app: app-nginx