CLI results in connection timed out when behind haproxy and 5240 is blocked

Subscribed ~field-high as this is blocking customer deployment.

An interesting note, when doing a tcpdump while running the login command on the external server, I see calls to port 80. When running other commands from the same external server, tcpdump only shows calls to 5240.

Confirmed with strace too.

$ snap list maas maas-cli
Name Version Rev Tracking Publisher Notes
maas 3.2.7-12037-g.c688dd446 26274 3.2/stable canonical✓ -
maas-cli 0.6.8 79 latest/stable canonical✓ -

$ maas list
admin http://192.168.12.19:80/MAAS/api/2.0/ <KEY>

^^^ port 80 is set

$ snap run --strace='-e network' maas admin version read -d

(...)

socket(AF_INET6, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET6, sin6_port=htons(0), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::1", &sin6_addr), sin6_scope_id=0}, 28) = 0
/usr/bin/strace: Process 365068 attached
[pid 365068] +++ exited with 0 +++
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=365068, si_uid=1000, si_status=0, si_utime=0, si_stime=0} ---
socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 6
setsockopt(6, SOL_TCP, TCP_NODELAY, [1], 4) = 0
connect(6, {sa_family=AF_INET, sin_port=htons(5240), sin_addr=inet_addr("192.168.12.19")}, 16) = 0
sendto(6, "GET /MAAS/api/2.0/version/ HTTP/"..., 423, 0, NULL, 0) = 423
recvfrom(6, "HTTP/1.1 200 OK\r\nServer: nginx/1"..., 8192, 0, NULL, NULL) = 573
getsockname(6, {sa_family=AF_INET, sin_port=htons(33808), sin_addr=inet_addr("192.168.12.19")}, [16]) = 0
getpeername(6, {sa_family=AF_INET, sin_port=htons(5240), sin_addr=inet_addr("192.168.12.19")}, [16]) = 0
200 OK

        Connection: keep-alive
    Content-Length: 285
  Content-Location: http://192.168.12.19:5240/MAAS/api/2.0/version/
      Content-Type: application/json; charset=utf-8
              Date: Sat, 04 Mar 2023 00:03:22 GMT
            Server: nginx/1.18.0 (Ubuntu)
            Status: 200
              Vary: Authorization
   X-Frame-Options: SAMEORIGIN
   X-Maas-Api-Hash: 7b01cae71568bea2f250e94d09454b13888f04fb

Success.

^^^ initial connect() is to sin_port=htons(5240) instead of 80.

Looks it's purely on the client side.

EDIT: it looks like cached endpoint information on the client.

$ rm ~/.maascli.db
$ grep -a -o http://192.168.12.19:5240/MAAS/api/2.0/version/ ~/.maascli.db
grep: .maascli.db: No such file or directory

$ maas login admin http://192.168.12.19:80/MAAS/api/2.0/ gqG3GBeepxPaaHcRPQ:ZAtMA4XXJ4WnqkNTG9:MnczDFaPSs4tKZnV8zWLBnd3SKwNa79v
connect: (192.168.12.19, 80) ************
send: b'GET /MAAS/api/2.0/users/?op=whoami HTTP/1.1\r\nHost: 192.168.12.19\r\nauthorization: OAuth oauth_nonce="10113373905774274851677893418", oauth_timestamp="1677893418", oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="gqG3GBeepxPaaHcRPQ", oauth_token="ZAtMA4XXJ4WnqkNTG9", oauth_signature="%26MnczDFaPSs4tKZnV8zWLBnd3SKwNa79v"\r\nuser-agent: Python-httplib2/0.14.0 (gzip)\r\naccept-encoding: gzip, deflate\r\n\r\n'
reply: 'HTTP/1.1 200 OK\r\n'
header: Server: nginx/1.18.0 (Ubuntu)
header: Date: Sat, 04 Mar 2023 01:30:18 GMT
header: Content-Type: application/json; charset=utf-8
header: Content-Length: 172
header: Connection: keep-alive
header: Vary: Authorization, Cookie
header: X-Maas-Api-Hash: 7b01cae71568bea2f250e94d09454b13888f04fb
header: X-Frame-Options: SAMEORIGIN
connect: (192.168.12.19, 80) ************
send: b'GET /MAAS/api/2.0/describe/ HTTP/1.1\r\nHost: 192.168.12.19\r\nuser-agent: Python-httplib2/0.14.0 (gzip)\r\naccept-encoding: gzip, deflate\r\n\r\n'
reply: 'HTTP/1.1 200 OK\r\n'
header: Server: nginx/1.18.0 (Ubuntu)
header: Date: Sat, 04 Mar 2023 01:30:18 GMT
header: Content-Type: application/json
header: Content-Length: 527323
header: Connection: keep-alive
header: X-Frame-Options: SAMEORIGIN

You are now logged in to the MAAS server at
http://192.168.12.19:80/MAAS/api/2.0/ with the profile name 'admin'.

For help with the available commands, try:

  maas admin --help

^^^ nothing mentions port 5240

$ grep -a -o http://192.168.12.19:5240/MAAS/api/2.0/version/ ~/.maascli.db
http://192.168.12.19:5240/MAAS/api/2.0/version/
http://192.168.12.19:5240/MAAS/api/2.0/version/

^^^ but it appears in maascli.db

An ugly workaround is something like:

sqlite3 ~/.maascli.db 'UPDATE profiles SET data = REPLACE(data, "http://192.168.12.19:5240/MAAS", "http://192.168.12.19:80/MAAS") WHERE name = "admin";'

Looks like the port 5240 is from /MAAS/api/2.0/describe/.

$ curl -s http://192.168.12.19:80/MAAS/api/2.0/describe/ | jq -r .handlers[].uri | head
http://192.168.12.19:5240/MAAS/api/2.0/nodes/
http://192.168.12.19:5240/MAAS/api/2.0/files/
http://192.168.12.19:5240/MAAS/api/2.0/machines/
http://192.168.12.19:5240/MAAS/api/2.0/nodes/
http://192.168.12.19:5240/MAAS/api/2.0/nodes/
http://192.168.12.19:5240/MAAS/api/2.0/nodes/
http://192.168.12.19:5240/MAAS/api/2.0/version/
http://192.168.12.19:5240/MAAS/api/2.0/account/
http://192.168.12.19:5240/MAAS/api/2.0/nodes/{system_id}/bcache-cache-set/{id}/
http://192.168.12.19:5240/MAAS/api/2.0/nodes/{system_id}/bcache-cache-sets/

$ strace -e network maas login admin http://192.168.12.19:80/MAAS/api/2.0/ gqG3GBeepxPaaHcRPQ:ZAtMA4XXJ4WnqkNTG9:MnczDFaPSs4tKZnV8zWLBnd3SKwNa79v
socket(AF_INET6, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET6, sin6_port=htons(0), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::1", &sin6_addr), sin6_scope_id=0}, 28) = 0
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=137639, si_uid=1000, si_status=0, si_utime=0, si_stime=0} ---
socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 3
setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.12.19")}, 16) = 0
sendto(3, "GET /MAAS/api/2.0/users/?op=whoa"..., 426, 0, NULL, 0) = 426
recvfrom(3, "HTTP/1.1 200 OK\r\nServer: nginx/1"..., 8192, 0, NULL, NULL) = 468
getsockname(3, {sa_family=AF_INET, sin_port=htons(60546), sin_addr=inet_addr("10.0.8.190")}, [16]) = 0
getpeername(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.12.19")}, [16]) = 0
socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 3
setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.12.19")}, 16) = 0
sendto(3, "GET /MAAS/api/2.0/describe/ HTTP"..., 136, 0, NULL, 0) = 136
recvfrom(3, "HTTP/1.1 200 OK\r\nServer: nginx/1"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, " String\\n\\n \", \"method\": \"POST\","..., 516096, 0, NULL, NULL) = 5801
recvfrom(3, ", \"name\": \"delete\", \"op\": null, "..., 507904, 0, NULL, NULL) = 28960
recvfrom(3, "am not_fabrics: Optional. Only "..., 483328, 0, NULL, NULL) = 10136
recvfrom(3, "et_serial\", \"restful\": false}], "..., 466944, 0, NULL, NULL) = 27112
recvfrom(3, "nterfacesHandler\", \"params\": [\"s"..., 442368, 0, NULL, NULL) = 7240
recvfrom(3, "privilege level. The def"..., 434176, 0, NULL, NULL) = 55424
recvfrom(3, "t (and when the enlistment is do"..., 376832, 0, NULL, NULL) = 14480
recvfrom(3, "n\", \"method\": \"GET\", \"name\": \"de"..., 368640, 0, NULL, NULL) = 17376
recvfrom(3, "endor_name``: The device vendor "..., 352256, 0, NULL, NULL) = 40544
recvfrom(3, "ubnets: Optional. Only nodes wi"..., 311296, 0, NULL, NULL) = 26064
recvfrom(3, "e VLAN\\n\\nRetrieves a VLAN on a "..., 278528, 0, NULL, NULL) = 43440
recvfrom(3, "oolean\\n\\n :param node: Optional"..., 237568, 0, NULL, NULL) = 125976
recvfrom(3, "ne or more devices containing th"..., 114688, 0, NULL, NULL) = 30408
recvfrom(3, " user.\\n\\nNotifications ...

@nobuto @cjohnston

Here is another way of how to fix it.
https://code.launchpad.net/~troyanov/maas/+git/maas/+merge/438884

Would be great to know if that works for you.