fix: always verify TLS on aibridgeproxyd upstream transport (#26131) (#26255)

jdomeracki-coder · ssncferreira · f0ssel · web-flow · commit d875dd1496dd · 2026-06-11T14:06:37.000-04:00
Cherry-pick backport to `release/2.33`.

---------

Co-authored-by: Susana Ferreira &lt;susana@coder.com&gt;
Co-authored-by: Garrett Delfosse &lt;delfossegarrett@gmail.com&gt;
diff --git a/docs/ai-coder/ai-gateway/ai-gateway-proxy/setup.md b/docs/ai-coder/ai-gateway/ai-gateway-proxy/setup.md
@@ -272,10 +272,6 @@ CODER_AIBRIDGE_PROXY_UPSTREAM_CA=/path/to/corporate-ca.crt
 
 If the system already trusts the upstream proxy's CA certificate, [`CODER_AIBRIDGE_PROXY_UPSTREAM_CA`](../../../reference/cli/server.md#--aibridge-proxy-upstream-ca) is not required.
 
-<!-- TODO(ssncferreira): Add Client Configuration section -->
-
-<!-- TODO(ssncferreira): Add Troubleshooting section -->
-
 ## Client Configuration
 
 To use AI Gateway Proxy, AI tools must be configured to:
@@ -374,3 +370,21 @@ This provides a seamless experience where users don't need to configure anything
 <!-- TODO(ssncferreira): Add registry link for AI Gateway Proxy module for Coder workspaces: https://github.com/coder/internal/issues/1187 -->
 
 For tool-specific configuration details, check the [client compatibility table](../clients/index.md#compatibility) for clients that require proxy-based integration.
+
+## Troubleshooting
+
+### TLS certificate verification failures
+
+When the Coder access URL uses HTTPS, AI Gateway Proxy must trust the TLS certificate served at that URL (either Coder's
+own certificate or a load balancer's, if TLS is terminated there) to forward intercepted requests to AI Gateway.
+This primarily affects deployments using a self-signed or internal CA, since publicly trusted CAs are typically already
+in the system trust store.
+If the certificate is signed by a CA not in the system trust store, the connection fails and the Coder server logs:
+
+```shell
+WARN: Cannot read TLS response from mitm'd server tls: failed to verify certificate: x509: certificate signed by unknown authority
+```
+
+To resolve, add the CA that signed that certificate to the [system trust store](#system-trust-store) of the host running
+AI Gateway Proxy (the same host as `coderd`, since the proxy runs in-process), then restart Coder so AI Gateway Proxy
+reloads the trust store.
diff --git a/enterprise/aibridgeproxyd/aibridgeproxyd.go b/enterprise/aibridgeproxyd/aibridgeproxyd.go
@@ -289,14 +289,21 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (*Server, error)
 		proxy.CertStore = NewCertCache()
 	}
 
-	// Always set secure TLS defaults, overriding goproxy's default.
-	// This ensures secure TLS connections for:
-	// - HTTPS upstream proxy connections
-	// - MITM'd requests if aibridge uses HTTPS
+	// Override goproxy's default transport, which has InsecureSkipVerify: true.
+	// This applies to all proxy.Tr traffic: MITM'd requests forwarded to aibridge,
+	// passthrough requests, and HTTPS upstream proxy connections. Proxy is
+	// intentionally unset so MITM'd requests go directly to aibridge, never
+	// through an upstream proxy or HTTPS_PROXY env var.
 	rootCAs, err := x509.SystemCertPool()
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load system certificate pool: %w", err)
 	}
+	proxy.Tr = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+			RootCAs:    rootCAs,
+		},
+	}
 
 	srv := &Server{
 		ctx:                      ctx,
@@ -333,15 +340,6 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (*Server, error)
 			}
 		}
 
-		// Set transport without Proxy to ensure MITM'd requests go directly to aibridge,
-		// not through any upstream proxy.
-		proxy.Tr = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				MinVersion: tls.VersionTLS12,
-				RootCAs:    rootCAs,
-			},
-		}
-
 		// Add custom CA certificate if provided (for corporate proxies with private CAs).
 		// If no CA certificate is provided, the system certificate pool is used.
 		if opts.UpstreamProxyCA != "" {
diff --git a/enterprise/aibridgeproxyd/aibridgeproxyd_test.go b/enterprise/aibridgeproxyd/aibridgeproxyd_test.go
@@ -1703,6 +1703,37 @@ func TestListenerTLS(t *testing.T) {
 	}
 }
 
+// TestProxy_AIBridgeTLSVerification verifies the proxy refuses to forward
+// MITM'd requests to an aibridge endpoint whose TLS certificate is not trusted.
+func TestProxy_AIBridgeTLSVerification(t *testing.T) {
+	t.Parallel()
+
+	// HTTPS server with a self-signed cert untrusted by the system pool,
+	// standing in for aibridge.
+	aibridgeServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(aibridgeServer.Close)
+
+	srv := newTestProxy(t,
+		withCoderAccessURL(aibridgeServer.URL),
+		withDomainAllowlist(aibridgeproxyd.HostAnthropic),
+	)
+
+	client := newProxyClient(t, srv, makeProxyAuthHeader("test-token"), getProxyCertPool(t), false)
+
+	req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
+		"https://"+aibridgeproxyd.HostAnthropic+"/v1/messages",
+		strings.NewReader(`{}`))
+	require.NoError(t, err)
+
+	resp, err := client.Do(req)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	require.Error(t, err, "proxy must refuse to forward MITM'd requests to an untrusted aibridge cert")
+}
+
 // TestServeCACert validates that a configured certificate file can be served correctly by the API.
 //
 // Note: Tests for certificate file errors (missing file, invalid PEM) are
