diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0763f3b..1d7a398 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -182,6 +182,15 @@ jobs:
 
   endpoint:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        endpoint:
+          - mycontext
+          - tcp://127.0.0.1:2375
+    env:
+      # only the explicit endpoint should provide access to a daemon.
+      DOCKER_HOST: unix:///tmp/nonexistent-docker.sock
     services:
       dind:
         image: docker:29.3-dind@sha256:4d90f1f6c400315c2dba96d3ec93c01e64198395cbba04f79d12adce4f737029
@@ -209,11 +218,19 @@ jobs:
           docker --context mycontext info
       -
         name: Set up Docker Buildx
+        id: buildx
         uses: ./
         with:
-          endpoint: mycontext
+          endpoint: ${{ matrix.endpoint }}
+      -
+        name: Build
         env:
-          DOCKER_CONTEXT: mycontext
+          BUILDER: ${{ steps.buildx.outputs.name }}
+        run: |
+          docker buildx build --builder "$BUILDER" - <<'EOF'
+          FROM scratch
+          LABEL test=endpoint
+          EOF
 
   buildkitd-config:
     runs-on: ubuntu-latest
diff --git a/src/main.ts b/src/main.ts
index e747149..78ed948 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -133,11 +133,13 @@ actionsToolkit.run(
 
     if (!standalone && inputs.driver == 'docker-container') {
       const buildkitImages = new Set<string>();
-      if (!builderExists) {
+      if (!builderExists && !inputs.endpoint) {
         buildkitImages.add(resolveBuildKitImage(inputs.driverOpts));
       }
       for (const node of appendNodes) {
-        buildkitImages.add(resolveBuildKitImage(node['driver-opts']));
+        if (!node.endpoint) {
+          buildkitImages.add(resolveBuildKitImage(node['driver-opts']));
+        }
       }
       if (buildkitImages.size > 0) {
         await core.group(`Pulling BuildKit image(s)`, async () => {