Skip to content

Commit ffc8a1a

Browse files
committed
fix: install tenant referencegrant crd whenever route sync is enabled
- route controllers watch virtual ReferenceGrants regardless of sync.toHost.gatewayApi.referenceGrants.enabled; with the flag "false" the CRD was never installed and the watch failed forever, silently blocking all HTTPRoute/TLSRoute sync - extract EnsureReferenceGrantCRD and call it from the HTTPRoute and TLSRoute mappers, keeping "false" semantics: grants never sync to the host and virtual grants stay authoritative for cross-namespace refs - add gatewayapi-grants-disabled e2e suite plus a unit test asserting route mappers ensure the grant CRD when grant sync is disabled Signed-off-by: Ryan Swanson <ryan.swanson@loft.sh>
1 parent 30ffee3 commit ffc8a1a

7 files changed

Lines changed: 279 additions & 1 deletion

File tree

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Suite: gatewayapi-grants-disabled-vcluster
2+
// vCluster: Gateway API route sync with referenceGrants.enabled "false" —
3+
// route controllers must start and virtual ReferenceGrants must stay
4+
// authoritative for cross-namespace refs without syncing to the host.
5+
// Run: just run-e2e 'pr && gatewayapi'
6+
package e2e_next
7+
8+
import (
9+
"context"
10+
_ "embed"
11+
12+
"github.com/loft-sh/e2e-framework/pkg/setup/cluster"
13+
"github.com/loft-sh/vcluster/e2e-next/clusters"
14+
"github.com/loft-sh/vcluster/e2e-next/labels"
15+
"github.com/loft-sh/vcluster/e2e-next/setup"
16+
"github.com/loft-sh/vcluster/e2e-next/setup/lazyvcluster"
17+
"github.com/loft-sh/vcluster/e2e-next/test_gatewayapi"
18+
. "github.com/onsi/ginkgo/v2"
19+
)
20+
21+
//go:embed vcluster-gatewayapi-grants-disabled.yaml
22+
var gatewayAPIGrantsDisabledVClusterYAML string
23+
24+
const gatewayAPIGrantsDisabledVClusterName = "gatewayapi-grants-disabled-vcluster"
25+
26+
func init() { suiteGatewayAPIGrantsDisabledVCluster() }
27+
28+
func suiteGatewayAPIGrantsDisabledVCluster() {
29+
// Ordered so all specs share one lazyvcluster bring-up; specs are independent.
30+
Describe("gatewayapi-grants-disabled-vcluster", labels.PR, labels.GatewayAPI, Ordered,
31+
cluster.Use(clusters.HostCluster),
32+
func() {
33+
BeforeAll(func(ctx context.Context) context.Context {
34+
return lazyvcluster.LazyVCluster(ctx,
35+
gatewayAPIGrantsDisabledVClusterName,
36+
gatewayAPIGrantsDisabledVClusterYAML,
37+
lazyvcluster.WithPreSetup(setup.GatewayAPIPreSetup()),
38+
)
39+
})
40+
41+
test_gatewayapi.GatewayAPIGrantsDisabledSpec()
42+
},
43+
)
44+
}
Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package test_gatewayapi
2+
3+
import (
4+
"context"
5+
6+
"github.com/loft-sh/vcluster/e2e-next/constants"
7+
"github.com/loft-sh/vcluster/e2e-next/labels"
8+
"github.com/loft-sh/vcluster/pkg/util/random"
9+
"github.com/loft-sh/vcluster/pkg/util/translate"
10+
. "github.com/onsi/ginkgo/v2"
11+
. "github.com/onsi/gomega"
12+
corev1 "k8s.io/api/core/v1"
13+
kerrors "k8s.io/apimachinery/pkg/api/errors"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/types"
16+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
17+
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
18+
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
19+
)
20+
21+
const grantsDisabledGatewayClassSelectorValue = "gatewayapi-grants-disabled-vcluster"
22+
23+
// GatewayAPIGrantsDisabledSpec registers route sync tests for a vCluster with
24+
// sync.toHost.gatewayApi.referenceGrants.enabled set to "false" while route
25+
// sync stays on. Disabling grant sync must not break the route controllers and
26+
// must keep virtual ReferenceGrants authoritative for cross-namespace refs.
27+
func GatewayAPIGrantsDisabledSpec() {
28+
Describe("Gateway API toHost with ReferenceGrant sync disabled", labels.GatewayAPI, func() {
29+
var (
30+
hostClient ctrlclient.Client
31+
vClusterClient ctrlclient.Client
32+
vClusterName string
33+
vClusterHostNS string
34+
)
35+
36+
BeforeEach(func(ctx context.Context) {
37+
clients := newGatewayAPIClients(ctx)
38+
hostClient = clients.HostClient
39+
vClusterClient = clients.VClusterClient
40+
vClusterName = clients.VClusterName
41+
vClusterHostNS = clients.VClusterHostNS
42+
43+
Eventually(func(g Gomega) {
44+
g.Expect(vClusterClient.List(ctx, &gatewayv1.HTTPRouteList{}, ctrlclient.InNamespace("default"))).To(Succeed())
45+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
46+
})
47+
48+
It("syncs HTTPRoutes to the host while ReferenceGrant sync is disabled", func(ctx context.Context) {
49+
suffix := random.String(6)
50+
class := createGatewayClass(ctx, hostClient, "gc-rgd-"+suffix, grantsDisabledGatewayClassSelectorValue, "grants disabled class")
51+
Eventually(func(g Gomega) {
52+
g.Expect(vClusterClient.Get(ctx, types.NamespacedName{Name: class.Name}, &gatewayv1.GatewayClass{})).To(Succeed())
53+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
54+
55+
ns := createTenantNamespace(ctx, vClusterClient, "rgd-app-"+suffix)
56+
gw := tenantGateway(ns.Name, "gw-rgd-"+suffix, class.Name)
57+
Expect(vClusterClient.Create(ctx, gw)).To(Succeed())
58+
DeferCleanup(func(ctx context.Context) {
59+
Expect(ctrlclient.IgnoreNotFound(vClusterClient.Delete(ctx, gw))).To(Succeed())
60+
})
61+
hostGatewayName := translate.SafeConcatName(gw.Name, "x", ns.Name, "x", vClusterName)
62+
Eventually(func(g Gomega) {
63+
g.Expect(hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostGatewayName}, &gatewayv1.Gateway{})).To(Succeed())
64+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
65+
66+
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "rgd-backend-" + suffix, Namespace: ns.Name}, Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 80}}}}
67+
Expect(vClusterClient.Create(ctx, svc)).To(Succeed())
68+
69+
route := tenantHTTPRoute(ns.Name, "route-rgd-"+suffix, gw.Name, svc.Name)
70+
Expect(vClusterClient.Create(ctx, route)).To(Succeed())
71+
DeferCleanup(func(ctx context.Context) {
72+
Expect(ctrlclient.IgnoreNotFound(vClusterClient.Delete(ctx, route))).To(Succeed())
73+
})
74+
75+
hostRouteName := translate.SafeConcatName(route.Name, "x", ns.Name, "x", vClusterName)
76+
Eventually(func(g Gomega) {
77+
g.Expect(hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostRouteName}, &gatewayv1.HTTPRoute{})).To(Succeed())
78+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
79+
})
80+
81+
It("authorizes cross-namespace backendRefs via virtual ReferenceGrants without syncing grants to the host", func(ctx context.Context) {
82+
suffix := random.String(6)
83+
class := createGatewayClass(ctx, hostClient, "gc-rgd-xns-"+suffix, grantsDisabledGatewayClassSelectorValue, "grants disabled cross-namespace class")
84+
Eventually(func(g Gomega) {
85+
g.Expect(vClusterClient.Get(ctx, types.NamespacedName{Name: class.Name}, &gatewayv1.GatewayClass{})).To(Succeed())
86+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
87+
88+
frontend := createTenantNamespace(ctx, vClusterClient, "rgd-frontend-"+suffix)
89+
backend := createTenantNamespace(ctx, vClusterClient, "rgd-backend-"+suffix)
90+
gw := tenantGateway(frontend.Name, "gw-rgd-xns-"+suffix, class.Name)
91+
Expect(vClusterClient.Create(ctx, gw)).To(Succeed())
92+
DeferCleanup(func(ctx context.Context) {
93+
Expect(ctrlclient.IgnoreNotFound(vClusterClient.Delete(ctx, gw))).To(Succeed())
94+
})
95+
hostGatewayName := translate.SafeConcatName(gw.Name, "x", frontend.Name, "x", vClusterName)
96+
Eventually(func(g Gomega) {
97+
g.Expect(hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostGatewayName}, &gatewayv1.Gateway{})).To(Succeed())
98+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
99+
100+
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "rgd-xns-backend-" + suffix, Namespace: backend.Name}, Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 80}}}}
101+
Expect(vClusterClient.Create(ctx, svc)).To(Succeed())
102+
103+
var hostRouteName string
104+
var route *gatewayv1.HTTPRoute
105+
var grant *gatewayv1beta1.ReferenceGrant
106+
107+
By("creating a route whose backendRef crosses into another namespace and expecting no host sync", func() {
108+
route = crossNamespaceRoute(frontend.Name, "route-rgd-xns-"+suffix, gw.Name, backend.Name, svc.Name)
109+
Expect(vClusterClient.Create(ctx, route)).To(Succeed())
110+
DeferCleanup(func(ctx context.Context) {
111+
Expect(ctrlclient.IgnoreNotFound(vClusterClient.Delete(ctx, route))).To(Succeed())
112+
})
113+
114+
hostRouteName = translate.SafeConcatName(route.Name, "x", frontend.Name, "x", vClusterName)
115+
Consistently(func(g Gomega) {
116+
err := hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostRouteName}, &gatewayv1.HTTPRoute{})
117+
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "host route %s should stay absent until a grant permits it, got error: %v", hostRouteName, err)
118+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutShort).Should(Succeed())
119+
})
120+
121+
By("creating a virtual ReferenceGrant and expecting the route to sync", func() {
122+
// The ReferenceGrant CRD must be served in the tenant cluster even
123+
// with grant sync disabled — virtual grants still govern
124+
// cross-namespace authorization.
125+
Eventually(func(g Gomega) {
126+
g.Expect(vClusterClient.List(ctx, &gatewayv1beta1.ReferenceGrantList{}, ctrlclient.InNamespace(backend.Name))).To(Succeed())
127+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
128+
129+
grant = &gatewayv1beta1.ReferenceGrant{
130+
ObjectMeta: metav1.ObjectMeta{Name: "allow-rgd-" + suffix, Namespace: backend.Name},
131+
Spec: gatewayv1beta1.ReferenceGrantSpec{
132+
From: []gatewayv1beta1.ReferenceGrantFrom{{Group: gatewayv1.GroupName, Kind: "HTTPRoute", Namespace: gatewayv1.Namespace(frontend.Name)}},
133+
To: []gatewayv1beta1.ReferenceGrantTo{{Group: "", Kind: "Service"}},
134+
},
135+
}
136+
Expect(vClusterClient.Create(ctx, grant)).To(Succeed())
137+
DeferCleanup(func(ctx context.Context) {
138+
Expect(ctrlclient.IgnoreNotFound(vClusterClient.Delete(ctx, grant))).To(Succeed())
139+
})
140+
Eventually(func(g Gomega) {
141+
g.Expect(hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostRouteName}, &gatewayv1.HTTPRoute{})).To(Succeed())
142+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
143+
})
144+
145+
By("expecting the virtual ReferenceGrant to never sync to the host", func() {
146+
hostGrantName := translate.SafeConcatName(grant.Name, "x", backend.Name, "x", vClusterName)
147+
Consistently(func(g Gomega) {
148+
err := hostClient.Get(ctx, types.NamespacedName{Namespace: vClusterHostNS, Name: hostGrantName}, &gatewayv1beta1.ReferenceGrant{})
149+
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "host grant %s should not exist with grant sync disabled, got error: %v", hostGrantName, err)
150+
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutShort).Should(Succeed())
151+
})
152+
})
153+
})
154+
}
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
controlPlane:
2+
statefulSet:
3+
image:
4+
registry: ""
5+
repository: {{ .Repository }}
6+
tag: {{ .Tag }}
7+
sync:
8+
fromHost:
9+
gatewayClasses:
10+
enabled: true
11+
selector:
12+
matchLabels:
13+
e2e.vcluster.loft.sh/gatewayclass: gatewayapi-grants-disabled-vcluster
14+
toHost:
15+
services:
16+
enabled: true
17+
gatewayApi:
18+
gateways:
19+
enabled: true
20+
httpRoutes:
21+
enabled: true
22+
referenceGrants:
23+
enabled: false

pkg/mappings/resources/httproutes.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,10 @@ func CreateHTTPRouteMapper(ctx *synccontext.RegisterContext) (synccontext.Mapper
2525
return nil, err
2626
}
2727

28+
err = EnsureReferenceGrantCRD(ctx)
29+
if err != nil {
30+
return nil, err
31+
}
32+
2833
return generic.NewMapper(ctx, &gatewayv1.HTTPRoute{}, translate.Default.HostName)
2934
}

pkg/mappings/resources/referencegrants.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,14 @@ func CreateReferenceGrantMapper(ctx *synccontext.RegisterContext) (synccontext.M
2323
return nil, err
2424
}
2525

26-
err = util.EnsureCRD(ctx.Context, ctx.VirtualManager.GetConfig(), []byte(referenceGrantsCRD), mappings.ReferenceGrants())
26+
err = EnsureReferenceGrantCRD(ctx)
2727
if err != nil {
2828
return nil, err
2929
}
3030

3131
return generic.NewMapper(ctx, &gatewayv1.ReferenceGrant{}, translate.Default.HostName)
3232
}
33+
34+
func EnsureReferenceGrantCRD(ctx *synccontext.RegisterContext) error {
35+
return util.EnsureCRD(ctx.Context, ctx.VirtualManager.GetConfig(), []byte(referenceGrantsCRD), mappings.ReferenceGrants())
36+
}

pkg/mappings/resources/register_gateway_test.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,13 @@ import (
77

88
pkgconfig "github.com/loft-sh/vcluster/pkg/config"
99
"github.com/loft-sh/vcluster/pkg/mappings"
10+
"github.com/loft-sh/vcluster/pkg/scheme"
1011
"github.com/loft-sh/vcluster/pkg/syncer/synccontext"
12+
"github.com/loft-sh/vcluster/pkg/util"
1113
gatewayapiutil "github.com/loft-sh/vcluster/pkg/util/gatewayapi"
14+
testingutil "github.com/loft-sh/vcluster/pkg/util/testing"
1215
"k8s.io/apimachinery/pkg/runtime/schema"
16+
"k8s.io/client-go/rest"
1317
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
1418
)
1519

@@ -57,3 +61,42 @@ func TestGatewayMappersUseLatestVersions(t *testing.T) {
5761
t.Fatalf("expected ReferenceGrant mapper to use the latest version %s, got %s", want, got)
5862
}
5963
}
64+
65+
func TestRouteMappersEnsureReferenceGrantCRDWhenGrantSyncDisabled(t *testing.T) {
66+
ensured := map[schema.GroupVersionKind]bool{}
67+
restoreEnsureCRD := util.EnsureCRD
68+
restoreKindExists := util.KindExists
69+
util.EnsureCRD = func(_ context.Context, _ *rest.Config, _ []byte, gvk schema.GroupVersionKind) error {
70+
ensured[gvk] = true
71+
return nil
72+
}
73+
util.KindExists = func(_ *rest.Config, _ schema.GroupVersionKind) (bool, error) {
74+
return true, nil
75+
}
76+
t.Cleanup(func() {
77+
util.EnsureCRD = restoreEnsureCRD
78+
util.KindExists = restoreKindExists
79+
})
80+
81+
fakeClient := testingutil.NewFakeClient(scheme.Scheme)
82+
ctx := &synccontext.RegisterContext{
83+
Context: context.Background(),
84+
Config: &pkgconfig.VirtualClusterConfig{},
85+
VirtualManager: testingutil.NewFakeManager(fakeClient),
86+
HostManager: testingutil.NewFakeManager(fakeClient),
87+
}
88+
ctx.Config.Sync.ToHost.GatewayAPI.ReferenceGrants.Enabled = "false"
89+
ctx.Config.Sync.ToHost.GatewayAPI.HTTPRoutes.Enabled = true
90+
ctx.Config.Sync.ToHost.GatewayAPI.TLSRoutes.Enabled = true
91+
92+
if _, err := CreateHTTPRouteMapper(ctx); err != nil {
93+
t.Fatalf("create HTTPRoute mapper: %v", err)
94+
}
95+
if _, err := CreateTLSRouteMapper(ctx); err != nil {
96+
t.Fatalf("create TLSRoute mapper: %v", err)
97+
}
98+
99+
if !ensured[mappings.ReferenceGrants()] {
100+
t.Fatalf("route mappers must ensure the tenant ReferenceGrant CRD even with grant sync disabled; route controllers watch virtual ReferenceGrants for cross-namespace authorization")
101+
}
102+
}

pkg/mappings/resources/tlsroutes.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,10 @@ func CreateTLSRouteMapper(ctx *synccontext.RegisterContext) (synccontext.Mapper,
3030
return nil, err
3131
}
3232

33+
err = EnsureReferenceGrantCRD(ctx)
34+
if err != nil {
35+
return nil, err
36+
}
37+
3338
return generic.NewMapper(ctx, &gatewayv1.TLSRoute{}, translate.Default.HostName)
3439
}

0 commit comments

Comments
 (0)