Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 63 additions & 4 deletions internal/controller/ingress/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package ingress

import (
"context"
"errors"
"fmt"
"sort"
"time"
Expand Down Expand Up @@ -130,6 +131,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
if statusErr := r.updateIngressStatus(ctx, ingress, config, err); statusErr != nil {
logger.Error(statusErr, "Failed to update Ingress status after reconcile error")
}
// Surface the failure on the class configuration as well. Otherwise the stale
// aggregate keeps reporting Ready=True, which hides the fact that the tunnel
// configuration no longer matches the Ingresses in the cluster.
r.updateConfigErrorStatus(ctx, config, err)
return ctrl.Result{RequeueAfter: 30 * time.Second}, err
}

Expand Down Expand Up @@ -343,6 +348,7 @@ func (r *Reconciler) handleDeletion(
logger.Error(err, "Failed to rebuild tunnel config, continuing with finalizer removal")
r.Recorder.Event(ingress, corev1.EventTypeWarning, "CleanupWarning",
"Failed to rebuild tunnel config: "+cf.SanitizeErrorMessage(err))
r.updateConfigErrorStatus(ctx, config, err)
}
// Don't block deletion on tunnel config rebuild failure
}
Expand Down Expand Up @@ -379,10 +385,40 @@ func (r *Reconciler) handleDeletion(
return ctrl.Result{}, nil
}

// handleIngressDeletion handles the case where an Ingress was deleted before we could process it
func (*Reconciler) handleIngressDeletion(_ context.Context, _ ctrl.Request) (ctrl.Result, error) {
// We can't determine which config this Ingress belonged to, so we need to rebuild all configs
// In practice, this is rare and the next reconcile will fix any inconsistencies
// handleIngressDeletion handles the case where an Ingress is gone by the time we reconcile it.
//
// This is the normal path for a deletion (the finalizer has already been removed) as well as
// the path for deletions we never observed. The Ingress may have carried rules that are no
// longer declared, and there is no other event left to trigger a rebuild, so every
// TunnelIngressClassConfig is rebuilt from the live Ingresses. Skipping this (as the
// controller used to) leaves the removed rules in the tunnel configuration indefinitely.
func (r *Reconciler) handleIngressDeletion(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)

configList := &networkingv1alpha2.TunnelIngressClassConfigList{}
if err := r.List(ctx, configList); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to list TunnelIngressClassConfigs: %w", err)
}

var errs []error
for i := range configList.Items {
config := &configList.Items[i]
if err := r.rebuildTunnelConfig(ctx, config, nil); err != nil {
logger.Error(err, "Failed to rebuild tunnel config after Ingress deletion",
"config", client.ObjectKeyFromObject(config))
r.updateConfigErrorStatus(ctx, config, err)
errs = append(errs, fmt.Errorf("config %s: %w", client.ObjectKeyFromObject(config), err))
continue
}
logger.Info("Rebuilt tunnel config after Ingress deletion",
"ingress", req.NamespacedName,
"config", client.ObjectKeyFromObject(config))
}

if len(errs) > 0 {
return ctrl.Result{}, errors.Join(errs...)
}

return ctrl.Result{}, nil
}

Expand Down Expand Up @@ -673,6 +709,29 @@ func (r *Reconciler) updateConfigStatus(ctx context.Context, config *networkingv
})
}

// updateConfigErrorStatus marks the TunnelIngressClassConfig as failed.
//
// A failed rebuild leaves the previously written aggregate in place, so the class
// configuration must not keep advertising Ready=True while its tunnel rules no longer
// reflect the Ingresses in the cluster.
func (r *Reconciler) updateConfigErrorStatus(
ctx context.Context,
config *networkingv1alpha2.TunnelIngressClassConfig,
reconcileErr error,
) {
logger := log.FromContext(ctx)

err := controller.UpdateStatusWithConflictRetry(ctx, r.Client, config, func() {
config.Status.State = "error"
config.Status.ObservedGeneration = config.Generation
controller.SetErrorCondition(&config.Status.Conditions, reconcileErr)
})
if err != nil {
logger.Error(err, "Failed to update TunnelIngressClassConfig error status",
"config", client.ObjectKeyFromObject(config))
}
}

// SetupWithManager sets up the controller with the Manager
func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Recorder = mgr.GetEventRecorderFor("cloudflare-ingress-controller")
Expand Down
228 changes: 228 additions & 0 deletions internal/controller/ingress/prune_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2025-2026 The Cloudflare Operator Authors

package ingress

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

networkingv1alpha2 "github.com/StringKe/cloudflare-operator/api/v1alpha2"
"github.com/StringKe/cloudflare-operator/internal/controller/tunnelconfig"
)

const pruneTestNamespace = "cloudflare-operator-system"

const pruneTestConfigName = "cf-tunnel"

func pruneTestIngress(name string, services ...string) *networkingv1.Ingress {
paths := make([]networkingv1.HTTPIngressPath, 0, len(services))
for _, svc := range services {
paths = append(paths, networkingv1.HTTPIngressPath{
Path: "/" + svc,
PathType: ptr.To(networkingv1.PathTypePrefix),
Backend: networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: svc,
Port: networkingv1.ServiceBackendPort{Number: 80},
},
},
})
}

return &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To(pruneTestClassName),
Rules: []networkingv1.IngressRule{{
Host: "app.example.com",
IngressRuleValue: networkingv1.IngressRuleValue{
HTTP: &networkingv1.HTTPIngressRuleValue{Paths: paths},
},
}},
},
}
}

const pruneTestClassName = "cloudflare-tunnel"

// pruneTestObjects returns the IngressClass, TunnelIngressClassConfig and Tunnel that
// the Ingress controller needs to build the aggregate for the "cloudflare-tunnel" class.
func pruneTestObjects() []client.Object {
return []client.Object{
&networkingv1.IngressClass{
ObjectMeta: metav1.ObjectMeta{Name: pruneTestClassName},
Spec: networkingv1.IngressClassSpec{
Controller: ControllerName,
Parameters: &networkingv1.IngressClassParametersReference{
APIGroup: ptr.To("networking.cloudflare-operator.io"),
Kind: "TunnelIngressClassConfig",
Name: pruneTestConfigName,
Namespace: ptr.To(pruneTestNamespace),
},
},
},
&networkingv1alpha2.TunnelIngressClassConfig{
ObjectMeta: metav1.ObjectMeta{Name: pruneTestConfigName, Namespace: pruneTestNamespace},
Spec: networkingv1alpha2.TunnelIngressClassConfigSpec{
TunnelRef: networkingv1alpha2.TunnelReference{Kind: "Tunnel", Name: "my-tunnel"},
},
},
&networkingv1alpha2.Tunnel{
ObjectMeta: metav1.ObjectMeta{Name: "my-tunnel", Namespace: pruneTestNamespace},
Status: networkingv1alpha2.TunnelStatus{
TunnelId: "tid-123",
TunnelName: "my-tunnel",
AccountId: "acc-123",
},
},
}
}

func pruneTestReconciler(t *testing.T, objs ...client.Object) (*Reconciler, client.Client) {
t.Helper()

scheme := setupTestScheme(t)
c := fake.NewClientBuilder().WithScheme(scheme).
WithStatusSubresource(&networkingv1alpha2.TunnelIngressClassConfig{}).
WithObjects(objs...).
Build()

return &Reconciler{
Client: c,
Scheme: scheme,
Recorder: record.NewFakeRecorder(20),
OperatorNamespace: pruneTestNamespace,
}, c
}

func readAggregatedRules(t *testing.T, ctx context.Context, c client.Client) []tunnelconfig.IngressRule {
t.Helper()

writer := tunnelconfig.NewWriter(c, pruneTestNamespace)
config, err := writer.GetTunnelConfig(ctx, "tid-123")
require.NoError(t, err)
require.NotNil(t, config, "tunnel config ConfigMap missing")

source, ok := config.Sources["Ingress/"+pruneTestNamespace+"/"+pruneTestConfigName]
require.True(t, ok, "aggregated Ingress source missing: %v", config.Sources)

return source.Rules
}

// TestRebuildTunnelConfigPrunesRemovedIngressPath verifies that removing a path from an
// Ingress removes the corresponding rule from the aggregated source (issue #49).
func TestRebuildTunnelConfigPrunesRemovedIngressPath(t *testing.T) {
ctx := context.Background()

ingress := pruneTestIngress("web", "one", "two")
objs := append(pruneTestObjects(), ingress)
r, c := pruneTestReconciler(t, objs...)

config := &networkingv1alpha2.TunnelIngressClassConfig{}
require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: pruneTestNamespace, Name: pruneTestConfigName}, config))

require.NoError(t, r.reconcileIngressConfig(ctx, ingress, config))
require.Len(t, readAggregatedRules(t, ctx, c), 2)

// Remove the /two path, as a user editing the Ingress would.
updated := &networkingv1.Ingress{}
require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(ingress), updated))
updated.Spec.Rules[0].HTTP.Paths = updated.Spec.Rules[0].HTTP.Paths[:1]
require.NoError(t, c.Update(ctx, updated))

require.NoError(t, r.reconcileIngressConfig(ctx, updated, config))

rules := readAggregatedRules(t, ctx, c)
require.Len(t, rules, 1)
assert.Equal(t, "/one(/.*)?$", rules[0].Path)
assert.Equal(t, "http://one.default.svc:80", rules[0].Service)
}

// TestHandleIngressDeletionRebuildsAllConfigs verifies that a deletion we only observe as
// "object is gone" still rebuilds every class aggregate. Previously this was a no-op, so
// rules that went away with the deleted Ingress were never pruned.
func TestHandleIngressDeletionRebuildsAllConfigs(t *testing.T) {
ctx := context.Background()

// The tunnel ConfigMap still holds the rule of an Ingress that no longer exists.
objs := append(pruneTestObjects(), pruneTestIngress("web", "one"))
r, c := pruneTestReconciler(t, objs...)

staleWriter := tunnelconfig.NewWriter(c, pruneTestNamespace)
require.NoError(t, staleWriter.WriteSourceConfig(ctx, "tid-123", "acc-123", &tunnelconfig.SourceConfig{
Kind: tunnelconfig.SourceKindIngress,
Namespace: pruneTestNamespace,
Name: pruneTestConfigName,
Rules: []tunnelconfig.IngressRule{
{Hostname: "app.example.com", Path: "/one(/.*)?$", Service: "http://one.default.svc:80", Priority: tunnelconfig.PriorityIngress},
{Hostname: "app.example.com", Path: "/two(/.*)?$", Service: "http://two.default.svc:80", Priority: tunnelconfig.PriorityIngress},
},
}, nil, metav1.GroupVersionKind{}))
require.Len(t, readAggregatedRules(t, ctx, c), 2)

_, err := r.handleIngressDeletion(ctx, ctrl.Request{
NamespacedName: client.ObjectKey{Namespace: "default", Name: "deleted"},
})
require.NoError(t, err)

rules := readAggregatedRules(t, ctx, c)
require.Len(t, rules, 1)
assert.Equal(t, "/one(/.*)?$", rules[0].Path)
}

// TestReconcileSurfacesRebuildFailureOnConfig verifies that a failed rebuild marks the
// TunnelIngressClassConfig as not ready instead of leaving it Ready=True with stale rules.
func TestReconcileSurfacesRebuildFailureOnConfig(t *testing.T) {
ctx := context.Background()

objs := []client.Object{
&networkingv1.IngressClass{
ObjectMeta: metav1.ObjectMeta{Name: pruneTestClassName},
Spec: networkingv1.IngressClassSpec{
Controller: ControllerName,
Parameters: &networkingv1.IngressClassParametersReference{
APIGroup: ptr.To("networking.cloudflare-operator.io"),
Kind: "TunnelIngressClassConfig",
Name: pruneTestConfigName,
Namespace: ptr.To(pruneTestNamespace),
},
},
},
&networkingv1alpha2.TunnelIngressClassConfig{
ObjectMeta: metav1.ObjectMeta{Name: pruneTestConfigName, Namespace: pruneTestNamespace},
Spec: networkingv1alpha2.TunnelIngressClassConfigSpec{
// Referenced tunnel does not exist, so the rebuild must fail.
TunnelRef: networkingv1alpha2.TunnelReference{Kind: "Tunnel", Name: "missing-tunnel"},
},
},
pruneTestIngress("web", "one"),
}
r, c := pruneTestReconciler(t, objs...)

ingress := &networkingv1.Ingress{}
require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: "default", Name: "web"}, ingress))

_, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ingress)})
require.Error(t, err)

config := &networkingv1alpha2.TunnelIngressClassConfig{}
require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: pruneTestNamespace, Name: pruneTestConfigName}, config))
assert.Equal(t, "error", config.Status.State)

ready := meta.FindStatusCondition(config.Status.Conditions, "Ready")
require.NotNil(t, ready)
assert.Equal(t, metav1.ConditionFalse, ready.Status)
assert.Contains(t, ready.Message, "missing-tunnel")
}
34 changes: 26 additions & 8 deletions internal/controller/tunnelconfig/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,12 +195,12 @@ func (*Reconciler) buildCloudflareConfig(config *TunnelConfig) cloudflare.Tunnel
cfConfig.Ingress = append(cfConfig.Ingress, cfRule)
}

// Add catch-all rule if we have any rules
if len(cfConfig.Ingress) > 0 {
cfConfig.Ingress = append(cfConfig.Ingress, cloudflare.UnvalidatedIngressRule{
Service: "http_status:404",
})
}
// Always terminate the configuration with a catch-all rule. Cloudflare rejects an
// ingress list that does not end with one, so omitting it when every source has been
// removed would make the push fail and leave the previously pushed rules in place.
cfConfig.Ingress = append(cfConfig.Ingress, cloudflare.UnvalidatedIngressRule{
Service: "http_status:404",
})

return cfConfig
}
Expand Down Expand Up @@ -301,7 +301,15 @@ func (r *Reconciler) updateConfigStatus(ctx context.Context, cm *corev1.ConfigMa
}
}

// updateConfigMap updates the ConfigMap with new configuration.
// updateConfigMap persists the sync bookkeeping fields on the ConfigMap.
//
// The configuration payload (Sources) is owned by the resource controllers (Ingress,
// Gateway, TunnelBinding) and may be rewritten while this reconcile is in flight.
// Writing back the whole snapshot parsed at the start of the reconcile would revert
// those concurrent writes - which is how rules removed from an Ingress used to be
// resurrected and then kept in place forever, because the reverted ConfigMap matched
// the recorded LastHash and every later reconcile short-circuited. Only the status
// fields are updated here, on a freshly read ConfigMap.
func (r *Reconciler) updateConfigMap(ctx context.Context, cm *corev1.ConfigMap, config *TunnelConfig) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
// Get fresh copy
Expand All @@ -310,8 +318,18 @@ func (r *Reconciler) updateConfigMap(ctx context.Context, cm *corev1.ConfigMap,
return err
}

freshConfig, err := ParseConfig(fresh)
if err != nil {
return err
}

// Only carry over the status fields of the configuration we just synced.
freshConfig.LastHash = config.LastHash
freshConfig.SyncStatus = config.SyncStatus
freshConfig.LastSyncTime = config.LastSyncTime

// Update data
data, err := config.ToConfigMapData()
data, err := freshConfig.ToConfigMapData()
if err != nil {
return err
}
Expand Down
Loading