Skip to content

Implement skipInvalidResources for syslog-ng logging #2116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 7, 2025
Merged

Implement skipInvalidResources for syslog-ng logging #2116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4221b0e
Implement skipInvalidResources for syslog-ng logging
Copilot Oct 3, 2025
5120b4f
fix: syslogng config test
csatib02 Oct 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions controllers/logging/logging_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,15 +461,17 @@ func (r *LoggingReconciler) clusterConfigurationSyslogNG(resources model.Logging

_, syslogngSpec := resources.GetSyslogNGSpec()
in := syslogngconfig.Input{
Name: resources.Logging.Name,
Namespace: resources.Logging.Namespace,
ClusterOutputs: resources.SyslogNG.ClusterOutputs,
Outputs: resources.SyslogNG.Outputs,
ClusterFlows: resources.SyslogNG.ClusterFlows,
Flows: resources.SyslogNG.Flows,
SecretLoaderFactory: &slf,
SourcePort: syslogng.ServicePort,
SyslogNGSpec: syslogngSpec,
Name: resources.Logging.Name,
Namespace: resources.Logging.Namespace,
ClusterOutputs: resources.SyslogNG.ClusterOutputs,
Outputs: resources.SyslogNG.Outputs,
ClusterFlows: resources.SyslogNG.ClusterFlows,
Flows: resources.SyslogNG.Flows,
SecretLoaderFactory: &slf,
SourcePort: syslogng.ServicePort,
SyslogNGSpec: syslogngSpec,
SkipInvalidResources: resources.Logging.Spec.SkipInvalidResources,
Logger: r.Log,
}
var b strings.Builder
if err := syslogngconfig.RenderConfigInto(in, &b); err != nil {
Expand Down
39 changes: 28 additions & 11 deletions pkg/sdk/logging/model/syslogng/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"emperror.dev/errors"
"github.com/cisco-open/operator-tools/pkg/secret"
"github.com/go-logr/logr"
"github.com/siliconbrain/go-seqs/seqs"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -45,15 +46,17 @@ func RenderConfigInto(in Input, out io.Writer) error {
}

type Input struct {
Name string
Namespace string
SyslogNGSpec *v1beta1.SyslogNGSpec
ClusterOutputs []v1beta1.SyslogNGClusterOutput
Outputs []v1beta1.SyslogNGOutput
ClusterFlows []v1beta1.SyslogNGClusterFlow
Flows []v1beta1.SyslogNGFlow
SecretLoaderFactory SecretLoaderFactory
SourcePort int
Name string
Namespace string
SyslogNGSpec *v1beta1.SyslogNGSpec
ClusterOutputs []v1beta1.SyslogNGClusterOutput
Outputs []v1beta1.SyslogNGOutput
ClusterFlows []v1beta1.SyslogNGClusterFlow
Flows []v1beta1.SyslogNGFlow
SecretLoaderFactory SecretLoaderFactory
SourcePort int
SkipInvalidResources bool
Logger logr.Logger
}

type outputInfo struct {
Expand Down Expand Up @@ -114,16 +117,30 @@ func configRenderer(in Input) (render.Renderer, error) {
logDefs := make([]render.Renderer, 0, len(in.ClusterFlows)+len(in.Flows))
for _, cf := range in.ClusterFlows {
if err := validateClusterOutputs(clusterOutputRefs, client.ObjectKeyFromObject(&cf).String(), cf.Spec.GlobalOutputRefs, cf.Kind); err != nil {
if in.SkipInvalidResources {
// Skip this cluster flow and continue with the next one
in.Logger.Error(err, "skipping invalid SyslogNGClusterFlow", "name", cf.Name, "namespace", cf.Namespace)
continue
}
errs = errors.Append(errs, err)
}
logDefs = append(logDefs, renderClusterFlow(in.Name, clusterOutputRefs, sourceName, cf, in.SecretLoaderFactory))
}
for _, f := range in.Flows {
var flowErrs error
if err := validateClusterOutputs(clusterOutputRefs, client.ObjectKeyFromObject(&f).String(), f.Spec.GlobalOutputRefs, f.Kind); err != nil {
errs = errors.Append(errs, err)
flowErrs = errors.Append(flowErrs, err)
}
if err := validateOutputs(outputRefs, client.ObjectKeyFromObject(&f).String(), f.Spec.LocalOutputRefs); err != nil {
errs = errors.Append(errs, err)
flowErrs = errors.Append(flowErrs, err)
}
if flowErrs != nil {
if in.SkipInvalidResources {
// Skip this flow and continue with the next one
in.Logger.Error(flowErrs, "skipping invalid SyslogNGFlow", "name", f.Name, "namespace", f.Namespace)
continue
}
errs = errors.Append(errs, flowErrs)
}
logDefs = append(logDefs, renderFlow(in.Name, clusterOutputRefs, sourceName, keyDelim(in.SyslogNGSpec.JSONKeyDelimiter), f, in.SecretLoaderFactory))
}
Expand Down
159 changes: 158 additions & 1 deletion pkg/sdk/logging/model/syslogng/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestRenderConfigInto(t *testing.T) {
testCases := map[string]struct {
input Input
wantOut string
wantErr any
wantErr bool
}{
"empty input": {
input: Input{
Expand Down Expand Up @@ -615,6 +615,163 @@ log {
},
wantErr: true,
},
"flow referencing non-existent cluster output with skipInvalidResources": {
input: Input{
SyslogNGSpec: &v1beta1.SyslogNGSpec{},
Name: "test",
Flows: []v1beta1.SyslogNGFlow{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-flow",
},
Spec: v1beta1.SyslogNGFlowSpec{
GlobalOutputRefs: []string{
"clusterout",
},
},
},
},
ClusterOutputs: nil,
SecretLoaderFactory: &TestSecretLoaderFactory{},
SourcePort: 601,
SkipInvalidResources: true,
},
wantErr: false,
wantOut: `@version: current

@include "scl.conf"

source "main_input" {
channel {
source {
network(flags("no-parse") port(601) transport("tcp"));
};
parser {
json-parser(prefix("json."));
};
};
};
`,
},
"clusterFlow referencing non-existent cluster output with skipInvalidResources": {
input: Input{
Name: "test",
SyslogNGSpec: &v1beta1.SyslogNGSpec{},
ClusterFlows: []v1beta1.SyslogNGClusterFlow{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-flow",
},
Spec: v1beta1.SyslogNGClusterFlowSpec{
GlobalOutputRefs: []string{
"clusterout",
},
},
},
},
ClusterOutputs: nil,
SecretLoaderFactory: &TestSecretLoaderFactory{},
SourcePort: 601,
SkipInvalidResources: true,
},
wantErr: false,
wantOut: `@version: current

@include "scl.conf"

source "main_input" {
channel {
source {
network(flags("no-parse") port(601) transport("tcp"));
};
parser {
json-parser(prefix("json."));
};
};
};
`,
},
"flow with mixed valid and invalid outputs with skipInvalidResources": {
input: Input{
SyslogNGSpec: &v1beta1.SyslogNGSpec{},
Name: "test",
Flows: []v1beta1.SyslogNGFlow{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-flow-invalid",
},
Spec: v1beta1.SyslogNGFlowSpec{
GlobalOutputRefs: []string{
"nonexistent",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test-flow-valid",
},
Spec: v1beta1.SyslogNGFlowSpec{
GlobalOutputRefs: []string{
"clusterout",
},
},
},
},
ClusterOutputs: []v1beta1.SyslogNGClusterOutput{
{
ObjectMeta: metav1.ObjectMeta{
Name: "clusterout",
Namespace: "logging",
},
Spec: v1beta1.SyslogNGClusterOutputSpec{
SyslogNGOutputSpec: v1beta1.SyslogNGOutputSpec{
Syslog: &output.SyslogOutput{
Host: "127.0.0.1",
},
},
},
},
},
SecretLoaderFactory: &TestSecretLoaderFactory{},
SourcePort: 601,
SkipInvalidResources: true,
},
wantErr: false,
wantOut: `@version: current

@include "scl.conf"

source "main_input" {
channel {
source {
network(flags("no-parse") port(601) transport("tcp"));
};
parser {
json-parser(prefix("json."));
};
};
};

destination "clusteroutput_logging_clusterout" {
syslog("127.0.0.1" persist_name("clusteroutput_logging_clusterout"));
};

filter "flow_default_test-flow-valid_ns_filter" {
match("default" value("json.kubernetes.namespace_name") type("string"));
};
log {
source("main_input");
filter("flow_default_test-flow-valid_ns_filter");
log {
destination("clusteroutput_logging_clusterout");
};
};
`,
},
"parser": {
input: Input{
Name: "test",
Expand Down
2 changes: 1 addition & 1 deletion pkg/sdk/logging/model/syslogng/config/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.24.1
require (
emperror.dev/errors v0.8.1
github.com/cisco-open/operator-tools v0.37.0
github.com/go-logr/logr v1.4.3
github.com/kube-logging/logging-operator/pkg/sdk v0.12.0
github.com/siliconbrain/go-seqs v0.15.0
github.com/stretchr/testify v1.11.0
Expand All @@ -22,7 +23,6 @@ require (
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.8.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-openapi/jsonpointer v0.21.1 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.1 // indirect
Expand Down