Skip to content

Commit 129e378

Browse files
howardjohnhowardjohn
committed
More robust parsing
1 parent 8214ef0 commit 129e378

File tree

2 files changed

+273
-28
lines changed

2 files changed

+273
-28
lines changed

‎pkg/decode.go‎

Lines changed: 53 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -42,39 +42,47 @@ func NewYAMLReader(r *bufio.Reader) *YAMLReader {
4242
}
4343
}
4444

45-
// Read returns a full YAML document.
4645
func (r *YAMLReader) Read() ([]byte, error) {
46+
if !r.isList {
47+
return r.readFlat()
48+
}
49+
firstLine := true
4750
var buffer bytes.Buffer
48-
listDiscard := make([]byte, 2, 2)
49-
firstLoop := true
5051
for {
51-
if r.isList {
52-
if !firstLoop {
53-
rr, err := r.reader.reader.Peek(1)
54-
if err != nil {
52+
start := r.reader.StartsList()
53+
if start && !firstLine {
54+
return buffer.Bytes(), nil
55+
}
56+
firstLine = false
57+
l, err := r.reader.Read()
58+
if err != nil {
59+
return nil, err
60+
}
61+
switch len(l) {
62+
case 0: // can have empty lines from YAMLs with newline in string.
63+
return nil, fmt.Errorf("invalid line: %q", string(l))
64+
case 1:
65+
if l[0] != '\n' {
66+
// Should not happen
67+
return nil, fmt.Errorf("invalid line: %q", string(l))
68+
}
69+
default: // Trim the start
70+
if l[0] == 'k' && l[1] == 'i' {
71+
if _, err := io.Copy(io.Discard, r.reader.reader); err != nil {
5572
return nil, err
5673
}
57-
if rr[0] == '-' {
58-
// We hit the next entry
59-
return buffer.Bytes(), nil
60-
}
61-
if rr[0] != ' ' && rr[0] != '\n' {
62-
// TODO: match 'kind: List' exactly instead
63-
// Not part of the list anymore, just be end of the list
64-
// Drain the list so we don't read more
65-
if _, err := io.Copy(io.Discard, r.reader.reader); err != nil {
66-
return nil, err
67-
}
68-
return buffer.Bytes(), nil
69-
}
70-
}
71-
_, err := io.ReadFull(r.reader.reader, listDiscard)
72-
if err != nil {
73-
return nil, err
74+
// End of list. TODO: more robust check
75+
return buffer.Bytes(), err
7476
}
75-
firstLoop = false
77+
l = l[2:]
7678
}
79+
buffer.Write(l)
80+
}
81+
}
7782

83+
func (r *YAMLReader) readFlat() ([]byte, error) {
84+
var buffer bytes.Buffer
85+
for {
7886
line, err := r.reader.Read()
7987
if err != nil && err != io.EOF {
8088
return nil, err
@@ -108,16 +116,33 @@ func (r *YAMLReader) Read() ([]byte, error) {
108116
}
109117

110118
type LineReader struct {
111-
reader *bufio.Reader
119+
reader *bufio.Reader
120+
nextLine []byte
112121
}
113122

114123
func (r *LineReader) Peak(n int) ([]byte, error) {
115124
return r.reader.Peek(n)
116125
}
117126

118-
// Read returns a single line (with '\n' ended) from the underlying reader.
119-
// An error is returned iff there is an error with the underlying reader.
127+
// StartsList checks if the next line starts a list
128+
func (r *LineReader) StartsList() bool {
129+
if r.nextLine != nil {
130+
return r.nextLine[0] == '-' && r.nextLine[1] == ' '
131+
}
132+
l, err := r.Read()
133+
if err != nil {
134+
return false
135+
}
136+
r.nextLine = l
137+
return r.nextLine[0] == '-' && r.nextLine[1] == ' '
138+
}
139+
120140
func (r *LineReader) Read() ([]byte, error) {
141+
if r.nextLine != nil {
142+
res := r.nextLine
143+
r.nextLine = nil
144+
return res, nil
145+
}
121146
var (
122147
isPrefix bool = true
123148
err error = nil

‎pkg/decode_test.go‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
package pkg
2+
3+
import (
4+
"bytes"
5+
"reflect"
6+
"strings"
7+
"testing"
8+
)
9+
10+
func TestYAMLReader(t *testing.T) {
11+
tests := []struct {
12+
name string
13+
input string
14+
want []string
15+
}{
16+
{
17+
name: "empty",
18+
input: "",
19+
want: []string{},
20+
},
21+
{
22+
name: "list with newlines in configmap",
23+
input: `apiVersion: v1
24+
items:
25+
- apiVersion: v1
26+
data:
27+
status: |+
28+
Cluster-autoscaler status at 2021-11-10 00:01:27.986516659 +0000 UTC:
29+
Cluster-wide:
30+
Health: Healthy (ready=6 unready=0 notStarted=0 longNotStarted=0 registered=6 longUnregistered=0)
31+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
32+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
33+
ScaleUp: NoActivity (ready=6 registered=6)
34+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
35+
LastTransitionTime: 2021-11-09 08:49:17.188499976 +0000 UTC m=+1222348.789143150
36+
ScaleDown: NoCandidates (candidates=0)
37+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
38+
LastTransitionTime: 2021-11-09 08:53:30.879288373 +0000 UTC m=+1222602.479931549
39+
40+
NodeGroups:
41+
Name: obscured
42+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
43+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
44+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
45+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
46+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
47+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
48+
ScaleDown: NoCandidates (candidates=0)
49+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
50+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
51+
52+
Name: obscured
53+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
54+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
55+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
56+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
57+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
58+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
59+
ScaleDown: NoCandidates (candidates=0)
60+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
61+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
62+
63+
Name: obscured
64+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
65+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
66+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
67+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
68+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
69+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
70+
ScaleDown: NoCandidates (candidates=0)
71+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
72+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
73+
74+
Name: obscured
75+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
76+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
77+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
78+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
79+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
80+
LastTransitionTime: 2021-11-09 08:49:17.188499976 +0000 UTC m=+1222348.789143150
81+
ScaleDown: NoCandidates (candidates=0)
82+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
83+
LastTransitionTime: 2021-11-09 08:53:30.879288373 +0000 UTC m=+1222602.479931549
84+
85+
Name: obscured
86+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
87+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
88+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
89+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
90+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
91+
LastTransitionTime: 2021-11-07 00:10:41.430022289 +0000 UTC m=+1018433.030665467
92+
ScaleDown: NoCandidates (candidates=0)
93+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
94+
LastTransitionTime: 2021-11-07 00:12:53.55169393 +0000 UTC m=+1018565.152337097
95+
96+
Name: obscure
97+
Health: Healthy (ready=1 unready=0 notStarted=0 longNotStarted=0 registered=1 longUnregistered=0 cloudProviderTarget=1 (minSize=1, maxSize=100))
98+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
99+
LastTransitionTime: 2021-10-26 05:20:47.050658932 +0000 UTC m=+238.651302118
100+
ScaleUp: NoActivity (ready=1 cloudProviderTarget=1)
101+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
102+
LastTransitionTime: 2021-11-08 08:48:47.137796002 +0000 UTC m=+1135918.738439166
103+
ScaleDown: NoCandidates (candidates=0)
104+
LastProbeTime: 2021-11-10 00:01:27.486224515 +0000 UTC m=+1277079.086867680
105+
LastTransitionTime: 2021-11-08 08:49:36.858799215 +0000 UTC m=+1135968.459442400
106+
107+
kind: ConfigMap
108+
metadata:
109+
annotations:
110+
cluster-autoscaler.kubernetes.io/last-updated: 2021-11-10 00:01:27.986516659
111+
+0000 UTC
112+
creationTimestamp: "2021-05-04T15:14:36Z"
113+
managedFields:
114+
- apiVersion: v1
115+
fieldsType: FieldsV1
116+
fieldsV1:
117+
f:data:
118+
.: {}
119+
f:status: {}
120+
f:metadata:
121+
f:annotations:
122+
.: {}
123+
f:cluster-autoscaler.kubernetes.io/last-updated: {}
124+
manager: cluster-autoscaler
125+
operation: Update
126+
time: "2021-05-04T15:14:36Z"
127+
name: cluster-autoscaler-status
128+
namespace: kube-system
129+
- apiVersion: v1
130+
data:
131+
root-cert.pem: |
132+
-----BEGIN CERTIFICATE-----
133+
MIIC/DCCAeSgAwIBAgIQYfZd7aU845ZUCPqVw6YoyDANBgkqhkiG9w0BAQsFADAY
134+
MRYwFAYDVQQKEw1jbHVzdGVyLmxvY2FsMB4XDTIxMTExMjAxMzkzOFoXDTMxMTEx
135+
MDAxMzkzOFowGDEWMBQGA1UEChMNY2x1c3Rlci5sb2NhbDCCASIwDQYJKoZIhvcN
136+
AQEBBQADggEPADCCAQoCggEBAMcJzGjz5YNyf2r/O683FjNsFtxGnQgKJVxIpakq
137+
gJziiJh6mfAI+piqfkJUDZoq2tGwnFHEpLil7Vb016ubdez7Vd3lbMYyY8Gn0JU2
138+
RDaP+NNhj4KwQv2QN+ebz3NVwdnJUog7aD7E7cI+CBHJ2q3z+UQCSQ2NjZPTqxI5
139+
xl9AIz5MQN9kjMG6oooexmKNqzW93CxQfceNrZWehibSzKQCLrKLhPa2OAMP6aqt
140+
YINBLz5/y3QvN9qectAWCEgU6+HOtCJKAc4h3ZjZwLjq5yxOPJ0K2ceNx/sGk5ya
141+
bc4gIROeiIfR8QauaEeDzwDmtPoX8YnMdRWtiEfpUS5nqdsCAwEAAaNCMEAwDgYD
142+
VR0PAQH/BAQDAgIEMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFL9uVSgnggVI
143+
+qdB/E/bMehwNerLMA0GCSqGSIb3DQEBCwUAA4IBAQC9OVksSsuVMaR5azJfrK3e
144+
L5Kl9Rx1T5+VxZ2fxxCmf3wyPmc9VjDnsb+FtmjQDs7IAt7/hxCHys7mC8ZBCLbf
145+
dOu91TuHzGmGP0NBOmWBFgeRSHnIwYB+mKAtTtlOpZwGdJhjvKcC2eSR2oou6I9j
146+
SZiVkv6FcEvGdzcYtqw5mulX9jM+zLMl76Bsm4I8ZvhmzUPwmUqaWpz3tQ88IrHc
147+
xOZSGJ13nTmIgaWrzRS5mxerY4wRSTOLPC8FvWZVToVoM8ZuMfM5pMhalb764E6q
148+
78UTleJ3XpFojiRqsFc20WuHAjtfwgz4vktmXo8Hck5UJs/f1aKfKIZgPAVSXaVj
149+
-----END CERTIFICATE-----
150+
kind: ConfigMap
151+
metadata:
152+
creationTimestamp: "2021-11-12T01:39:39Z"
153+
labels:
154+
istio.io/config: "true"
155+
name: istio-ca-root-cert
156+
namespace: default
157+
resourceVersion: "5288"
158+
uid: 992bb94c-c1c9-404e-bdec-87e62b853b79
159+
- apiVersion: v1
160+
data:
161+
ca.crt: |
162+
-----BEGIN CERTIFICATE-----
163+
MIIC/jCCAeagAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
164+
cm5ldGVzMB4XDTIxMTExMjAwNTQyNVoXDTMxMTExMDAwNTQyNVowFTETMBEGA1UE
165+
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL5H
166+
/7DMS34Un4AB/b1Ykh8EzIfc3vawaz0vcSXhxBazrwLaVBp1CjOZf4zPzWOEQ6a5
167+
s3gYMa16n05dN1LJNKBxOvmEetuD54pqwpXVXuCN+wt2S3DsssKOresYAcJPf7Qf
168+
YLVrQ5ZPftj5utN+0xcaSl7YC+bAtVnu/i+ZXPrGS/0uGOAL2gCZs1xcazmOzSwK
169+
HvR78b4vu8ch29huehIwT8XPzfH5EZyd2jgaLwhQUlzm182z1IiwGM9YGXhlSAtn
170+
ksrlwctf+ARbYKV4+OS9YZ36qLoqktQemMur18t67iuWiomeoaUJ3skPijotl5Kk
171+
dwSdFKn5OWilnI+104UCAwEAAaNZMFcwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
172+
/wQFMAMBAf8wHQYDVR0OBBYEFKocHx1yer/illx/cr4yJ2afUExQMBUGA1UdEQQO
173+
MAyCCmt1YmVybmV0ZXMwDQYJKoZIhvcNAQELBQADggEBABLoNrCvqHb32PoZra4Z
174+
qznXiIJd4s5WfQudtLypdnXX9NvE/uOASLCxc3hjpTzii6wiKLRThLq+CaznNwi0
175+
gVNkPielMhEmkD8x/NxSKRlOr94pwxmFGNWNpjkkuLaxQRUlLI1qDa/tio4Fo7YE
176+
RahAw0Aa1Mwh/+49xPFpFk+RYOEBssTOOzBi69mhaGbfGcA3YtLZ3R6liqmVqKhD
177+
ovtWj0CWyJUB7rLKKhwzYvv6MrYZ87WYeT8zTHq8L8EhbdsDOvuFs/IK2BbQdyd2
178+
7aXsHT08dc1QCH3rP2bdkehcLtGXSP++7gJNXxL280EnyWFt+KFkikbLLI/s+oVE
179+
1gc=
180+
-----END CERTIFICATE-----
181+
kind: ConfigMap
182+
metadata:
183+
annotations:
184+
kubernetes.io/description: Contains a CA bundle that can be used to verify the
185+
kube-apiserver when using internal endpoints such as the internal service
186+
IP or kubernetes.default.svc. No other usage is guaranteed across distributions
187+
of Kubernetes clusters.
188+
creationTimestamp: "2021-11-12T00:54:57Z"
189+
name: kube-root-ca.crt
190+
namespace: default
191+
resourceVersion: "422"
192+
uid: 193862c7-8b99-4838-889e-9e1e3a15c930
193+
kind: List
194+
metadata:
195+
resourceVersion: ""
196+
selfLink: ""
197+
`,
198+
want: []string{
199+
"ConfigMap/cluster-autoscaler-status.kube-system",
200+
"ConfigMap/istio-ca-root-cert.default",
201+
"ConfigMap/kube-root-ca.crt.default",
202+
},
203+
},
204+
}
205+
for _, tt := range tests {
206+
t.Run(tt.name, func(t *testing.T) {
207+
o := bytes.Buffer{}
208+
if err := GrepResources(Selector{}, strings.NewReader(tt.input), &o, Summary, false); err != nil {
209+
t.Fatal(err)
210+
}
211+
l := strings.Split(o.String(), "\n")
212+
if l[len(l)-1] == "" {
213+
l = l[:len(l)-1]
214+
}
215+
if !reflect.DeepEqual(l, tt.want) {
216+
t.Errorf("got = %v, want %v", l, tt.want)
217+
}
218+
})
219+
}
220+
}

0 commit comments

Comments
 (0)