Skip to content

Commit 3e4261f

Browse files
fmassotfmassot
committed
Remove elasticsearch fields.
1 parent 9ab18c2 commit 3e4261f

File tree

3 files changed

+16
-115
lines changed

3 files changed

+16
-115
lines changed

‎pkg/quickwit/data_query.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ func processLogsQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defa
345345
func processDocumentQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defaultTimeField string) {
346346
metric := q.Metrics[0]
347347
b.Sort(es.SortOrderDesc, defaultTimeField, "boolean")
348-
b.Sort(es.SortOrderDesc, "_doc", "")
348+
// b.Sort(es.SortOrderDesc, "_doc", "")
349349
// Note: not supported in Quickwit
350350
// b.AddDocValueField(defaultTimeField)
351351
b.Size(stringToIntWithDefaultValue(metric.Settings.Get("size").MustString(), defaultSize))

‎pkg/quickwit/response_parser.go‎

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,7 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields
107107
}
108108

109109
doc := map[string]interface{}{
110-
"_id": hit["_id"],
111-
"_type": hit["_type"],
112-
"_index": hit["_index"],
113-
"sort": hit["sort"],
114-
"highlight": hit["highlight"],
115-
"_source": flattened,
110+
"_source": flattened,
116111
}
117112

118113
for k, v := range flattened {
@@ -179,13 +174,7 @@ func processRawDataResponse(res *es.SearchResponse, target *Query, configuredFie
179174
flattened = flatten(hit["_source"].(map[string]interface{}))
180175
}
181176

182-
doc := map[string]interface{}{
183-
"_id": hit["_id"],
184-
"_type": hit["_type"],
185-
"_index": hit["_index"],
186-
"sort": hit["sort"],
187-
"highlight": hit["highlight"],
188-
}
177+
doc := map[string]interface{}{}
189178

190179
for k, v := range flattened {
191180
doc[k] = v
@@ -212,13 +201,7 @@ func processRawDataResponse(res *es.SearchResponse, target *Query, configuredFie
212201
func processRawDocumentResponse(res *es.SearchResponse, target *Query, queryRes *backend.DataResponse) error {
213202
docs := make([]map[string]interface{}, len(res.Hits.Hits))
214203
for hitIdx, hit := range res.Hits.Hits {
215-
doc := map[string]interface{}{
216-
"_id": hit["_id"],
217-
"_type": hit["_type"],
218-
"_index": hit["_index"],
219-
"sort": hit["sort"],
220-
"highlight": hit["highlight"],
221-
}
204+
doc := map[string]interface{}{}
222205

223206
if hit["_source"] != nil {
224207
source, ok := hit["_source"].(map[string]interface{})

‎pkg/quickwit/response_parser_test.go‎

Lines changed: 12 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@ func TestProcessLogsResponse(t *testing.T) {
9999
logsFrame := frames[0]
100100

101101
meta := logsFrame.Meta
102-
require.Equal(t, map[string]interface{}{"searchWords": []string{"hello", "message"}, "limit": 100}, meta.Custom)
103102
require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization))
104103

105104
logsFieldMap := make(map[string]*data.Field)
@@ -122,13 +121,6 @@ func TestProcessLogsResponse(t *testing.T) {
122121
require.Contains(t, logsFieldMap, "_source")
123122
require.Equal(t, data.FieldTypeNullableJSON, logsFieldMap["_source"].Type())
124123

125-
requireStringAt(t, "fdsfs", logsFieldMap["_id"], 0)
126-
requireStringAt(t, "kdospaidopa", logsFieldMap["_id"], 1)
127-
requireStringAt(t, "_doc", logsFieldMap["_type"], 0)
128-
requireStringAt(t, "_doc", logsFieldMap["_type"], 1)
129-
requireStringAt(t, "mock-index", logsFieldMap["_index"], 0)
130-
requireStringAt(t, "mock-index", logsFieldMap["_index"], 1)
131-
132124
actualJson1, err := json.Marshal(logsFieldMap["_source"].At(0).(*json.RawMessage))
133125
require.NoError(t, err)
134126
actualJson2, err := json.Marshal(logsFieldMap["_source"].At(1).(*json.RawMessage))
@@ -312,7 +304,7 @@ func TestProcessLogsResponse(t *testing.T) {
312304
require.Len(t, dataframes, 1)
313305
frame := dataframes[0]
314306

315-
require.Equal(t, 16, len(frame.Fields))
307+
require.Equal(t, 11, len(frame.Fields))
316308
// Fields have the correct length
317309
require.Equal(t, 2, frame.Fields[0].Len())
318310
// First field is timeField
@@ -321,90 +313,18 @@ func TestProcessLogsResponse(t *testing.T) {
321313
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
322314
require.Equal(t, "line", frame.Fields[1].Name)
323315
// Correctly renames lvl field to level
324-
require.Equal(t, "level", frame.Fields[10].Name)
316+
require.Equal(t, "level", frame.Fields[6].Name)
325317
// Correctly uses string types
326318
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
327319
// Correctly detects float64 types
328-
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[7].Type())
320+
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[4].Type())
329321
// Correctly detects json types
330-
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[8].Type())
322+
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[2].Type())
331323
// Correctly flattens fields
332-
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
333-
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
324+
require.Equal(t, "nested.field.double_nested", frame.Fields[8].Name)
325+
require.Equal(t, data.FieldTypeNullableString, frame.Fields[8].Type())
334326
// Correctly detects type even if first value is null
335-
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
336-
})
337-
338-
t.Run("Log query with highlight", func(t *testing.T) {
339-
targets := map[string]string{
340-
"A": `{
341-
"metrics": [{ "type": "logs" }]
342-
}`,
343-
}
344-
345-
response := `{
346-
"responses":[
347-
{
348-
"hits":{
349-
"total":{
350-
"value":109,
351-
"relation":"eq"
352-
},
353-
"max_score":null,
354-
"hits":[
355-
{
356-
"_index":"logs-2023.02.08",
357-
"_id":"GB2UMYYBfCQ-FCMjayJa",
358-
"_score":null,
359-
"highlight": {
360-
"line": [
361-
"@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"
362-
],
363-
"duplicated": ["@HIGHLIGHT@hello@/HIGHLIGHT@"]
364-
},
365-
"_source":{
366-
"@timestamp":"2023-02-08T15:10:55.830Z",
367-
"line":"log text [479231733]"
368-
}
369-
},
370-
{
371-
"_index":"logs-2023.02.08",
372-
"_id":"GB2UMYYBfCQ-FCMjayJa",
373-
"_score":null,
374-
"highlight": {
375-
"line": [
376-
"@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"
377-
],
378-
"duplicated": ["@HIGHLIGHT@hello@/HIGHLIGHT@"]
379-
},
380-
"_source":{
381-
"@timestamp":"2023-02-08T15:10:55.830Z",
382-
"line":"log text [479231733]"
383-
}
384-
}
385-
]
386-
},
387-
"status":200
388-
}
389-
]
390-
}`
391-
392-
result, err := parseTestResponse(targets, response)
393-
require.NoError(t, err)
394-
require.Len(t, result.Responses, 1)
395-
396-
queryRes := result.Responses["A"]
397-
require.NotNil(t, queryRes)
398-
dataframes := queryRes.Frames
399-
require.Len(t, dataframes, 1)
400-
frame := dataframes[0]
401-
402-
customMeta := frame.Meta.Custom
403-
404-
require.Equal(t, map[string]interface{}{
405-
"searchWords": []string{"hello", "message"},
406-
"limit": 100,
407-
}, customMeta)
327+
require.Equal(t, data.FieldTypeNullableString, frame.Fields[10].Type())
408328
})
409329
}
410330

@@ -507,22 +427,20 @@ func TestProcessRawDataResponse(t *testing.T) {
507427
require.Len(t, dataframes, 1)
508428
frame := dataframes[0]
509429

510-
require.Equal(t, 15, len(frame.Fields))
430+
require.Equal(t, 10, len(frame.Fields))
511431
// Fields have the correct length
512432
require.Equal(t, 2, frame.Fields[0].Len())
513433
// First field is timeField
514434
require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type())
515435
// Correctly uses string types
516436
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
517437
// Correctly detects float64 types
518-
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[5].Type())
519-
// Correctly detects json types
520-
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[6].Type())
438+
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[2].Type())
521439
// Correctly flattens fields
522-
require.Equal(t, "nested.field.double_nested", frame.Fields[11].Name)
523-
require.Equal(t, data.FieldTypeNullableString, frame.Fields[11].Type())
440+
require.Equal(t, "nested.field.double_nested", frame.Fields[7].Name)
441+
require.Equal(t, data.FieldTypeNullableString, frame.Fields[7].Type())
524442
// Correctly detects type even if first value is null
525-
require.Equal(t, data.FieldTypeNullableString, frame.Fields[14].Type())
443+
require.Equal(t, data.FieldTypeNullableString, frame.Fields[9].Type())
526444
})
527445

528446
t.Run("Raw data query filterable fields", func(t *testing.T) {

0 commit comments

Comments
 (0)