Skip to content

Commit cebd96c

Browse files
committed
chore: clippy fmt
Signed-off-by: Elyas Mehtabuddin <[email protected]>
1 parent 0b76385 commit cebd96c

File tree

5 files changed

+197
-103
lines changed

5 files changed

+197
-103
lines changed

lib/llm/src/protocols/openai/chat_completions/aggregator.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,8 @@ mod tests {
639639
#[tokio::test]
640640
async fn test_tool_calling_finish_reason_override_from_stop() {
641641
// Test that when tool calls are present but finish reason is Stop, it gets overridden to ToolCalls
642-
let tool_call_json = r#"{"name": "get_weather", "arguments": {"location": "New York", "unit": "celsius"}}"#;
642+
let tool_call_json =
643+
r#"{"name": "get_weather", "arguments": {"location": "New York", "unit": "celsius"}}"#;
643644

644645
let annotated_delta = create_test_delta(
645646
0,
@@ -909,7 +910,7 @@ mod tests {
909910
}
910911

911912
#[tokio::test]
912-
async fn test_tool_calling_finish_reason_override_from_stop() {
913+
async fn test_tool_calling_finish_reason_override_from_stop_alternative() {
913914
// Test that when tool calls are present but finish reason is Stop, it gets overridden to ToolCalls
914915
let tool_call_json =
915916
r#"{"name": "get_weather", "arguments": {"location": "New York", "unit": "celsius"}}"#;

lib/parsers/src/tool_calling/harmony/harmony_parser.rs

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -185,25 +185,22 @@ pub fn detect_tool_call_start_harmony(
185185

186186
// Check for partial start tokens (streaming scenario)
187187
// This handles cases where start tokens are split across multiple chunks
188-
config
189-
.tool_call_start_tokens
190-
.iter()
191-
.any(|token| {
192-
if token.is_empty() {
193-
return false;
194-
}
195-
// Check if the chunk could be a prefix of this start token
196-
// Handle Unicode character boundaries properly
197-
for i in 1..=token.chars().count() {
198-
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
199-
let prefix_str = &prefix[..prefix.len()];
200-
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
201-
return true;
202-
}
188+
config.tool_call_start_tokens.iter().any(|token| {
189+
if token.is_empty() {
190+
return false;
191+
}
192+
// Check if the chunk could be a prefix of this start token
193+
// Handle Unicode character boundaries properly
194+
for i in 1..=token.chars().count() {
195+
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
196+
let prefix_str = &prefix[..prefix.len()];
197+
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
198+
return true;
203199
}
204200
}
205-
false
206-
})
201+
}
202+
false
203+
})
207204
} else {
208205
// Non-strict mode: check complete tokens and some heuristics
209206
let has_complete_token = config
@@ -216,25 +213,22 @@ pub fn detect_tool_call_start_harmony(
216213
}
217214

218215
// Check for partial start tokens or known patterns
219-
let has_partial_token = config
220-
.tool_call_start_tokens
221-
.iter()
222-
.any(|token| {
223-
if token.is_empty() {
224-
return false;
225-
}
226-
// Check if the chunk could be a prefix of this start token
227-
// Handle Unicode character boundaries properly
228-
for i in 1..=token.chars().count() {
229-
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
230-
let prefix_str = &prefix[..prefix.len()];
231-
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
232-
return true;
233-
}
216+
let has_partial_token = config.tool_call_start_tokens.iter().any(|token| {
217+
if token.is_empty() {
218+
return false;
219+
}
220+
// Check if the chunk could be a prefix of this start token
221+
// Handle Unicode character boundaries properly
222+
for i in 1..=token.chars().count() {
223+
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
224+
let prefix_str = &prefix[..prefix.len()];
225+
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
226+
return true;
234227
}
235228
}
236-
false
237-
});
229+
}
230+
false
231+
});
238232

239233
has_partial_token || trimmed.contains("<|channel|>")
240234
}
@@ -393,13 +387,31 @@ mod detect_parser_tests {
393387
};
394388

395389
// Test various partial prefixes in strict mode
396-
assert!(detect_tool_call_start_harmony("<", &config, true), "'<' should be detected as potential start");
397-
assert!(detect_tool_call_start_harmony("<|", &config, true), "'<|' should be detected as potential start");
398-
assert!(detect_tool_call_start_harmony("<|start|>", &config, true), "'<|start|>' should be detected as potential start");
399-
assert!(detect_tool_call_start_harmony("<|start|>assistant", &config, true), "'<|start|>assistant' should be detected as potential start");
390+
assert!(
391+
detect_tool_call_start_harmony("<", &config, true),
392+
"'<' should be detected as potential start"
393+
);
394+
assert!(
395+
detect_tool_call_start_harmony("<|", &config, true),
396+
"'<|' should be detected as potential start"
397+
);
398+
assert!(
399+
detect_tool_call_start_harmony("<|start|>", &config, true),
400+
"'<|start|>' should be detected as potential start"
401+
);
402+
assert!(
403+
detect_tool_call_start_harmony("<|start|>assistant", &config, true),
404+
"'<|start|>assistant' should be detected as potential start"
405+
);
400406

401407
// Test that unrelated text is not detected in strict mode
402-
assert!(!detect_tool_call_start_harmony("hello world", &config, true), "'hello world' should not be detected in strict mode");
403-
assert!(!detect_tool_call_start_harmony("xyz", &config, true), "'xyz' should not be detected in strict mode");
408+
assert!(
409+
!detect_tool_call_start_harmony("hello world", &config, true),
410+
"'hello world' should not be detected in strict mode"
411+
);
412+
assert!(
413+
!detect_tool_call_start_harmony("xyz", &config, true),
414+
"'xyz' should not be detected in strict mode"
415+
);
404416
}
405417
}

lib/parsers/src/tool_calling/json/base_json_parser.rs

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -325,26 +325,23 @@ pub fn detect_tool_call_start_basic_json(chunk: &str, config: &JsonParserConfig)
325325

326326
// Check for partial start tokens (streaming scenario)
327327
// This handles cases where start tokens are split across multiple chunks
328-
let has_partial_token = config
329-
.tool_call_start_tokens
330-
.iter()
331-
.any(|token| {
332-
if token.is_empty() {
333-
return false;
334-
}
335-
// Check if the chunk could be a prefix of this start token
336-
// We need to be careful to avoid false positives
337-
// Handle Unicode character boundaries properly
338-
for i in 1..=token.chars().count() {
339-
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
340-
let prefix_str = &prefix[..prefix.len()];
341-
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
342-
return true;
343-
}
328+
let has_partial_token = config.tool_call_start_tokens.iter().any(|token| {
329+
if token.is_empty() {
330+
return false;
331+
}
332+
// Check if the chunk could be a prefix of this start token
333+
// We need to be careful to avoid false positives
334+
// Handle Unicode character boundaries properly
335+
for i in 1..=token.chars().count() {
336+
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
337+
let prefix_str = &prefix[..prefix.len()];
338+
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
339+
return true;
344340
}
345341
}
346-
false
347-
});
342+
}
343+
false
344+
});
348345

349346
has_partial_token || trimmed.contains('{') || trimmed.contains('[')
350347
}
@@ -475,7 +472,10 @@ mod detect_parser_tests {
475472
..Default::default()
476473
};
477474
let result = detect_tool_call_start_basic_json(text, &config);
478-
assert!(result, "Should detect 'fun' as potential start of 'functools'");
475+
assert!(
476+
result,
477+
"Should detect 'fun' as potential start of 'functools'"
478+
);
479479
}
480480

481481
#[test]
@@ -487,7 +487,10 @@ mod detect_parser_tests {
487487
..Default::default()
488488
};
489489
let result = detect_tool_call_start_basic_json(text, &config);
490-
assert!(result, "Should detect 'func' as potential start of 'functools'");
490+
assert!(
491+
result,
492+
"Should detect 'func' as potential start of 'functools'"
493+
);
491494
}
492495

493496
#[test]
@@ -499,7 +502,10 @@ mod detect_parser_tests {
499502
..Default::default()
500503
};
501504
let result = detect_tool_call_start_basic_json(text, &config);
502-
assert!(result, "Should detect 'f' as potential start of 'functools'");
505+
assert!(
506+
result,
507+
"Should detect 'f' as potential start of 'functools'"
508+
);
503509
}
504510

505511
#[test]
@@ -512,7 +518,10 @@ mod detect_parser_tests {
512518
..Default::default()
513519
};
514520
let result = detect_tool_call_start_basic_json(text, &config);
515-
assert!(result, "Should detect text ending with 'fun' as potential tool call start");
521+
assert!(
522+
result,
523+
"Should detect text ending with 'fun' as potential tool call start"
524+
);
516525
}
517526

518527
#[test]
@@ -540,6 +549,9 @@ mod detect_parser_tests {
540549
..Default::default()
541550
};
542551
let result = detect_tool_call_start_basic_json(text, &config);
543-
assert!(!result, "Should not detect unrelated text as tool call start");
552+
assert!(
553+
!result,
554+
"Should not detect unrelated text as tool call start"
555+
);
544556
}
545557
}

lib/parsers/src/tool_calling/json/deepseek_parser.rs

Lines changed: 38 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -119,25 +119,22 @@ pub fn detect_tool_call_start_deepseek_v3_1(chunk: &str, config: &JsonParserConf
119119

120120
// Check for partial start tokens (streaming scenario)
121121
// This handles cases where start tokens are split across multiple chunks
122-
config
123-
.tool_call_start_tokens
124-
.iter()
125-
.any(|token| {
126-
if token.is_empty() {
127-
return false;
128-
}
129-
// Check if the chunk could be a prefix of this start token
130-
// Handle Unicode character boundaries properly
131-
for i in 1..=token.chars().count() {
132-
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
133-
let prefix_str = &prefix[..prefix.len()];
134-
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
135-
return true;
136-
}
122+
config.tool_call_start_tokens.iter().any(|token| {
123+
if token.is_empty() {
124+
return false;
125+
}
126+
// Check if the chunk could be a prefix of this start token
127+
// Handle Unicode character boundaries properly
128+
for i in 1..=token.chars().count() {
129+
if let Some(prefix) = token.chars().take(i).collect::<String>().get(..) {
130+
let prefix_str = &prefix[..prefix.len()];
131+
if trimmed == prefix_str || trimmed.ends_with(prefix_str) {
132+
return true;
137133
}
138134
}
139-
false
140-
})
135+
}
136+
false
137+
})
141138
}
142139

143140
#[cfg(test)]
@@ -304,13 +301,31 @@ mod detect_parser_tests {
304301
};
305302

306303
// Test various partial prefixes
307-
assert!(detect_tool_call_start_deepseek_v3_1("<", &config), "'<' should be detected as potential start");
308-
assert!(detect_tool_call_start_deepseek_v3_1("<|", &config), "'<|' should be detected as potential start");
309-
assert!(detect_tool_call_start_deepseek_v3_1("<|tool", &config), "'<|tool' should be detected as potential start");
310-
assert!(detect_tool_call_start_deepseek_v3_1("<|tool▁calls", &config), "'<|tool▁calls' should be detected as potential start");
304+
assert!(
305+
detect_tool_call_start_deepseek_v3_1("<", &config),
306+
"'<' should be detected as potential start"
307+
);
308+
assert!(
309+
detect_tool_call_start_deepseek_v3_1("<|", &config),
310+
"'<|' should be detected as potential start"
311+
);
312+
assert!(
313+
detect_tool_call_start_deepseek_v3_1("<|tool", &config),
314+
"'<|tool' should be detected as potential start"
315+
);
316+
assert!(
317+
detect_tool_call_start_deepseek_v3_1("<|tool▁calls", &config),
318+
"'<|tool▁calls' should be detected as potential start"
319+
);
311320

312321
// Test that unrelated text is not detected
313-
assert!(!detect_tool_call_start_deepseek_v3_1("hello world", &config), "'hello world' should not be detected");
314-
assert!(!detect_tool_call_start_deepseek_v3_1("xyz", &config), "'xyz' should not be detected");
322+
assert!(
323+
!detect_tool_call_start_deepseek_v3_1("hello world", &config),
324+
"'hello world' should not be detected"
325+
);
326+
assert!(
327+
!detect_tool_call_start_deepseek_v3_1("xyz", &config),
328+
"'xyz' should not be detected"
329+
);
315330
}
316331
}

0 commit comments

Comments
 (0)