Skip to content

Commit 1fed2c7

Browse files
DanielRosenwasserDanielRosenwasser
authored
More Python translator fixes (#246)
* Add tests for preamble. * Update snapshots. * Fix translator - correct ordering and include assistant response on recovery. * Update snapshots. * Catch JSON parse errors. * Add test for invalid JSON responses. * Copy lists from the translator so that output is correctly snapshotted/not duplicated. * Updated snapshot. * Flatten string, give more context. * Update snapshots.
1 parent 9f644b6 commit 1fed2c7

File tree

3 files changed

+308
-14
lines changed

3 files changed

+308
-14
lines changed

‎python/src/typechat/_internal/translator.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,10 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti
6464

6565
messages: list[PromptSection] = []
6666

67-
messages.append({"role": "user", "content": input})
6867
if prompt_preamble:
6968
if isinstance(prompt_preamble, str):
7069
prompt_preamble = [{"role": "user", "content": prompt_preamble}]
71-
else:
72-
messages.extend(prompt_preamble)
70+
messages.extend(prompt_preamble)
7371

7472
messages.append({"role": "user", "content": self._create_request_prompt(input)})
7573

@@ -85,16 +83,21 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti
8583
error_message: str
8684
if 0 <= first_curly < last_curly:
8785
trimmed_response = text_response[first_curly:last_curly]
88-
parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False)
89-
result = self.validator.validate_object(parsed_response)
90-
if isinstance(result, Success):
91-
return result
92-
error_message = result.message
86+
try:
87+
parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False)
88+
except ValueError as e:
89+
error_message = f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}"
90+
else:
91+
result = self.validator.validate_object(parsed_response)
92+
if isinstance(result, Success):
93+
return result
94+
error_message = result.message
9395
else:
94-
error_message = "Response did not contain any text resembling JSON."
96+
error_message = f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}"
9597
if num_repairs_attempted >= self._max_repair_attempts:
9698
return Failure(error_message)
9799
num_repairs_attempted += 1
100+
messages.append({"role": "assistant", "content": text_response})
98101
messages.append({"role": "user", "content": self._create_repair_prompt(error_message)})
99102

100103
def _create_request_prompt(self, intent: str) -> str:

‎python/tests/__snapshots__/test_translator.ambr‎

Lines changed: 253 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,39 @@
55
'kind': 'CLIENT REQUEST',
66
'payload': list([
77
dict({
8-
'content': 'Get me stuff.',
8+
'content': '''
9+
10+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
11+
```
12+
interface ExampleABC {
13+
a: string;
14+
b: boolean;
15+
c: number;
16+
}
17+
18+
```
19+
The following is a user request:
20+
'''
21+
Get me stuff.
22+
'''
23+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
24+
25+
''',
926
'role': 'user',
1027
}),
28+
]),
29+
}),
30+
dict({
31+
'kind': 'MODEL RESPONSE',
32+
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
33+
}),
34+
])
35+
# ---
36+
# name: test_translator_with_invalid_json
37+
list([
38+
dict({
39+
'kind': 'CLIENT REQUEST',
40+
'payload': list([
1141
dict({
1242
'content': '''
1343

@@ -33,7 +63,57 @@
3363
}),
3464
dict({
3565
'kind': 'MODEL RESPONSE',
36-
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
66+
'payload': '{ "a": "hello" "b": true }',
67+
}),
68+
dict({
69+
'kind': 'CLIENT REQUEST',
70+
'payload': list([
71+
dict({
72+
'content': '''
73+
74+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
75+
```
76+
interface ExampleABC {
77+
a: string;
78+
b: boolean;
79+
c: number;
80+
}
81+
82+
```
83+
The following is a user request:
84+
'''
85+
Get me stuff.
86+
'''
87+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
88+
89+
''',
90+
'role': 'user',
91+
}),
92+
dict({
93+
'content': '{ "a": "hello" "b": true }',
94+
'role': 'assistant',
95+
}),
96+
dict({
97+
'content': '''
98+
99+
The above JSON object is invalid for the following reason:
100+
'''
101+
Error: expected `,` or `}` at line 1 column 16
102+
103+
Attempted to parse:
104+
105+
{ "a": "hello" "b": true }
106+
'''
107+
The following is a revised JSON object:
108+
109+
''',
110+
'role': 'user',
111+
}),
112+
]),
113+
}),
114+
dict({
115+
'kind': 'MODEL RESPONSE',
116+
'payload': '{ "a": "hello" "b": true, "c": 1234 }',
37117
}),
38118
])
39119
# ---
@@ -43,9 +123,94 @@
43123
'kind': 'CLIENT REQUEST',
44124
'payload': list([
45125
dict({
46-
'content': 'Get me stuff.',
126+
'content': '''
127+
128+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
129+
```
130+
interface ExampleABC {
131+
a: string;
132+
b: boolean;
133+
c: number;
134+
}
135+
136+
```
137+
The following is a user request:
138+
'''
139+
Get me stuff.
140+
'''
141+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
142+
143+
''',
144+
'role': 'user',
145+
}),
146+
]),
147+
}),
148+
dict({
149+
'kind': 'MODEL RESPONSE',
150+
'payload': '{ "a": "hello", "b": true }',
151+
}),
152+
dict({
153+
'kind': 'CLIENT REQUEST',
154+
'payload': list([
155+
dict({
156+
'content': '''
157+
158+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
159+
```
160+
interface ExampleABC {
161+
a: string;
162+
b: boolean;
163+
c: number;
164+
}
165+
166+
```
167+
The following is a user request:
168+
'''
169+
Get me stuff.
170+
'''
171+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
172+
173+
''',
174+
'role': 'user',
175+
}),
176+
dict({
177+
'content': '{ "a": "hello", "b": true }',
178+
'role': 'assistant',
179+
}),
180+
dict({
181+
'content': '''
182+
183+
The above JSON object is invalid for the following reason:
184+
'''
185+
Validation path `c` failed for value `{"a": "hello", "b": true}` because:
186+
Field required
187+
'''
188+
The following is a revised JSON object:
189+
190+
''',
191+
'role': 'user',
192+
}),
193+
]),
194+
}),
195+
dict({
196+
'kind': 'MODEL RESPONSE',
197+
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
198+
}),
199+
])
200+
# ---
201+
# name: test_translator_with_single_failure_and_list_preamble_1
202+
list([
203+
dict({
204+
'kind': 'CLIENT REQUEST',
205+
'payload': list([
206+
dict({
207+
'content': 'Hey, I need some stuff.',
47208
'role': 'user',
48209
}),
210+
dict({
211+
'content': 'Okay, what kind of stuff?',
212+
'role': 'assistant',
213+
}),
49214
dict({
50215
'content': '''
51216

@@ -67,6 +232,48 @@
67232
''',
68233
'role': 'user',
69234
}),
235+
]),
236+
}),
237+
dict({
238+
'kind': 'MODEL RESPONSE',
239+
'payload': '{ "a": "hello", "b": true }',
240+
}),
241+
dict({
242+
'kind': 'CLIENT REQUEST',
243+
'payload': list([
244+
dict({
245+
'content': 'Hey, I need some stuff.',
246+
'role': 'user',
247+
}),
248+
dict({
249+
'content': 'Okay, what kind of stuff?',
250+
'role': 'assistant',
251+
}),
252+
dict({
253+
'content': '''
254+
255+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
256+
```
257+
interface ExampleABC {
258+
a: string;
259+
b: boolean;
260+
c: number;
261+
}
262+
263+
```
264+
The following is a user request:
265+
'''
266+
Get me stuff.
267+
'''
268+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
269+
270+
''',
271+
'role': 'user',
272+
}),
273+
dict({
274+
'content': '{ "a": "hello", "b": true }',
275+
'role': 'assistant',
276+
}),
70277
dict({
71278
'content': '''
72279

@@ -82,6 +289,44 @@
82289
}),
83290
]),
84291
}),
292+
dict({
293+
'kind': 'MODEL RESPONSE',
294+
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
295+
}),
296+
])
297+
# ---
298+
# name: test_translator_with_single_failure_and_str_preamble
299+
list([
300+
dict({
301+
'kind': 'CLIENT REQUEST',
302+
'payload': list([
303+
dict({
304+
'content': 'Just so you know, I need some stuff.',
305+
'role': 'user',
306+
}),
307+
dict({
308+
'content': '''
309+
310+
You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
311+
```
312+
interface ExampleABC {
313+
a: string;
314+
b: boolean;
315+
c: number;
316+
}
317+
318+
```
319+
The following is a user request:
320+
'''
321+
Get me stuff.
322+
'''
323+
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:
324+
325+
''',
326+
'role': 'user',
327+
}),
328+
]),
329+
}),
85330
dict({
86331
'kind': 'MODEL RESPONSE',
87332
'payload': '{ "a": "hello", "b": true }',
@@ -90,7 +335,7 @@
90335
'kind': 'CLIENT REQUEST',
91336
'payload': list([
92337
dict({
93-
'content': 'Get me stuff.',
338+
'content': 'Just so you know, I need some stuff.',
94339
'role': 'user',
95340
}),
96341
dict({
@@ -114,6 +359,10 @@
114359
''',
115360
'role': 'user',
116361
}),
362+
dict({
363+
'content': '{ "a": "hello", "b": true }',
364+
'role': 'assistant',
365+
}),
117366
dict({
118367
'content': '''
119368

0 commit comments

Comments
 (0)