Skip to content

More Python translator fixes #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Apr 30, 2024
Merged

More Python translator fixes #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c945078
Add tests for preamble.
DanielRosenwasser Apr 30, 2024
b0a3e1f
Update snapshots.
DanielRosenwasser Apr 30, 2024
9a94431
Fix translator - correct ordering and include assistant response on r…
DanielRosenwasser Apr 30, 2024
e16eb27
Update snapshots.
DanielRosenwasser Apr 30, 2024
564b4cb
Catch JSON parse errors.
DanielRosenwasser Apr 30, 2024
0e6329f
Add test for invalid JSON responses.
DanielRosenwasser Apr 30, 2024
e943312
Copy lists from the translator so that output is correctly snapshotte…
DanielRosenwasser Apr 30, 2024
5e65614
Updated snapshot.
DanielRosenwasser Apr 30, 2024
194e3b0
Flatten string, give more context.
DanielRosenwasser Apr 30, 2024
0a8fdb7
Update snapshots.
DanielRosenwasser Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions python/src/typechat/_internal/translator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,10 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti

messages: list[PromptSection] = []

messages.append({"role": "user", "content": input})
if prompt_preamble:
if isinstance(prompt_preamble, str):
prompt_preamble = [{"role": "user", "content": prompt_preamble}]
else:
messages.extend(prompt_preamble)
messages.extend(prompt_preamble)

messages.append({"role": "user", "content": self._create_request_prompt(input)})

Expand All @@ -85,16 +83,21 @@ async def translate(self, input: str, *, prompt_preamble: str | list[PromptSecti
error_message: str
if 0 <= first_curly < last_curly:
trimmed_response = text_response[first_curly:last_curly]
parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False)
result = self.validator.validate_object(parsed_response)
if isinstance(result, Success):
return result
error_message = result.message
try:
parsed_response = pydantic_core.from_json(trimmed_response, allow_inf_nan=False, cache_strings=False)
except ValueError as e:
error_message = f"Error: {e}\n\nAttempted to parse:\n\n{trimmed_response}"
else:
result = self.validator.validate_object(parsed_response)
if isinstance(result, Success):
return result
error_message = result.message
else:
error_message = "Response did not contain any text resembling JSON."
error_message = f"Response did not contain any text resembling JSON.\nResponse was\n\n{text_response}"
if num_repairs_attempted >= self._max_repair_attempts:
return Failure(error_message)
num_repairs_attempted += 1
messages.append({"role": "assistant", "content": text_response})
messages.append({"role": "user", "content": self._create_repair_prompt(error_message)})

def _create_request_prompt(self, intent: str) -> str:
Expand Down
257 changes: 253 additions & 4 deletions python/tests/__snapshots__/test_translator.ambr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,39 @@
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Get me stuff.',
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
}),
])
# ---
# name: test_translator_with_invalid_json
list([
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': '''

Expand All @@ -33,7 +63,57 @@
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
'payload': '{ "a": "hello" "b": true }',
}),
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
dict({
'content': '{ "a": "hello" "b": true }',
'role': 'assistant',
}),
dict({
'content': '''

The above JSON object is invalid for the following reason:
'''
Error: expected `,` or `}` at line 1 column 16

Attempted to parse:

{ "a": "hello" "b": true }
'''
The following is a revised JSON object:

''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello" "b": true, "c": 1234 }',
}),
])
# ---
Expand All @@ -43,9 +123,94 @@
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Get me stuff.',
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true }',
}),
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
dict({
'content': '{ "a": "hello", "b": true }',
'role': 'assistant',
}),
dict({
'content': '''

The above JSON object is invalid for the following reason:
'''
Validation path `c` failed for value `{"a": "hello", "b": true}` because:
Field required
'''
The following is a revised JSON object:

''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
}),
])
# ---
# name: test_translator_with_single_failure_and_list_preamble_1
list([
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Hey, I need some stuff.',
'role': 'user',
}),
dict({
'content': 'Okay, what kind of stuff?',
'role': 'assistant',
}),
dict({
'content': '''

Expand All @@ -67,6 +232,48 @@
''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true }',
}),
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Hey, I need some stuff.',
'role': 'user',
}),
dict({
'content': 'Okay, what kind of stuff?',
'role': 'assistant',
}),
dict({
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
dict({
'content': '{ "a": "hello", "b": true }',
'role': 'assistant',
}),
dict({
'content': '''

Expand All @@ -82,6 +289,44 @@
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true, "c": 1234 }',
}),
])
# ---
# name: test_translator_with_single_failure_and_str_preamble
list([
dict({
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Just so you know, I need some stuff.',
'role': 'user',
}),
dict({
'content': '''

You are a service that translates user requests into JSON objects of type "ExampleABC" according to the following TypeScript definitions:
```
interface ExampleABC {
a: string;
b: boolean;
c: number;
}

```
The following is a user request:
'''
Get me stuff.
'''
The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:

''',
'role': 'user',
}),
]),
}),
dict({
'kind': 'MODEL RESPONSE',
'payload': '{ "a": "hello", "b": true }',
Expand All @@ -90,7 +335,7 @@
'kind': 'CLIENT REQUEST',
'payload': list([
dict({
'content': 'Get me stuff.',
'content': 'Just so you know, I need some stuff.',
'role': 'user',
}),
dict({
Expand All @@ -114,6 +359,10 @@
''',
'role': 'user',
}),
dict({
'content': '{ "a": "hello", "b": true }',
'role': 'assistant',
}),
dict({
'content': '''

Expand Down
Loading