diff --git a/pkg/assemble/assemble.go b/pkg/assemble/assemble.go deleted file mode 100644 index ad44d1dd..00000000 --- a/pkg/assemble/assemble.go +++ /dev/null @@ -1,17 +0,0 @@ -package assemble - -import ( - "encoding/json" - "io" - - "github.com/gptscript-ai/gptscript/pkg/types" -) - -var Header = []byte("GPTSCRIPT!") - -func Assemble(prg types.Program, output io.Writer) error { - if _, err := output.Write(Header); err != nil { - return err - } - return json.NewEncoder(output).Encode(prg) -} diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index 5c818902..0a664b4d 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -23,7 +23,7 @@ func Authorize(ctx engine.Context, input string) (runner.AuthorizerResponse, err var result bool err := survey.AskOne(&survey.Confirm{ - Help: fmt.Sprintf("The full source of the tools is as follows:\n\n%s", ctx.Tool.String()), + Help: fmt.Sprintf("The full source of the tools is as follows:\n\n%s", ctx.Tool.Print()), Default: true, Message: ConfirmMessage(ctx, input), }, &result) diff --git a/pkg/cli/fmt.go b/pkg/cli/fmt.go index 72696756..8e669349 100644 --- a/pkg/cli/fmt.go +++ b/pkg/cli/fmt.go @@ -43,9 +43,9 @@ func (e *Fmt) Run(_ *cobra.Command, args []string) error { } if e.Write && loc != "" { - return os.WriteFile(loc, []byte(doc.String()), 0644) + return os.WriteFile(loc, []byte(doc.Print()), 0644) } - fmt.Print(doc.String()) + fmt.Print(doc.Print()) return nil } diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index 16f9152d..718c6f90 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -14,7 +14,6 @@ import ( "github.com/fatih/color" "github.com/gptscript-ai/cmd" gptscript2 "github.com/gptscript-ai/go-gptscript" - "github.com/gptscript-ai/gptscript/pkg/assemble" "github.com/gptscript-ai/gptscript/pkg/auth" "github.com/gptscript-ai/gptscript/pkg/builtin" "github.com/gptscript-ai/gptscript/pkg/cache" @@ -58,7 +57,6 @@ type GPTScript struct { // Input should not be using GPTSCRIPT_INPUT env var because that is the same value that is set in tool executions Input string `usage:"Read input from a file (\"-\" for stdin)" short:"f" env:"GPTSCRIPT_INPUT_FILE"` SubTool string `usage:"Use tool of this name, not the first tool in file" local:"true"` - Assemble bool `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true" local:"true"` ListModels bool `usage:"List the models available and exit" local:"true"` ListTools bool `usage:"List built-in tools and exit" local:"true"` ListenAddress string `usage:"Server listen address" default:"127.0.0.1:0" hidden:"true"` @@ -439,20 +437,6 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) (retErr error) { return cmd.Help() } - if r.Assemble { - var out io.Writer = os.Stdout - if r.Output != "" && r.Output != "-" { - f, err := os.Create(r.Output) - if err != nil { - return fmt.Errorf("opening %s: %w", r.Output, err) - } - defer f.Close() - out = f - } - - return assemble.Assemble(prg, out) - } - toolInput, err := input.FromCLI(r.Input, args) if err != nil { return err diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index 902d0ed9..8e3914b5 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -17,7 +17,6 @@ import ( "github.com/getkin/kin-openapi/openapi3" "github.com/gptscript-ai/gptscript/internal" - "github.com/gptscript-ai/gptscript/pkg/assemble" "github.com/gptscript-ai/gptscript/pkg/builtin" "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -132,36 +131,6 @@ func loadLocal(base *source, name string) (*source, bool, error) { }, true, nil } -func loadProgram(data []byte, into *types.Program, targetToolName, defaultModel string) (types.Tool, error) { - var ext types.Program - - if err := json.Unmarshal(data[len(assemble.Header):], &ext); err != nil { - return types.Tool{}, err - } - - into.ToolSet = make(map[string]types.Tool, len(ext.ToolSet)) - for k, v := range ext.ToolSet { - if builtinTool, ok := builtin.DefaultModel(k, defaultModel); ok { - v = builtinTool - } - into.ToolSet[k] = v - } - - tool := into.ToolSet[ext.EntryToolID] - if targetToolName == "" { - return tool, nil - } - - tool, ok := into.ToolSet[tool.LocalTools[strings.ToLower(targetToolName)]] - if !ok { - return tool, &types.ErrToolNotFound{ - ToolName: targetToolName, - } - } - - return tool, nil -} - func loadOpenAPI(prg *types.Program, data []byte) *openapi3.T { var ( openAPICacheKey = hash.Digest(data) @@ -189,14 +158,6 @@ func loadOpenAPI(prg *types.Program, data []byte) *openapi3.T { func readTool(ctx context.Context, cache *cache.Client, prg *types.Program, base *source, targetToolName, defaultModel string) ([]types.Tool, error) { data := base.Content - if bytes.HasPrefix(data, assemble.Header) { - tool, err := loadProgram(data, prg, targetToolName, defaultModel) - if err != nil { - return nil, err - } - return []types.Tool{tool}, nil - } - var ( tools []types.Tool isOpenAPI bool @@ -231,11 +192,19 @@ func readTool(ctx context.Context, cache *cache.Client, prg *types.Program, base // If we didn't get any tools from trying to parse it as OpenAPI, try to parse it as a GPTScript if len(tools) == 0 { var err error - tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{ - AssignGlobals: true, - }) - if err != nil { - return nil, err + _, marshaled, ok := strings.Cut(string(data), "#!GPTSCRIPT") + if ok { + err = json.Unmarshal([]byte(marshaled), &tools) + if err != nil { + return nil, fmt.Errorf("error parsing marshalled script: %w", err) + } + } else { + tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{ + AssignGlobals: true, + }) + if err != nil { + return nil, err + } } } diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 4862271b..65bc2ae8 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -282,10 +282,7 @@ func toMessages(request types.CompletionRequest, compat bool) (result []openai.C chatMessage.ToolCalls = append(chatMessage.ToolCalls, toToolCall(*content.ToolCall)) } if content.Text != "" { - chatMessage.MultiContent = append(chatMessage.MultiContent, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, - Text: content.Text, - }) + chatMessage.MultiContent = append(chatMessage.MultiContent, textToMultiContent(content.Text)...) } } @@ -307,6 +304,35 @@ func toMessages(request types.CompletionRequest, compat bool) (result []openai.C return } +const imagePrefix = "data:image/png;base64," + +func textToMultiContent(text string) []openai.ChatMessagePart { + var chatParts []openai.ChatMessagePart + parts := strings.Split(text, "\n") + for i := len(parts) - 1; i >= 0; i-- { + if strings.HasPrefix(parts[i], imagePrefix) { + chatParts = append(chatParts, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: parts[i], + }, + }) + parts = parts[:i] + } else { + break + } + } + if len(parts) > 0 { + chatParts = append(chatParts, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeText, + Text: strings.Join(parts, "\n"), + }) + } + + slices.Reverse(chatParts) + return chatParts +} + func (c *Client) Call(ctx context.Context, messageRequest types.CompletionRequest, env []string, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) { if err := c.ValidAuth(); err != nil { if err := c.RetrieveAPIKey(ctx, env); err != nil { diff --git a/pkg/openai/client_test.go b/pkg/openai/client_test.go index 30f1705b..78f3eac2 100644 --- a/pkg/openai/client_test.go +++ b/pkg/openai/client_test.go @@ -9,6 +9,44 @@ import ( "github.com/hexops/valast" ) +func TestTextToMultiContent(t *testing.T) { + autogold.Expect([]openai.ChatMessagePart{{ + Type: "text", + Text: "hi\ndata:image/png;base64,xxxxx\n", + }}).Equal(t, textToMultiContent("hi\ndata:image/png;base64,xxxxx\n")) + + autogold.Expect([]openai.ChatMessagePart{ + { + Type: "text", + Text: "hi", + }, + { + Type: "image_url", + ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"}, + }, + }).Equal(t, textToMultiContent("hi\ndata:image/png;base64,xxxxx")) + + autogold.Expect([]openai.ChatMessagePart{{ + Type: "image_url", + ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"}, + }}).Equal(t, textToMultiContent("data:image/png;base64,xxxxx")) + + autogold.Expect([]openai.ChatMessagePart{ + { + Type: "text", + Text: "\none\ntwo", + }, + { + Type: "image_url", + ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"}, + }, + { + Type: "image_url", + ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,yyyyy"}, + }, + }).Equal(t, textToMultiContent("\none\ntwo\ndata:image/png;base64,xxxxx\ndata:image/png;base64,yyyyy")) +} + func Test_appendMessage(t *testing.T) { autogold.Expect(types.CompletionMessage{Content: []types.ContentPart{ {ToolCall: &types.CompletionToolCall{ diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 626056a7..e1b0b9fa 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -263,7 +263,7 @@ func writeSep(buf *strings.Builder, lastText bool) { } } -func (d Document) String() string { +func (d Document) Print() string { buf := strings.Builder{} lastText := false for _, node := range d.Nodes { @@ -274,7 +274,7 @@ func (d Document) String() string { } if node.ToolNode != nil { writeSep(&buf, lastText) - buf.WriteString(node.ToolNode.Tool.String()) + buf.WriteString(node.ToolNode.Tool.Print()) lastText = false } } diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 7e1282ca..a3263539 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -304,7 +304,7 @@ body !metadata:first:package.json foo=base f -`).Equal(t, tools[0].String()) +`).Equal(t, tools[0].Print()) } func TestFormatWithBadInstruction(t *testing.T) { @@ -316,9 +316,9 @@ func TestFormatWithBadInstruction(t *testing.T) { Instructions: "foo: bar", }, } - autogold.Expect("Name: foo\n===\nfoo: bar\n").Equal(t, input.String()) + autogold.Expect("Name: foo\n===\nfoo: bar\n").Equal(t, input.Print()) - tools, err := ParseTools(strings.NewReader(input.String())) + tools, err := ParseTools(strings.NewReader(input.Print())) require.NoError(t, err) if reflect.DeepEqual(input, tools[0]) { t.Errorf("expected %v, got %v", input, tools[0]) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index df3ef172..bd05e96a 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -655,6 +655,17 @@ func (r *Runner) newDispatcher(ctx context.Context) dispatcher { return newParallelDispatcher(ctx) } +func idForToolCall(id string, state *engine.Return) string { + if state == nil || state.State == nil { + return id + } + tc, ok := state.State.Pending[id] + if !ok || tc.Index == nil { + return id + } + return fmt.Sprintf("%03d", *tc.Index) +} + func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, state *State, toolCategory engine.ToolCategory) (*State, []SubCallResult, error) { var ( resultLock sync.Mutex @@ -698,7 +709,9 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, // Sort the id so if sequential the results are predictable ids := maps.Keys(state.Continuation.Calls) - sort.Strings(ids) + sort.Slice(ids, func(i, j int) bool { + return idForToolCall(ids[i], state.Continuation) < idForToolCall(ids[j], state.Continuation) + }) for _, id := range ids { call := state.Continuation.Calls[id] diff --git a/pkg/sdkserver/routes.go b/pkg/sdkserver/routes.go index b1bd4c3b..66449749 100644 --- a/pkg/sdkserver/routes.go +++ b/pkg/sdkserver/routes.go @@ -114,7 +114,7 @@ func (s *server) listTools(w http.ResponseWriter, r *http.Request) { // Don't print instructions tool.Instructions = "" - lines = append(lines, tool.String()) + lines = append(lines, tool.Print()) } writeResponse(logger, w, map[string]any{"stdout": strings.Join(lines, "\n---\n")}) @@ -339,5 +339,5 @@ func (s *server) fmtDocument(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, map[string]string{"stdout": doc.String()}) + writeResponse(logger, w, map[string]string{"stdout": doc.Print()}) } diff --git a/pkg/sdkserver/types.go b/pkg/sdkserver/types.go index a4332557..278a8c78 100644 --- a/pkg/sdkserver/types.go +++ b/pkg/sdkserver/types.go @@ -1,8 +1,8 @@ package sdkserver import ( + "encoding/json" "maps" - "strings" "time" "github.com/gptscript-ai/gptscript/pkg/cache" @@ -30,15 +30,12 @@ const ( type toolDefs []types.ToolDef func (t toolDefs) String() string { - s := new(strings.Builder) - for i, tool := range t { - s.WriteString(tool.String()) - if i != len(t)-1 { - s.WriteString("\n\n---\n\n") - } + data, err := json.Marshal(t) + if err != nil { + panic(err) } - return s.String() + return "#!GPTSCRIPT" + string(data) } type ( diff --git a/pkg/types/tool.go b/pkg/types/tool.go index fcd0d53d..0c8bd77c 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -389,6 +389,14 @@ func (t Tool) GetToolRefsFromNames(names []string) (result []ToolReference, _ er } func (t ToolDef) String() string { + data, err := json.Marshal([]any{t}) + if err != nil { + panic(err) + } + return "#!GPTSCRIPT" + string(data) +} + +func (t ToolDef) Print() string { buf := &strings.Builder{} if t.Parameters.GlobalModelName != "" { _, _ = fmt.Fprintf(buf, "Global Model Name: %s\n", t.Parameters.GlobalModelName) diff --git a/pkg/types/tool_test.go b/pkg/types/tool_test.go index a146955e..1160b4f8 100644 --- a/pkg/types/tool_test.go +++ b/pkg/types/tool_test.go @@ -6,7 +6,7 @@ import ( "github.com/hexops/autogold/v2" ) -func TestToolDef_String(t *testing.T) { +func TestToolDef_Print(t *testing.T) { tool := ToolDef{ Parameters: Parameters{ Name: "Tool Sample", @@ -82,7 +82,7 @@ This is a sample instruction // blah blah some ugly JSON } -`).Equal(t, tool.String()) +`).Equal(t, tool.Print()) } // float32Ptr is used to return a pointer to a given float32 value