From 6ddbe9190a64a50567379ab540825e63cc8bf4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Eriksson?= Date: Wed, 6 Aug 2025 09:21:28 +0200 Subject: [PATCH] Use ENCORE_APP_META_PATH for 'encore test' --- cli/daemon/run/exec_command.go | 2 +- cli/daemon/run/exec_script.go | 2 +- cli/daemon/run/run.go | 2 +- cli/daemon/run/runtime_config2.go | 26 ++++++++++----- cli/daemon/run/tests.go | 13 +++++++- e2e-tests/echo_app_test.go | 2 +- pkg/builder/builder.go | 1 + pkg/builder/builderimpl/builders.go | 2 +- pkg/clientgen/client_test.go | 2 +- pkg/scrub/metascrub/metascrub_test.go | 2 +- v2/tsbuilder/tsbuilder.go | 17 ++++++++-- v2/v2builder/v2builder.go | 48 +++++++++++++++++++-------- 12 files changed, 87 insertions(+), 32 deletions(-) diff --git a/cli/daemon/run/exec_command.go b/cli/daemon/run/exec_command.go index 529a7ea098..7e7bd4f137 100644 --- a/cli/daemon/run/exec_command.go +++ b/cli/daemon/run/exec_command.go @@ -165,7 +165,7 @@ func (mgr *Manager) ExecCommand(ctx context.Context, p ExecCommandParams) (err e Gateways: gateways, DefinedSecrets: secrets, SvcConfigs: cfg.Configs, - IncludeMetaEnv: bld.NeedsMeta(), + IncludeMeta: bld.NeedsMeta(), } procConf, err := configGen.AllInOneProc() if err != nil { diff --git a/cli/daemon/run/exec_script.go b/cli/daemon/run/exec_script.go index ece65995f9..953ab31d42 100644 --- a/cli/daemon/run/exec_script.go +++ b/cli/daemon/run/exec_script.go @@ -197,7 +197,7 @@ func (mgr *Manager) ExecScript(ctx context.Context, p ExecScriptParams) (err err Gateways: gateways, DefinedSecrets: secrets, SvcConfigs: cfg.Configs, - IncludeMetaEnv: bld.NeedsMeta(), + IncludeMeta: bld.NeedsMeta(), } procConf, err := configGen.AllInOneProc() if err != nil { diff --git a/cli/daemon/run/run.go b/cli/daemon/run/run.go index ea8bc9b0c8..0018b8490f 100644 --- a/cli/daemon/run/run.go +++ b/cli/daemon/run/run.go @@ -551,7 +551,7 @@ func (r *Run) StartProcGroup(params *StartProcGroupParams) (p *ProcGroup, err er DefinedSecrets: params.Secrets, SvcConfigs: params.ServiceConfigs, DeployID: option.Some(fmt.Sprintf("run_%s", xid.New().String())), - IncludeMetaEnv: r.Builder.NeedsMeta(), + IncludeMeta: r.Builder.NeedsMeta(), }, Experiments: params.Experiments, Meta: params.Meta, diff --git a/cli/daemon/run/runtime_config2.go b/cli/daemon/run/runtime_config2.go index acf2a2603a..341b457daa 100644 --- a/cli/daemon/run/runtime_config2.go +++ b/cli/daemon/run/runtime_config2.go @@ -39,6 +39,7 @@ const ( serviceCfgEnvPrefix = "ENCORE_CFG_" listenEnvVar = "ENCORE_LISTEN_ADDR" metaEnvVar = "ENCORE_APP_META" + metaPathEnvVar = "ENCORE_APP_META_PATH" ) type RuntimeConfigGenerator struct { @@ -76,8 +77,11 @@ type RuntimeConfigGenerator struct { Gateways map[string]GatewayConfig AuthKey config.EncoreAuthKey - // Whether to include the metadata as an environment variable. - IncludeMetaEnv bool + // Whether to include the metadata. + IncludeMeta bool + // If set, write the metadata to the given path + // instead of including it as an environment variable. + MetaPath option.Option[string] // The values of defined secrets. DefinedSecrets map[string]string @@ -96,7 +100,6 @@ type GatewayConfig struct { func (g *RuntimeConfigGenerator) initialize() error { return g.initOnce.Do(func() error { g.conf = rtconfgen.NewBuilder() - newRid := func() string { return "res_" + xid.New().String() } if deployID, ok := g.DeployID.Get(); ok { @@ -719,7 +722,7 @@ func (g *RuntimeConfigGenerator) ForTests(newRuntimeConf bool) (envs []string, e svcNames := fns.Map(g.md.Svcs, func(svc *meta.Service) string { return svc.Name }) envs = append(envs, g.encodeConfigs(svcNames...)...) - if g.IncludeMetaEnv { + if g.IncludeMeta { metaBytes, err := proto.Marshal(g.md) if err != nil { return nil, errors.Wrap(err, "failed to marshal metadata") @@ -779,14 +782,21 @@ func (g *RuntimeConfigGenerator) ProcEnvs(proc *ProcConfig, useRuntimeConfigV2 b env = append(env, fmt.Sprintf("%s=%s", runtimeCfgEnvVar, runtimeCfgStr)) } - if g.IncludeMetaEnv { + if g.IncludeMeta { metaBytes, err := proto.Marshal(g.md) if err != nil { return nil, errors.Wrap(err, "failed to marshal metadata") } - gzipped := gzipBytes(metaBytes) - metaEnvStr := "gzip:" + base64.StdEncoding.EncodeToString(gzipped) - env = append(env, fmt.Sprintf("%s=%s", metaEnvVar, metaEnvStr)) + if metaPath, ok := g.MetaPath.Get(); ok { + if err := os.WriteFile(metaPath, metaBytes, 0644); err != nil { + return nil, errors.Wrap(err, "failed to write metadata") + } + env = append(env, fmt.Sprintf("%s=%s", metaPathEnvVar, metaPath)) + } else { + gzipped := gzipBytes(metaBytes) + metaEnvStr := "gzip:" + base64.StdEncoding.EncodeToString(gzipped) + env = append(env, fmt.Sprintf("%s=%s", metaEnvVar, metaEnvStr)) + } } if runtimeLibPath := encoreEnv.EncoreRuntimeLib(); runtimeLibPath != "" { diff --git a/cli/daemon/run/tests.go b/cli/daemon/run/tests.go index 52a56d9012..4cccab3ea9 100644 --- a/cli/daemon/run/tests.go +++ b/cli/daemon/run/tests.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "path/filepath" "runtime" "strings" @@ -191,6 +192,15 @@ func (mgr *Manager) testSpec(ctx context.Context, bld builder.Impl, expSet *expe return nil, err } + var metaPath option.Option[string] + if bld.NeedsMeta() { + tempDir, err := bld.TempDir() + if err != nil { + return nil, err + } + metaPath = option.Some(filepath.Join(tempDir, "meta.pb")) + } + authKey := genAuthKey() configGen := &RuntimeConfigGenerator{ app: params.App, @@ -206,7 +216,8 @@ func (mgr *Manager) testSpec(ctx context.Context, bld builder.Impl, expSet *expe EnvName: option.Some("test"), EnvType: option.Some(runtimev1.Environment_TYPE_TEST), DeployID: option.Some(fmt.Sprintf("clitest_%s", xid.New().String())), - IncludeMetaEnv: bld.NeedsMeta(), + IncludeMeta: bld.NeedsMeta(), + MetaPath: metaPath, } env, err := configGen.ForTests(bld.UseNewRuntimeConfig()) diff --git a/e2e-tests/echo_app_test.go b/e2e-tests/echo_app_test.go index 5a76a540f2..2efb2edd81 100644 --- a/e2e-tests/echo_app_test.go +++ b/e2e-tests/echo_app_test.go @@ -590,7 +590,7 @@ func TestProcClosedOnCtxCancel(t *testing.T) { ResourceManager: rm, ListenAddr: "127.0.0.1:34212", SvcProxy: svcProxy, - Builder: v2builder.BuilderImpl{}, + Builder: v2builder.New(), } parse, build, _ := testBuild(c, appRoot, append(os.Environ(), "ENCORE_EXPERIMENT=v2")) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index e9a328f0ec..cf857e4bd7 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -283,5 +283,6 @@ type Impl interface { GenUserFacing(context.Context, GenUserFacingParams) error UseNewRuntimeConfig() bool NeedsMeta() bool + TempDir() (string, error) Close() error } diff --git a/pkg/builder/builderimpl/builders.go b/pkg/builder/builderimpl/builders.go index 6374e809b7..8aeb82b9a3 100644 --- a/pkg/builder/builderimpl/builders.go +++ b/pkg/builder/builderimpl/builders.go @@ -12,5 +12,5 @@ func Resolve(lang appfile.Lang, expSet *experiments.Set) builder.Impl { if lang == appfile.LangTS || experiments.TypeScript.Enabled(expSet) { return tsbuilder.New() } - return v2builder.BuilderImpl{} + return v2builder.New() } diff --git a/pkg/clientgen/client_test.go b/pkg/clientgen/client_test.go index 662aa6a5bd..4f6d1d5823 100644 --- a/pkg/clientgen/client_test.go +++ b/pkg/clientgen/client_test.go @@ -33,7 +33,7 @@ func TestClientCodeGenerationFromGoApp(t *testing.T) { c.Assert(err, qt.IsNil) ctx := context.Background() - bld := v2builder.BuilderImpl{} + bld := v2builder.New() for _, path := range tests { path := path diff --git a/pkg/scrub/metascrub/metascrub_test.go b/pkg/scrub/metascrub/metascrub_test.go index f189b16b23..3218a7cd86 100644 --- a/pkg/scrub/metascrub/metascrub_test.go +++ b/pkg/scrub/metascrub/metascrub_test.go @@ -166,7 +166,7 @@ func testParse(c *qt.C, code string) *meta.Data { err := txtar.Write(ar, root) c.Assert(err, qt.IsNil) - bld := v2builder.BuilderImpl{} + bld := v2builder.New() ctx := context.Background() res, err := bld.Parse(ctx, builder.ParseParams{ diff --git a/v2/tsbuilder/tsbuilder.go b/v2/tsbuilder/tsbuilder.go index 9a04985d9b..a3407b7ed3 100644 --- a/v2/tsbuilder/tsbuilder.go +++ b/v2/tsbuilder/tsbuilder.go @@ -34,8 +34,9 @@ func New() *BuilderImpl { } type BuilderImpl struct { - mu sync.Mutex - cmds map[*runningCmd]bool + mu sync.Mutex + cmds map[*runningCmd]bool + tempDirs []string } type parseInput struct { @@ -79,6 +80,10 @@ func (i *BuilderImpl) Close() error { _ = c.cmd.Cancel() } + for _, dir := range i.tempDirs { + _ = os.RemoveAll(dir) + } + for c := range cmds { <-c.done } @@ -207,6 +212,14 @@ type compileInput struct { NodeJSRuntime NodeJSRuntime `json:"nodejs_runtime"` } +func (i *BuilderImpl) TempDir() (string, error) { + dir, err := os.MkdirTemp("", "encore-tsbuilder") + if err == nil { + i.tempDirs = append(i.tempDirs, dir) + } + return dir, err +} + func (i *BuilderImpl) Compile(ctx context.Context, p builder.CompileParams) (*builder.CompileResult, error) { data := p.Parse.Data.(*data) diff --git a/v2/v2builder/v2builder.go b/v2/v2builder/v2builder.go index 5523fe0c78..0db656b46f 100644 --- a/v2/v2builder/v2builder.go +++ b/v2/v2builder/v2builder.go @@ -41,11 +41,23 @@ import ( "encr.dev/v2/parser/resource" ) -type BuilderImpl struct{} +func New() *BuilderImpl { + return &BuilderImpl{} +} -func (BuilderImpl) Close() error { return nil } +type BuilderImpl struct { + tempDirs []string +} -func (BuilderImpl) Parse(ctx context.Context, p builder.ParseParams) (*builder.ParseResult, error) { +func (i *BuilderImpl) Close() error { + for _, dir := range i.tempDirs { + _ = os.RemoveAll(dir) + } + + return nil +} + +func (*BuilderImpl) Parse(ctx context.Context, p builder.ParseParams) (*builder.ParseResult, error) { return etrace.Sync2(ctx, "", "v2builder.Parse", func(ctx context.Context) (res *builder.ParseResult, err error) { defer func() { err, _ = perr.CatchBailoutAndPanic(err, recover()) @@ -116,7 +128,7 @@ type parseData struct { traceNodes *legacymeta.TraceNodes } -func (BuilderImpl) Compile(ctx context.Context, p builder.CompileParams) (*builder.CompileResult, error) { +func (*BuilderImpl) Compile(ctx context.Context, p builder.CompileParams) (*builder.CompileResult, error) { return etrace.Sync2(ctx, "", "v2builder.Compile", func(ctx context.Context) (res *builder.CompileResult, err error) { defer func() { err, _ = perr.CatchBailoutAndPanic(err, recover()) @@ -206,7 +218,15 @@ func (BuilderImpl) Compile(ctx context.Context, p builder.CompileParams) (*build }) } -func (i BuilderImpl) ServiceConfigs(ctx context.Context, p builder.ServiceConfigsParams) (res *builder.ServiceConfigsResult, err error) { +func (i *BuilderImpl) TempDir() (string, error) { + dir, err := os.MkdirTemp("", "encore-gobuilder") + if err == nil { + i.tempDirs = append(i.tempDirs, dir) + } + return dir, err +} + +func (*BuilderImpl) ServiceConfigs(ctx context.Context, p builder.ServiceConfigsParams) (res *builder.ServiceConfigsResult, err error) { defer func() { if l, ok := perr.CatchBailout(recover()); ok && l.Len() > 0 { err = l.AsError() @@ -224,15 +244,15 @@ func (i BuilderImpl) ServiceConfigs(ctx context.Context, p builder.ServiceConfig }, nil } -func (i BuilderImpl) UseNewRuntimeConfig() bool { +func (*BuilderImpl) UseNewRuntimeConfig() bool { return false } -func (i BuilderImpl) NeedsMeta() bool { +func (*BuilderImpl) NeedsMeta() bool { return false } -func (i BuilderImpl) RunTests(ctx context.Context, p builder.RunTestsParams) error { +func (i *BuilderImpl) RunTests(ctx context.Context, p builder.RunTestsParams) error { return etrace.Sync1(ctx, "", "v2builder.Test", func(ctx context.Context) (err error) { defer func() { err, _ = perr.CatchBailoutAndPanic(err, recover()) @@ -262,7 +282,7 @@ type testBuilderData struct { pc *parsectx.Context } -func (i BuilderImpl) TestSpec(ctx context.Context, p builder.TestSpecParams) (*builder.TestSpecResult, error) { +func (i *BuilderImpl) TestSpec(ctx context.Context, p builder.TestSpecParams) (*builder.TestSpecResult, error) { return etrace.Sync2(ctx, "", "v2builder.TestSpec", func(ctx context.Context) (res *builder.TestSpecResult, err error) { spec, err := i.generateTestSpec(ctx, p) if err != nil { @@ -284,7 +304,7 @@ func (i BuilderImpl) TestSpec(ctx context.Context, p builder.TestSpecParams) (*b }) } -func (i BuilderImpl) generateTestSpec(ctx context.Context, p builder.TestSpecParams) (*build.TestSpec, error) { +func (i *BuilderImpl) generateTestSpec(ctx context.Context, p builder.TestSpecParams) (*build.TestSpec, error) { return etrace.Sync2(ctx, "", "v2builder.generateTestSpec", func(ctx context.Context) (res *build.TestSpec, err error) { defer func() { err, _ = perr.CatchBailoutAndPanic(err, recover()) @@ -337,7 +357,7 @@ func (i BuilderImpl) generateTestSpec(ctx context.Context, p builder.TestSpecPar // testEnvVars takes a list of env vars and filters them down to the ones // that should be embedded within the test binary. -func (i BuilderImpl) testEnvVarsToEmbed(args, envs []string) map[string]string { +func (i *BuilderImpl) testEnvVarsToEmbed(args, envs []string) map[string]string { if !slices.Contains(args, "-c") { return nil } @@ -413,7 +433,7 @@ func pickupConfigFiles(errs *perr.List, mainModule *pkginfo.Module) fs.FS { path = filepath.Dir(path) } - for i := 0; i < 30; i++ { + for range 30 { base := filepath.Base(path) if strings.ToLower(base) == "cue.mod" { return true @@ -439,7 +459,7 @@ func pickupConfigFiles(errs *perr.List, mainModule *pkginfo.Module) fs.FS { return configFiles } -func (i BuilderImpl) GenUserFacing(ctx context.Context, p builder.GenUserFacingParams) error { +func (i *BuilderImpl) GenUserFacing(ctx context.Context, p builder.GenUserFacingParams) error { return etrace.Sync1(ctx, "", "v2builder.GenUserFacing", func(ctx context.Context) (err error) { defer func() { err, _ = perr.CatchBailoutAndPanic(err, recover()) @@ -490,7 +510,7 @@ func (i BuilderImpl) GenUserFacing(ctx context.Context, p builder.GenUserFacingP // writeOrDeleteFile writes the given data to dst. If data is empty, it will // instead delete the file at dst. -func (i BuilderImpl) writeOrDeleteFile(errs *perr.List, data []byte, dst paths.FS) { +func (i *BuilderImpl) writeOrDeleteFile(errs *perr.List, data []byte, dst paths.FS) { if len(data) == 0 { // No need for any generated code. Try to remove the existing file // if it's there as it's no longer needed.