Skip to content

Commit 6440a9f

Browse files
Ericson2314Ericson2314
committed
Store StructuredAttrs directly in Derivation
Instead of parsing a structured attrs at some later point, we parsed it right away when parsing the A-Term format, and likewise serialize it to `__json = <JSON dump>` when serializing a derivation to A-Term. The JSON format can directly contain the JSON structured attrs without so encoding it, so we just do that. Currently tests pass. The problem is we went with saying a derivation *either* has structured attrs *or* environment variables, but not both. The problem is that this is not true. E.g. output name environment variables (with output paths) for fixed-output derivations do in fact today get put in the environment map in the structured attrs case. We need to decide what we want to do about this.
1 parent 4777734 commit 6440a9f

27 files changed

+595
-313
lines changed

‎src/libexpr/get-drvs.cc‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "nix/store/derivations.hh"
44
#include "nix/store/store-api.hh"
55
#include "nix/store/path-with-outputs.hh"
6+
#include "nix/util/json-utils.hh"
67

78
#include <cstring>
89
#include <regex>
@@ -33,7 +34,14 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
3334

3435
outputName =
3536
selectedOutputs.empty()
36-
? getOr(drv.env, "outputName", "out")
37+
? std::visit(overloaded{
38+
[&](const StringPairs & drvEnv) -> const std::string {
39+
return getOr(drvEnv, "outputName", "out");
40+
},
41+
[&](const StructuredAttrs & structuredAttrs) -> const std::string {
42+
return getString(getOr(getObject(structuredAttrs.structuredAttrs), "outputName", "out"));
43+
},
44+
}, drv.env)
3745
: *selectedOutputs.begin();
3846

3947
auto i = drv.outputs.find(outputName);

‎src/libexpr/primops.cc‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "nix/store/store-api.hh"
1111
#include "nix/util/util.hh"
1212
#include "nix/util/processes.hh"
13+
#include "nix/util/json-utils.hh"
1314
#include "nix/expr/value-to-json.hh"
1415
#include "nix/expr/value-to-xml.hh"
1516
#include "nix/expr/primops.hh"
@@ -199,7 +200,14 @@ void derivationToValue(EvalState & state, const PosIdx pos, const SourcePath & p
199200
attrs.alloc(state.sDrvPath).mkString(path2, {
200201
NixStringContextElem::DrvDeep { .drvPath = storePath },
201202
});
202-
attrs.alloc(state.sName).mkString(drv.env["name"]);
203+
attrs.alloc(state.sName).mkString(std::visit(overloaded{
204+
[&](const StringPairs & drvEnv) -> const std::string & {
205+
return drvEnv.at("name");
206+
},
207+
[&](const StructuredAttrs & structuredAttrs) -> const std::string & {
208+
return getString(valueAt(structuredAttrs.structuredAttrs, "name"));
209+
},
210+
}, drv.env));
203211

204212
auto list = state.buildList(drv.outputs.size());
205213
for (const auto & [i, o] : enumerate(drv.outputs)) {
@@ -1370,9 +1378,9 @@ static void derivationStrictInternal(
13701378
the environment. */
13711379
else {
13721380

1373-
if (jsonObject) {
1374-
1375-
if (i->name == state.sStructuredAttrs) continue;
1381+
std::visit(overloaded{
1382+
[&](StructuredAttrs & structuredAttrs) {
1383+
if (i->name == state.sStructuredAttrs) return;
13761384

13771385
jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
13781386

@@ -1409,18 +1417,19 @@ static void derivationStrictInternal(
14091417
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead", drvName);
14101418

14111419

1412-
} else {
1420+
},
1421+
[&](StringPairs & drvEnv) {
14131422
auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
1414-
drv.env.emplace(key, s);
1423+
drvEnv.emplace(key, s);
14151424
if (i->name == state.sBuilder) drv.builder = std::move(s);
14161425
else if (i->name == state.sSystem) drv.platform = std::move(s);
14171426
else if (i->name == state.sOutputHash) outputHash = std::move(s);
14181427
else if (i->name == state.sOutputHashAlgo) outputHashAlgo = parseHashAlgoOpt(s);
14191428
else if (i->name == state.sOutputHashMode) handleHashMode(s);
14201429
else if (i->name == state.sOutputs)
14211430
handleOutputs(tokenizeString<Strings>(s));
1422-
}
1423-
1431+
},
1432+
}, drv.env);
14241433
}
14251434

14261435
} catch (Error & e) {
@@ -1431,8 +1440,7 @@ static void derivationStrictInternal(
14311440
}
14321441

14331442
if (jsonObject) {
1434-
drv.env.emplace("__json", jsonObject->dump());
1435-
jsonObject.reset();
1443+
drv.env = StructuredAttrs{std::move(*jsonObject)};
14361444
}
14371445

14381446
/* Everything in the context of the strings in the derivation
@@ -1509,7 +1517,8 @@ static void derivationStrictInternal(
15091517
},
15101518
};
15111519

1512-
drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
1520+
if (auto * drvEnv = std::get_if<StringPairs>(&drv.env))
1521+
(*drvEnv)["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
15131522
drv.outputs.insert_or_assign("out", std::move(dof));
15141523
}
15151524

@@ -1522,7 +1531,8 @@ static void derivationStrictInternal(
15221531
auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive);
15231532

15241533
for (auto & i : outputs) {
1525-
drv.env[i] = hashPlaceholder(i);
1534+
if (auto * drvEnv = std::get_if<StringPairs>(&drv.env))
1535+
(*drvEnv)[i] = hashPlaceholder(i);
15261536
if (isImpure)
15271537
drv.outputs.insert_or_assign(i,
15281538
DerivationOutput::Impure {
@@ -1546,7 +1556,8 @@ static void derivationStrictInternal(
15461556
that changes in the set of output names do get reflected in
15471557
the hash. */
15481558
for (auto & i : outputs) {
1549-
drv.env[i] = "";
1559+
if (auto * drvEnv = std::get_if<StringPairs>(&drv.env))
1560+
(*drvEnv)[i] = "";
15501561
drv.outputs.insert_or_assign(i,
15511562
DerivationOutput::Deferred { });
15521563
}
@@ -1562,7 +1573,8 @@ static void derivationStrictInternal(
15621573
i
15631574
).atPos(v).debugThrow();
15641575
auto outPath = state.store->makeOutputPath(i, *h, drvName);
1565-
drv.env[i] = state.store->printStorePath(outPath);
1576+
if (auto * drvEnv = std::get_if<StringPairs>(&drv.env))
1577+
(*drvEnv)[i] = state.store->printStorePath(outPath);
15661578
drv.outputs.insert_or_assign(
15671579
i,
15681580
DerivationOutput::InputAddressed {

‎src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,6 @@
44
"echo hello > $out"
55
],
66
"builder": "/bin/bash",
7-
"env": {
8-
"__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}",
9-
"dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
10-
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
11-
},
127
"inputDrvs": {},
138
"inputSrcs": [],
149
"name": "advanced-attributes-structured-attrs-defaults",
@@ -22,5 +17,16 @@
2217
"method": "nar"
2318
}
2419
},
20+
"structuredAttrs": {
21+
"builder": "/bin/bash",
22+
"name": "advanced-attributes-structured-attrs-defaults",
23+
"outputHashAlgo": "sha256",
24+
"outputHashMode": "recursive",
25+
"outputs": [
26+
"out",
27+
"dev"
28+
],
29+
"system": "my-system"
30+
},
2531
"system": "my-system"
2632
}

‎src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,6 @@
44
"echo hello > $out"
55
],
66
"builder": "/bin/bash",
7-
"env": {
8-
"__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
9-
"bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m",
10-
"dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
11-
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
12-
},
137
"inputDrvs": {
148
"/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
159
"dynamicOutputs": {},
@@ -44,5 +38,62 @@
4438
"method": "nar"
4539
}
4640
},
41+
"structuredAttrs": {
42+
"__darwinAllowLocalNetworking": true,
43+
"__impureHostDeps": [
44+
"/usr/bin/ditto"
45+
],
46+
"__noChroot": true,
47+
"__sandboxProfile": "sandcastle",
48+
"allowSubstitutes": false,
49+
"builder": "/bin/bash",
50+
"exportReferencesGraph": {
51+
"refs1": [
52+
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
53+
],
54+
"refs2": [
55+
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
56+
]
57+
},
58+
"impureEnvVars": [
59+
"UNICORN"
60+
],
61+
"name": "advanced-attributes-structured-attrs",
62+
"outputChecks": {
63+
"bin": {
64+
"disallowedReferences": [
65+
"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
66+
],
67+
"disallowedRequisites": [
68+
"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
69+
]
70+
},
71+
"dev": {
72+
"maxClosureSize": 5909,
73+
"maxSize": 789
74+
},
75+
"out": {
76+
"allowedReferences": [
77+
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
78+
],
79+
"allowedRequisites": [
80+
"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
81+
]
82+
}
83+
},
84+
"outputHashAlgo": "sha256",
85+
"outputHashMode": "recursive",
86+
"outputs": [
87+
"out",
88+
"bin",
89+
"dev"
90+
],
91+
"preferLocalBuild": true,
92+
"requiredSystemFeatures": [
93+
"rainbow",
94+
"uid-range"
95+
],
96+
"system": "my-system"
97+
},
4798
"system": "my-system"
4899
}

‎src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,6 @@
44
"echo hello > $out"
55
],
66
"builder": "/bin/bash",
7-
"env": {
8-
"__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}",
9-
"dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev",
10-
"out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
11-
},
127
"inputDrvs": {},
138
"inputSrcs": [],
149
"name": "advanced-attributes-structured-attrs-defaults",
@@ -20,5 +15,14 @@
2015
"path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
2116
}
2217
},
18+
"structuredAttrs": {
19+
"builder": "/bin/bash",
20+
"name": "advanced-attributes-structured-attrs-defaults",
21+
"outputs": [
22+
"out",
23+
"dev"
24+
],
25+
"system": "my-system"
26+
},
2327
"system": "my-system"
2428
}

‎src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,6 @@
44
"echo hello > $out"
55
],
66
"builder": "/bin/bash",
7-
"env": {
8-
"__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
9-
"bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin",
10-
"dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
11-
"out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
12-
},
137
"inputDrvs": {
148
"/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
159
"dynamicOutputs": {},
@@ -41,5 +35,60 @@
4135
"path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
4236
}
4337
},
38+
"structuredAttrs": {
39+
"__darwinAllowLocalNetworking": true,
40+
"__impureHostDeps": [
41+
"/usr/bin/ditto"
42+
],
43+
"__noChroot": true,
44+
"__sandboxProfile": "sandcastle",
45+
"allowSubstitutes": false,
46+
"builder": "/bin/bash",
47+
"exportReferencesGraph": {
48+
"refs1": [
49+
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
50+
],
51+
"refs2": [
52+
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
53+
]
54+
},
55+
"impureEnvVars": [
56+
"UNICORN"
57+
],
58+
"name": "advanced-attributes-structured-attrs",
59+
"outputChecks": {
60+
"bin": {
61+
"disallowedReferences": [
62+
"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
63+
],
64+
"disallowedRequisites": [
65+
"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
66+
]
67+
},
68+
"dev": {
69+
"maxClosureSize": 5909,
70+
"maxSize": 789
71+
},
72+
"out": {
73+
"allowedReferences": [
74+
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
75+
],
76+
"allowedRequisites": [
77+
"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
78+
]
79+
}
80+
},
81+
"outputs": [
82+
"out",
83+
"bin",
84+
"dev"
85+
],
86+
"preferLocalBuild": true,
87+
"requiredSystemFeatures": [
88+
"rainbow",
89+
"uid-range"
90+
],
91+
"system": "my-system"
92+
},
4493
"system": "my-system"
4594
}

0 commit comments

Comments
 (0)