Skip to content

Commit 2aa2d38

Browse files
mgmeiermgmeier
authored
Merge pull request #5519 from input-output-hk/andreabedini/workbench-fix-source-repo
Include commits from source-repos in workbench manifest
2 parents 735e6c9 + afbbe12 commit 2aa2d38

File tree

4 files changed

+107
-74
lines changed

4 files changed

+107
-74
lines changed

‎nix/workbench/default.nix‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ with lib;
99

1010
let
1111
# recover CHaP location from cardano's project
12-
chapPackages = "${cardanoNodeProject.args.inputMap."https://input-output-hk.github.io/cardano-haskell-packages"}/foliage/packages.json";
12+
chap = cardanoNodeProject.args.inputMap."https://input-output-hk.github.io/cardano-haskell-packages";
1313
# build plan as computed by nix
1414
nixPlanJson = cardanoNodeProject.plan-nix.json;
1515

@@ -31,8 +31,8 @@ let
3131
wrapProgram "$out/bin/wb" \
3232
--argv0 wb \
3333
--prefix PATH ":" ${makeBinPath tools} \
34-
--set WB_CHAP_PACKAGES ${chapPackages} \
35-
--set WB_NIX_PLAN ${nixPlanJson}
34+
--set WB_CHAP_PATH ${chap} \
35+
--set WB_NIX_PLAN ${nixPlanJson}
3636
'';
3737

3838
installPhase = ''

‎nix/workbench/manifest.sh‎

Lines changed: 101 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
#shellcheck shell=bash
2+
#
13
usage_manifest() {
24
usage "manifest" "Manifest" <<EOF
3-
$(helpcmd collect-from-checkout DIR [NODE_REV=HEAD] [PKGS_TO_MENTION..])
5+
$(helpcmd collect-from-checkout DIR "[NODE_REV=HEAD]" "[PKGS_TO_MENTION..]")
46
$(blk c collect) Collect software manifest from the current 'cardano-node' checkout
57
$(helpcmd contributions-by-repository)
68
$(blk by byrepo) Show per-repository list of git hashes involved.
@@ -9,9 +11,12 @@ usage_manifest() {
911
EOF
1012
}
1113

12-
## WARNING: Keep in sync with Cardano.Analysis.API.Context.manifestPackages.
13-
## Better yet, move manifest collection _into_ locli.
14-
## ..but since the manifest is part of meta.json, that'll cause more thinking.
14+
# WARNING: Keep in sync with Cardano.Analysis.API.Context.manifestPackages.
15+
# Better yet, move manifest collection _into_ locli.
16+
# ..but since the manifest is part of meta.json, that'll cause more thinking.
17+
#
18+
# WB_MANIFEST_PACKAGES is defined here and used in ./wb
19+
# shellcheck disable=SC2034
1520
WB_MANIFEST_PACKAGES=(
1621
'cardano-node'
1722
'ouroboros-consensus'
@@ -30,52 +35,56 @@ case "${op}" in
3035
local usage="USAGE: wb manifest $0"
3136

3237
jq '
33-
def pkg_id: ."pkg-name" + "-" + ."pkg-version";
34-
def repo_hash: .url | split("?")[0];
38+
def pkg_id:
39+
."pkg-name" + "-" + ."pkg-version"
40+
;
3541
36-
def distill_package_data:
37-
(.url | split("?")[0] | split("/")) as $url_parts
42+
def is_from_chap:
43+
."pkg-src".repo?.uri?
44+
| strings
45+
| . == "https://input-output-hk.github.io/cardano-haskell-packages" or startswith("file:" + $chappath)
46+
;
47+
48+
def extract_package_data:
49+
(.url | split("?")[0] | split("/")) as $url_parts
3850
| { pkg_id: pkg_id
3951
, repository: $url_parts[0:2] | join("/")
4052
, commit: $url_parts[2]
4153
}
4254
;
4355
44-
def group_nicely:
45-
group_by(.repository)
46-
| map({
56+
# index chap packages by package-id
57+
$chapfile[0]
58+
| INDEX(pkg_id) as $chapdata
59+
60+
# take the packages in the plan that come from chap,
61+
# match them with chapdata by package-id
62+
# and extract the data
63+
| [ JOIN(
64+
$chapdata;
65+
$planfile[0]."install-plan"[] | select(is_from_chap);
66+
pkg_id
67+
)
68+
| .[1] # metadata from chap
69+
| extract_package_data
70+
]
71+
| group_by(.repository)
72+
| map({
4773
repository: .[0].repository,
48-
contributions: map({key: .pkg_id, value: .commit}) | from_entries
74+
contributions: INDEX(.pkg_id) | map_values(.commit)
4975
})
50-
;
76+
' --null-input \
77+
--slurpfile planfile "$WB_NIX_PLAN" \
78+
--slurpfile chapfile "$WB_CHAP_PATH/foliage/packages.json" \
79+
--arg chappath "$WB_CHAP_PATH"
5180

52-
# key chap packages by package id, so we can match them easily in the next step
53-
$chap
54-
| map({ key: pkg_id, value: distill_package_data })
55-
| from_entries as $package_origin
56-
57-
| $plan."install-plan"
58-
| map(select(."pkg-src".repo.uri == "https://input-output-hk.github.io/cardano-haskell-packages"))
59-
| unique_by(pkg_id)
60-
| map($package_origin[pkg_id])
61-
| group_nicely[]
62-
' --null-input \
63-
--argfile plan $WB_NIX_PLAN \
64-
--argfile chap $WB_CHAP_PACKAGES
6581
;;
82+
6683
collect-from-checkout | collect | c )
6784
local usage="USAGE: wb manifest $op CARDANO-NODE-CHECKOUT NODE-REV [PACKAGE...]"
6885
local dir=${1:?$usage}; shift
6986
local node_rev=${1:-}; shift || true
7087
local real_dir=$(realpath "$dir")
71-
local pkgnames=($*)
72-
73-
if test -z "$node_rev"
74-
then node_rev=$(manifest_git_head_commit "$dir"); fi
75-
76-
# TODO put this back when we support localisations
77-
#, "cardano-node-package-localisations": (. | split("\n") | unique | map(select(. != "")))
78-
# where . is the output of manifest_cabal_package_localisations "$dir"
7988

8089
# If WB_MODE_CABAL is "t", the workbench configured to use a local cabal build, therefore
8190
# we fetch the plan from cabal's dist-newstyle directory.
@@ -90,55 +99,79 @@ case "${op}" in
9099
plan_path=$WB_NIX_PLAN
91100
fi
92101

93-
for pkg in ${pkgnames[*]}; do echo \"$pkg\"; done |
94-
jq --slurp '
102+
if test -z "$node_rev"
103+
then node_rev=$(manifest_git_head_commit "$dir"); fi
104+
105+
jq '
95106
def pkg_id:
96107
."pkg-name" + "-" + ."pkg-version"
97108
;
98109
99-
def pkg_commit:
100-
.url | split("?")[0] | split("/")[2]
110+
def commit_from_chap($chap_data):
111+
$chap_data[pkg_id]
112+
# parse the commit from the url
113+
| .url
114+
| strings
115+
| split("?")[0]
116+
| split("/")[2]
117+
;
118+
119+
def commit_from_srp:
120+
."pkg-src"."source-repo".tag
101121
;
102122
103-
# key chap packages by package id, so we can match them easily in the next step
104-
.
105-
# | debug
106-
| (map({ key: ., value: true }) | from_entries) as $pkgs
107-
| $chap
108-
| map({ key: pkg_id, value: pkg_commit })
109-
| from_entries as $chap_data
110-
111-
# map each package in the plan to chap data (if present)
112-
| $plan."install-plan"
113-
| map({ key: ."pkg-name"
114-
, value: { name: ."pkg-name"
115-
, version: ."pkg-version"
116-
, commit: $chap_data[pkg_id]
117-
}
118-
})
119-
| from_entries as $full_package_data
120-
| $full_package_data
121-
| map_values (select(.name | in($pkgs))) as $package_data
122-
123-
# assemble the manifest
124-
| $package_data *
125-
{ "cardano-node": { commit: $node_rev
123+
# index chap packages by package-id
124+
$chapfile[0]
125+
| INDEX(pkg_id) as $chap_data
126+
127+
# index the plan by package name
128+
| $planfile[0]."install-plan"
129+
| INDEX(."pkg-name") as $plan
130+
131+
# take the list of packages from the command line
132+
| $ARGS.positional
133+
# find them in the plan
134+
| JOIN($plan; .)
135+
# produce the desired data
136+
| map(
137+
.[1] # the plan entry
138+
| { name: ."pkg-name"
139+
, version: ."pkg-version"
140+
, commit: (
141+
# if a package is pre-existing we cannot tell where it came from
142+
# to simplify, we take the following approach:
143+
# - if it is from a source-repo use that
144+
# - otherwise see if it matches any package on chap
145+
# - otherwise return null
146+
# Note: in case a package exists on both hackage and chap, this
147+
# will always report the commit from chap
148+
commit_from_srp // commit_from_chap($chap_data) // null
149+
)
150+
}
151+
)
152+
# index again by package name
153+
| INDEX(.name)
154+
# add cardano-node data
155+
* { "cardano-node": { commit: $node_rev
126156
, branch: $node_branch
127157
, status: $node_status
128158
} }
129-
' --arg node_rev $node_rev \
130-
--arg node_branch $(manifest_local_repo_branch "$dir") \
131-
--arg node_status $(manifest_git_checkout_state_desc "$dir") \
132-
--arg pkgs $(manifest_git_checkout_state_desc "$dir") \
133-
--argfile plan $plan_path \
134-
--argfile chap $WB_CHAP_PACKAGES
159+
' --null-input \
160+
--arg node_rev "$node_rev" \
161+
--arg node_branch "$(manifest_local_repo_branch "$dir")" \
162+
--arg node_status "$(manifest_git_checkout_state_desc "$dir")" \
163+
--arg pkgs "$(manifest_git_checkout_state_desc "$dir")" \
164+
--slurpfile planfile "$plan_path" \
165+
--slurpfile chapfile "$WB_CHAP_PATH/foliage/packages.json" \
166+
--args "$@"
135167
;;
136168

137169
render )
138170
local usage="USAGE: wb manifest $0 MANIFEST-JSON-VALUE"
139171
local json=${1:?$usage}
140172

141-
jq 'include "lib";
173+
jq '
174+
include "lib";
142175
143176
def unwords: join(" ");
144177
def unlines: join("\n");

‎nix/workbench/shell.nix‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ with lib;
1919
let
2020

2121
# recover CHaP location from cardano's project
22-
chapPackages = "${project.args.inputMap."https://input-output-hk.github.io/cardano-haskell-packages"}/foliage/packages.json";
22+
chap = project.args.inputMap."https://input-output-hk.github.io/cardano-haskell-packages";
2323

2424
# build plan as computed by nix
2525
nixPlanJson = project.plan-nix.json;
@@ -52,7 +52,7 @@ in project.shellFor {
5252
workbenchDevMode
5353
''
5454
export WB_CARDANO_NODE_REPO_ROOT=$(git rev-parse --show-toplevel)
55-
export WB_CHAP_PACKAGES=${chapPackages}
55+
export WB_CHAP_PATH=${chap}
5656
export WB_NIX_PLAN=${nixPlanJson}
5757
export WB_EXTRA_FLAGS=
5858

‎nix/workbench/wb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ start()
222222
local manifest=$(manifest collect-from-checkout \
223223
"$node_source" \
224224
"$node_rev" \
225-
${WB_MANIFEST_PACKAGES[*]}
225+
"${WB_MANIFEST_PACKAGES[@]}"
226226
)
227227
manifest render "$manifest"
228228

0 commit comments

Comments
 (0)