Skip to content
This repository was archived by the owner on Jul 18, 2025. It is now read-only.

Commit 8aebcab

Browse files
cdupuiscdupuis
committed
Make cve and base image query async
1 parent ebed97b commit 8aebcab

File tree

9 files changed

+132
-74
lines changed

9 files changed

+132
-74
lines changed

‎commands/cmd.go

Lines changed: 9 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ import (
3232
"github.com/docker/index-cli-plugin/query"
3333
"github.com/docker/index-cli-plugin/sbom"
3434
"github.com/docker/index-cli-plugin/types"
35-
v1 "github.com/google/go-containerregistry/pkg/v1"
3635
"github.com/moby/term"
3736
"github.com/pkg/errors"
3837
"github.com/sirupsen/logrus"
@@ -117,34 +116,14 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
117116
var sb *types.Sbom
118117

119118
if ociDir == "" {
120-
sb, _, err = sbom.IndexImage(image, dockerCli)
119+
sb, err = sbom.IndexImage(image, dockerCli)
121120
} else {
122-
sb, _, err = sbom.IndexPath(ociDir, image, dockerCli)
121+
sb, err = sbom.IndexPath(ociDir, image, dockerCli)
123122
}
124123
if err != nil {
125124
return err
126125
}
127-
if includeCves {
128-
workspace, _ := config.PluginConfig("index", "workspace")
129-
apiKey, _ := config.PluginConfig("index", "api-key")
130-
cves, err := query.QueryCves(sb, "", workspace, apiKey)
131-
if err != nil {
132-
skill.Log.Warnf("error running cve query")
133-
}
134-
if cves != nil {
135-
sb.Vulnerabilities = *cves
136-
}
137-
}
138-
if includeBaseImages {
139-
bi, err := query.ForBaseImageInGraphQL(sb.Source.Image.Config, true)
140-
if err != nil {
141-
skill.Log.Warnf("error running base image query")
142-
}
143-
if bi != nil && len(bi.ImagesByDiffIds) > 0 {
144-
sb.Source.BaseImages = bi.ImagesByDiffIds
145-
}
146-
}
147-
126+
sb = query.ForCvesAndBaseImagesAsync(sb, includeCves, includeBaseImages, "", "")
148127
js, err := json.MarshalIndent(sb, "", " ")
149128
if err != nil {
150129
return err
@@ -190,16 +169,15 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
190169
}
191170

192171
var sb *types.Sbom
193-
var img *v1.Image
194172
if ociDir == "" {
195-
sb, img, err = sbom.IndexImage(image, dockerCli)
173+
sb, err = sbom.IndexImage(image, dockerCli)
196174
} else {
197-
sb, img, err = sbom.IndexPath(ociDir, image, dockerCli)
175+
sb, err = sbom.IndexPath(ociDir, image, dockerCli)
198176
}
199177
if err != nil {
200178
return err
201179
}
202-
err = sbom.UploadSbom(sb, img, workspace, apiKey)
180+
err = sbom.UploadSbom(sb, workspace, apiKey)
203181

204182
return nil
205183
},
@@ -220,12 +198,11 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
220198
cve := args[0]
221199
var err error
222200
var sb *types.Sbom
223-
var img *v1.Image
224201

225202
if ociDir == "" {
226-
sb, img, err = sbom.IndexImage(image, dockerCli)
203+
sb, err = sbom.IndexImage(image, dockerCli)
227204
} else {
228-
sb, img, err = sbom.IndexPath(ociDir, image, dockerCli)
205+
sb, err = sbom.IndexPath(ociDir, image, dockerCli)
229206
}
230207
if err != nil {
231208
return err
@@ -237,7 +214,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
237214
return err
238215
}
239216

240-
format.Cves(cve, cves, img, sb, remediate, dockerCli, workspace, apiKey)
217+
format.Cves(cve, cves, sb, remediate, dockerCli, workspace, apiKey)
241218

242219
if len(*cves) > 0 {
243220
os.Exit(1)

‎format/cve.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,9 @@ import (
2424
"github.com/docker/index-cli-plugin/internal"
2525
"github.com/docker/index-cli-plugin/query"
2626
"github.com/docker/index-cli-plugin/types"
27-
v1 "github.com/google/go-containerregistry/pkg/v1"
2827
)
2928

30-
func Cves(cve string, cves *[]types.Cve, img *v1.Image, sb *types.Sbom, remediate bool, dockerCli command.Cli, workspace string, apiKey string) {
29+
func Cves(cve string, cves *[]types.Cve, sb *types.Sbom, remediate bool, dockerCli command.Cli, workspace string, apiKey string) {
3130
if len(*cves) > 0 {
3231
for _, c := range *cves {
3332
Cve(sb, &c)
@@ -56,7 +55,7 @@ func Cves(cve string, cves *[]types.Cve, img *v1.Image, sb *types.Sbom, remediat
5655
// see if the package comes in via the base image
5756
s := internal.StartInfoSpinner("Detecting base image", dockerCli.Out().IsTerminal())
5857
defer s.Stop()
59-
baseImages, index, _ := query.Detect(img, true, workspace, apiKey)
58+
baseImages, index, _ := query.Detect(sb, true, workspace, apiKey)
6059
s.Stop()
6160
var baseImage *types.Image
6261
if layerIndex <= index && baseImages != nil && len(*baseImages) > 0 {
@@ -71,7 +70,7 @@ func Cves(cve string, cves *[]types.Cve, img *v1.Image, sb *types.Sbom, remediat
7170
if baseImage != nil {
7271
s := internal.StartInfoSpinner("Finding alternative base images", dockerCli.Out().IsTerminal())
7372
defer s.Stop()
74-
aBaseImage, _ := query.ForBaseImageWithoutCve(c.SourceId, baseImage.Repository.Name, img, workspace, apiKey)
73+
aBaseImage, _ := query.ForBaseImageWithoutCve(c.SourceId, baseImage.Repository.Name, sb, workspace, apiKey)
7574
s.Stop()
7675

7776
if aBaseImage != nil && len(*aBaseImage) > 0 {

‎query/async.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright © 2022 Docker, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package query
18+
19+
import (
20+
"sync"
21+
22+
"github.com/docker/index-cli-plugin/types"
23+
)
24+
25+
type queryResult struct {
26+
Cves []types.Cve
27+
BaseImages []types.BaseImage
28+
Error error
29+
}
30+
31+
func ForCvesAndBaseImagesAsync(sb *types.Sbom, includeCves bool, includeBaseImages bool, workspace string, apiKey string) *types.Sbom {
32+
resultChan := make(chan queryResult, 2)
33+
var wg sync.WaitGroup
34+
if includeCves {
35+
wg.Add(1)
36+
go func() {
37+
defer wg.Done()
38+
cves, err := QueryCves(sb, "", workspace, apiKey)
39+
if err != nil {
40+
resultChan <- queryResult{
41+
Error: err,
42+
}
43+
}
44+
if cves != nil {
45+
resultChan <- queryResult{
46+
Cves: *cves,
47+
}
48+
}
49+
}()
50+
}
51+
if includeBaseImages {
52+
wg.Add(1)
53+
go func() {
54+
defer wg.Done()
55+
bi, err := ForBaseImageInGraphQL(sb.Source.Image.Config, true)
56+
if err != nil {
57+
resultChan <- queryResult{
58+
Error: err,
59+
}
60+
}
61+
if bi != nil && len(bi.ImagesByDiffIds) > 0 {
62+
resultChan <- queryResult{
63+
BaseImages: bi.ImagesByDiffIds,
64+
}
65+
}
66+
}()
67+
}
68+
wg.Wait()
69+
close(resultChan)
70+
71+
for result := range resultChan {
72+
if result.BaseImages != nil {
73+
sb.Source.BaseImages = result.BaseImages
74+
}
75+
if result.Cves != nil {
76+
sb.Vulnerabilities = result.Cves
77+
}
78+
}
79+
80+
return sb
81+
}

‎query/base.go

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,11 @@ var baseImageCveQuery string
5757
//go:embed repository_query.edn
5858
var repositoryQuery string
5959

60-
func Detect(img *v1.Image, excludeSelf bool, workspace string, apiKey string) (*[]types.Image, int, error) {
60+
func Detect(sb *types.Sbom, excludeSelf bool, workspace string, apiKey string) (*[]types.Image, int, error) {
6161
digests := make([]digest.Digest, 0)
62-
layers, _ := (*img).Layers()
62+
layers := (*sb).Source.Image.Config.RootFS.DiffIDs
6363
for _, layer := range layers {
64-
d, _ := layer.DiffID()
65-
parsed, _ := digest.Parse(d.String())
64+
parsed, _ := digest.Parse(layer.String())
6665
digests = append(digests, parsed)
6766
}
6867
if excludeSelf {
@@ -134,8 +133,8 @@ func ForBaseImageInIndex(digest digest.Digest, workspace string, apiKey string)
134133
return nil, nil
135134
}
136135

137-
func ForBaseImageWithoutCve(cve string, name string, img *v1.Image, workspace string, apiKey string) (*[]types.Image, error) {
138-
cf, _ := (*img).ConfigFile()
136+
func ForBaseImageWithoutCve(cve string, name string, sb *types.Sbom, workspace string, apiKey string) (*[]types.Image, error) {
137+
cf := (*sb).Source.Image.Config
139138
resp, err := query(fmt.Sprintf(baseImageCveQuery, cve, name, cf.OS, cf.Architecture, cf.Variant), "base_image_cve_query", workspace, apiKey)
140139

141140
var result ImageQueryResult
@@ -224,7 +223,7 @@ func ForBaseImageInGraphQL(cfg *v1.ConfigFile, excludeSelf bool) (*types.BaseIma
224223
diffIds = append(diffIds, graphql.ID(d.String()))
225224
}
226225
if excludeSelf {
227-
diffIds = diffIds[0 : len(diffIds)-1]
226+
// diffIds = diffIds[0 : len(diffIds)-1]
228227
}
229228

230229
if len(diffIds) == 0 {

‎registry/save.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func (c *ImageCache) StoreImage() error {
9797
skill.Log.Infof("Copied image")
9898
return nil
9999
} else if format == "tar" {
100-
u := make(chan v1.Update, 200)
100+
u := make(chan v1.Update, 0)
101101
errchan := make(chan error)
102102
go func() {
103103
if err := tarball.WriteToFile(c.ImagePath, *c.Ref, *c.Image, tarball.WithProgress(u)); err != nil {

‎sbom/index.go

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -37,53 +37,58 @@ import (
3737

3838
type ImageIndexResult struct {
3939
Input string
40-
Image *v1.Image
4140
Sbom *types.Sbom
4241
Error error
4342
}
4443

4544
func indexImageAsync(wg *sync.WaitGroup, image string, cli command.Cli, resultChan chan<- ImageIndexResult) {
4645
defer wg.Done()
47-
sbom, img, err := IndexImage(image, cli)
46+
sbom, err := IndexImage(image, cli)
4847
cves, err := query.QueryCves(sbom, "", "", "")
4948
if err == nil {
5049
sbom.Vulnerabilities = *cves
5150
}
5251
resultChan <- ImageIndexResult{
5352
Input: image,
54-
Image: img,
5553
Sbom: sbom,
5654
Error: err,
5755
}
5856
}
5957

60-
func IndexPath(path string, name string, cli command.Cli) (*types.Sbom, *v1.Image, error) {
58+
func IndexPath(path string, name string, cli command.Cli) (*types.Sbom, error) {
6159
cache, err := registry.ReadImage(name, path)
6260
if err != nil {
63-
return nil, nil, errors.Wrap(err, "failed to read image")
61+
return nil, errors.Wrap(err, "failed to read image")
6462
}
6563
return indexImage(cache, cli)
6664
}
6765

68-
func IndexImage(image string, cli command.Cli) (*types.Sbom, *v1.Image, error) {
66+
func IndexImage(image string, cli command.Cli) (*types.Sbom, error) {
67+
if strings.HasPrefix(image, "sha256:") {
68+
configFilePath := cli.ConfigFile().Filename
69+
sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "sbom", "sha256", image[7:], "sbom.json")
70+
if sbom := cachedSbom(sbomFilePath); sbom != nil {
71+
return sbom, nil
72+
}
73+
}
6974
cache, err := registry.SaveImage(image, cli)
7075
if err != nil {
71-
return nil, nil, errors.Wrap(err, "failed to copy image")
76+
return nil, errors.Wrap(err, "failed to copy image")
7277
}
7378
return indexImage(cache, cli)
7479
}
7580

76-
func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, *v1.Image, error) {
81+
func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error) {
7782
configFilePath := cli.ConfigFile().Filename
7883
sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "sbom", "sha256", cache.Digest[7:], "sbom.json")
79-
if sbom := cachedSbom(cache, sbomFilePath); sbom != nil {
80-
return sbom, cache.Image, nil
84+
if sbom := cachedSbom(sbomFilePath); sbom != nil {
85+
return sbom, nil
8186
}
8287

8388
err := cache.StoreImage()
8489
defer cache.Cleanup()
8590
if err != nil {
86-
return nil, nil, errors.Wrapf(err, "failed to copy image")
91+
return nil, errors.Wrapf(err, "failed to copy image")
8792
}
8893

8994
lm := createLayerMapping(*cache.Image)
@@ -101,7 +106,7 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, *v1.I
101106
trivyResult.Packages, err = types.NormalizePackages(trivyResult.Packages)
102107
syftResult.Packages, err = types.NormalizePackages(syftResult.Packages)
103108
if err != nil {
104-
return nil, nil, errors.Wrapf(err, "failed to normalize packagess: %s", cache.Name)
109+
return nil, errors.Wrapf(err, "failed to normalize packagess: %s", cache.Name)
105110
}
106111

107112
packages := types.MergePackages(syftResult, trivyResult)
@@ -119,7 +124,7 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, *v1.I
119124
if cache.Name != "" {
120125
ref, err := name.ParseReference(cache.Name)
121126
if err != nil {
122-
return nil, nil, errors.Wrapf(err, "failed to parse reference: %s", cache.Name)
127+
return nil, errors.Wrapf(err, "failed to parse reference: %s", cache.Name)
123128
}
124129
cache.Name = ref.Context().String()
125130
if !strings.HasPrefix(ref.Identifier(), "sha256:") {
@@ -162,18 +167,18 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, *v1.I
162167
if err == nil {
163168
err = os.MkdirAll(filepath.Dir(sbomFilePath), os.ModePerm)
164169
if err != nil {
165-
return nil, nil, errors.Wrapf(err, "failed create to sbom folder")
170+
return nil, errors.Wrapf(err, "failed create to sbom folder")
166171
}
167172
err = os.WriteFile(sbomFilePath, js, 0644)
168173
if err != nil {
169-
return nil, nil, errors.Wrapf(err, "failed to write sbom")
174+
return nil, errors.Wrapf(err, "failed to write sbom")
170175
}
171176
}
172177

173-
return &sbom, cache.Image, nil
178+
return &sbom, nil
174179
}
175180

176-
func cachedSbom(cache *registry.ImageCache, sbomFilePath string) *types.Sbom {
181+
func cachedSbom(sbomFilePath string) *types.Sbom {
177182
// see if we can re-use an existing sbom
178183
if _, ok := os.LookupEnv("ATOMIST_NO_CACHE"); !ok {
179184
if _, err := os.Stat(sbomFilePath); !os.IsNotExist(err) {

‎sbom/monitor.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func WatchImages(cli command.Cli) error {
4444

4545
func indexImageWorker(cli command.Cli, indexJobs <-chan types.ImageSummary) {
4646
for img := range indexJobs {
47-
_, _, err := IndexImage(img.ID, cli)
47+
_, err := IndexImage(img.ID, cli)
4848
if err != nil {
4949
skill.Log.Warnf("Failed to index image %s", img.ID)
5050
delete(imageCache, img.ID)

‎sbom/upload.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,13 @@ import (
3434
)
3535

3636
// UploadSbom transact an image and its data into the data plane
37-
func UploadSbom(sb *types.Sbom, img *v1.Image, workspace string, apikey string) error {
37+
func UploadSbom(sb *types.Sbom, workspace string, apikey string) error {
3838
host, name, err := parseReference(sb)
3939
if err != nil {
4040
return errors.Wrapf(err, "failed to obtain host and repository")
4141
}
42-
config, err := (*img).ConfigFile()
43-
if err != nil {
44-
return errors.Wrapf(err, "failed to obtain config")
45-
}
46-
manifest, err := (*img).Manifest()
47-
if err != nil {
48-
return errors.Wrapf(err, "failed to obtain manifest")
49-
}
42+
config := (*sb).Source.Image.Config
43+
manifest := (*sb).Source.Image.Manifest
5044

5145
now := time.Now()
5246
correlationId := uuid.NewString()

0 commit comments

Comments
 (0)