Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gosum support #475

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
62 changes: 49 additions & 13 deletions extractor/filesystem/language/golang/gomod/gomod.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,38 @@ const (
// including the stdlib version by using the top level go version
//
// The output is not sorted and will not be in a consistent order
type Extractor struct{}
type Extractor struct {
deduplicateSumDependencies bool
}

// Config contains the configuration options for the extractor.
type Config struct {
// DeduplicateSumDependencies controls whether dependencies in the go.sum file
// should be deduplicated against those in the go.mod file.
//
// When two dependencies with the same name and version exist, one in go.mod
// and the other in go.sum:
// - If set to true, only the dependency from go.mod will be retained.
// - If set to false, both dependencies (from go.mod and go.sum) will be kept.
DeduplicateSumDependencies bool
}

// DefaultConfig returns the default configuration for the extractor.
func DefaultConfig() Config {
return Config{
DeduplicateSumDependencies: true,
}
}

// NewDefault returns a new instance of the extractor using the default configuration.
func NewDefault() filesystem.Extractor { return New(DefaultConfig()) }

// New returns a new instance of the extractor.
func New() filesystem.Extractor { return &Extractor{} }
func New(cfg Config) filesystem.Extractor {
return &Extractor{
deduplicateSumDependencies: cfg.DeduplicateSumDependencies,
}
}

// Name of the extractor.
func (e Extractor) Name() string { return Name }
Expand Down Expand Up @@ -81,6 +109,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
type mapKey struct {
name string
version string
isGoSum bool
}
packages := map[mapKey]*extractor.Inventory{}

Expand Down Expand Up @@ -138,13 +167,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
log.Warnf("Error reading go.sum file: %s", err)
} else {
for _, p := range sumPackages {
key := mapKey{name: p.Name, version: p.Version}

// Skip if the package is already present in the go.mod
if _, ok := packages[key]; ok {
continue
}
packages[key] = p
packages[mapKey{name: p.Name, version: p.Version, isGoSum: true}] = p
}
}
}
Expand All @@ -158,11 +181,24 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
}
}

// The map values might have changed after replacement so we need to run another
// deduplication pass.
// An additional deduplication pass is required.
// This is necessary because the values in the map may have changed after the replacement,
// and to ensure that sum dependencies are deduplicated when specified.
dedupedPs := map[mapKey]*extractor.Inventory{}
for _, p := range packages {
dedupedPs[mapKey{name: p.Name, version: p.Version}] = p
for key, p := range packages {
keepGoSumSeparated := !e.deduplicateSumDependencies
s := mapKey{
name: p.Name,
version: p.Version,
isGoSum: key.isGoSum && keepGoSumSeparated,
}

// Do not override `go.mod` dependencies with by `go.sum` ones
if _, ok := dedupedPs[s]; ok && key.isGoSum {
continue
}

dedupedPs[s] = p
}
return maps.Values(dedupedPs), nil
}
Expand Down
64 changes: 62 additions & 2 deletions extractor/filesystem/language/golang/gomod/gomod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,13 @@ func TestExtractor_FileRequired(t *testing.T) {
}

func TestExtractor_Extract(t *testing.T) {
tests := []extracttest.TestTableEntry{
tests := []struct {
ExtractorConfig *gomod.Config
Name string
InputConfig extracttest.ScanInputMockConfig
WantInventory []*extractor.Inventory
WantErr error
}{
{
Name: "invalid",
InputConfig: extracttest.ScanInputMockConfig{
Expand Down Expand Up @@ -322,11 +328,65 @@ func TestExtractor_Extract(t *testing.T) {
},
},
},
{
Name: "test extractor for go <=1.16 without deduplication",
ExtractorConfig: &gomod.Config{
DeduplicateSumDependencies: false,
},
InputConfig: extracttest.ScanInputMockConfig{
Path: "testdata/indirect-1.16.mod",
},
WantInventory: []*extractor.Inventory{
{
Name: "github.com/sirupsen/logrus",
Version: "1.9.3",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/davecgh/go-spew",
Version: "1.1.1",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/pmezard/go-difflib",
Version: "1.0.0",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/sirupsen/logrus",
Version: "1.9.3",
Locations: []string{"testdata/indirect-1.16.mod"},
},
{
Name: "github.com/stretchr/testify",
Version: "1.7.0",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "golang.org/x/sys",
Version: "0.0.0-20220715151400-c0bba94af5f8",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "gopkg.in/yaml.v3",
Version: "3.0.0-20200313102051-9f266ea9e77c",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "stdlib",
Version: "1.16",
Locations: []string{"testdata/indirect-1.16.mod"},
},
},
},
}

for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
extr := gomod.Extractor{}
extr := gomod.NewDefault()
if tt.ExtractorConfig != nil {
extr = gomod.New(*tt.ExtractorConfig)
}

scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig)
defer extracttest.CloseTestScanInput(t, scanInput)
Expand Down
2 changes: 1 addition & 1 deletion extractor/filesystem/list/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ var (
// Go extractors.
Go = InitMap{
gobinary.Name: {gobinary.NewDefault},
gomod.Name: {gomod.New},
gomod.Name: {gomod.NewDefault},
}
// Dart extractors.
Dart = InitMap{pubspec.Name: {pubspec.New}}
Expand Down