Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gosum support #475

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
70 changes: 69 additions & 1 deletion extractor/filesystem/language/golang/gomod/gomod.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,17 @@
package gomod

import (
"bufio"
"context"
"fmt"
"go/version"
"io"
"path/filepath"
"strings"

"github.com/google/osv-scalibr/extractor"
"github.com/google/osv-scalibr/extractor/filesystem"
"github.com/google/osv-scalibr/log"
"github.com/google/osv-scalibr/plugin"
"github.com/google/osv-scalibr/purl"
"golang.org/x/exp/maps"
Expand Down Expand Up @@ -112,8 +115,25 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
}
}

isGoVersionSpecified := parsedLockfile.Go != nil && parsedLockfile.Go.Version != ""

// At go 1.17 and above, the go command adds an indirect requirement for each module that provides any
// package imported (even indirectly) by a package or test in the main module or passed as an argument to go get.
//
// for versions below extract indirect dependencies from the go.sum file
if isGoVersionSpecified && version.Compare("go"+parsedLockfile.Go.Version, "go1.17") < 0 {
sumPackages, err := extractFromSum(input)
if err != nil {
log.Warnf("Error reading go.sum file: %s", err)
} else {
for _, p := range sumPackages {
packages[mapKey{name: p.Name, version: p.Version}] = p
}
}
}

// Add the Go stdlib as an explicit dependency.
if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" {
if isGoVersionSpecified {
packages[mapKey{name: "stdlib"}] = &extractor.Inventory{
Name: "stdlib",
Version: parsedLockfile.Go.Version,
Expand Down Expand Up @@ -144,4 +164,52 @@ func (e Extractor) Ecosystem(i *extractor.Inventory) string {
return "Go"
}

// extractFromSum extracts dependencies from the go.sum file.
//
// Note: This function may produce false positives, as the go.sum file might be outdated.
func extractFromSum(input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
goSumPath := strings.TrimSuffix(input.Path, ".mod") + ".sum"
f, err := input.FS.Open(goSumPath)
if err != nil {
return nil, err
}

scanner := bufio.NewScanner(f)
packages := []*extractor.Inventory{}

for lineNumber := 0; scanner.Scan(); lineNumber++ {
line := scanner.Text()

if line == "" {
continue
}

parts := strings.Fields(line)
if len(parts) != 3 {
return nil, fmt.Errorf("Error reading go.sum file: wrongly formatted line %s:%d", goSumPath, lineNumber)
}

name := parts[0]
version := strings.TrimPrefix(parts[1], "v")

// skip a line if the version contains "/go.mod" because lines
// containing "/go.mod" are duplicates used to verify the hash of the go.mod file
if strings.Contains(version, "/go.mod") {
continue
}

packages = append(packages, &extractor.Inventory{
Name: name,
Version: version,
Locations: []string{goSumPath},
})
}

if err := scanner.Err(); err != nil {
return nil, err
}

return packages, nil
}

var _ filesystem.Extractor = Extractor{}
66 changes: 66 additions & 0 deletions extractor/filesystem/language/golang/gomod/gomod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,72 @@ func TestExtractor_Extract(t *testing.T) {
},
},
},
{
Name: "test extractor for go > 1.16",
InputConfig: extracttest.ScanInputMockConfig{
Path: "testdata/indirect-1.23.mod",
},
WantInventory: []*extractor.Inventory{
{
Name: "github.com/sirupsen/logrus",
Version: "1.9.3",
Locations: []string{"testdata/indirect-1.23.mod"},
},
{
Name: "golang.org/x/sys",
Version: "0.0.0-20220715151400-c0bba94af5f8",
Locations: []string{"testdata/indirect-1.23.mod"},
},
{
Name: "stdlib",
Version: "1.23",
Locations: []string{"testdata/indirect-1.23.mod"},
},
},
},
{
Name: "test extractor for go <=1.16",
InputConfig: extracttest.ScanInputMockConfig{
Path: "testdata/indirect-1.16.mod",
},
WantInventory: []*extractor.Inventory{
{
Name: "github.com/davecgh/go-spew",
Version: "1.1.1",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/pmezard/go-difflib",
Version: "1.0.0",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/sirupsen/logrus",
Version: "1.9.3",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "github.com/stretchr/testify",
Version: "1.7.0",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "golang.org/x/sys",
Version: "0.0.0-20220715151400-c0bba94af5f8",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "gopkg.in/yaml.v3",
Version: "3.0.0-20200313102051-9f266ea9e77c",
Locations: []string{"testdata/indirect-1.16.sum"},
},
{
Name: "stdlib",
Version: "1.16",
Locations: []string{"testdata/indirect-1.16.mod"},
},
},
},
}

for _, tt := range tests {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module test

go 1.16

require github.com/sirupsen/logrus v1.9.3
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
module test

go 1.23

require github.com/sirupsen/logrus v1.9.3

require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Wrongly formatted, this file should not be used!!!
Loading