Gosum support

extractor/filesystem/language/golang/gomod/gomod.go

@@ -16,14 +16,17 @@
 package gomod
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"go/version"
 	"io"
 	"path/filepath"
 	"strings"
 
 	"github.com/google/osv-scalibr/extractor"
 	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/log"
 	"github.com/google/osv-scalibr/plugin"
 	"github.com/google/osv-scalibr/purl"
 	"golang.org/x/exp/maps"
@@ -112,8 +115,25 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
 		}
 	}
 
+	isGoVersionSpecified := parsedLockfile.Go != nil && parsedLockfile.Go.Version != ""
+
+	// At go 1.17 and above, the go command adds an indirect requirement for each module that provides any
+	// package imported (even indirectly) by a package or test in the main module or passed as an argument to go get.
+	//
+	// for versions below extract indirect dependencies from the go.sum file
+	if isGoVersionSpecified && version.Compare("go"+parsedLockfile.Go.Version, "go1.17") < 0 {
+		sumPackages, err := extractFromSum(input)
+		if err != nil {
+			log.Warnf("Error reading go.sum file: %s", err)
+		} else {
+			for _, p := range sumPackages {
+				packages[mapKey{name: p.Name, version: p.Version}] = p
+			}
+		}
+	}
+
 	// Add the Go stdlib as an explicit dependency.
-	if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" {
+	if isGoVersionSpecified {
 		packages[mapKey{name: "stdlib"}] = &extractor.Inventory{
 			Name:      "stdlib",
 			Version:   parsedLockfile.Go.Version,
@@ -144,4 +164,52 @@ func (e Extractor) Ecosystem(i *extractor.Inventory) string {
 	return "Go"
 }
 
+// extractFromSum extracts dependencies from the go.sum file.
+//
+// Note: This function may produce false positives, as the go.sum file might be outdated.
+func extractFromSum(input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
+	goSumPath := strings.TrimSuffix(input.Path, ".mod") + ".sum"
+	f, err := input.FS.Open(goSumPath)
+	if err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(f)
+	packages := []*extractor.Inventory{}
+
+	for lineNumber := 0; scanner.Scan(); lineNumber++ {
+		line := scanner.Text()
+
+		if line == "" {
+			continue
+		}
+
+		parts := strings.Fields(line)
+		if len(parts) != 3 {
+			return nil, fmt.Errorf("Error reading go.sum file: wrongly formatted line %s:%d", goSumPath, lineNumber)
+		}
+
+		name := parts[0]
+		version := strings.TrimPrefix(parts[1], "v")
+
+		// skip a line if the version contains "/go.mod" because lines
+		// containing "/go.mod" are duplicates used to verify the hash of the go.mod file
+		if strings.Contains(version, "/go.mod") {
+			continue
+		}
+
+		packages = append(packages, &extractor.Inventory{
+			Name:      name,
+			Version:   version,
+			Locations: []string{goSumPath},
+		})
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return packages, nil
+}
+
 var _ filesystem.Extractor = Extractor{}

extractor/filesystem/language/golang/gomod/gomod_test.go

@@ -256,6 +256,72 @@ func TestExtractor_Extract(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "test extractor for go > 1.16",
+			InputConfig: extracttest.ScanInputMockConfig{
+				Path: "testdata/indirect-1.23.mod",
+			},
+			WantInventory: []*extractor.Inventory{
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+				{
+					Name:      "golang.org/x/sys",
+					Version:   "0.0.0-20220715151400-c0bba94af5f8",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+				{
+					Name:      "stdlib",
+					Version:   "1.23",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+			},
+		},
+		{
+			Name: "test extractor for go <=1.16",
+			InputConfig: extracttest.ScanInputMockConfig{
+				Path: "testdata/indirect-1.16.mod",
+			},
+			WantInventory: []*extractor.Inventory{
+				{
+					Name:      "github.com/davecgh/go-spew",
+					Version:   "1.1.1",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/pmezard/go-difflib",
+					Version:   "1.0.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/stretchr/testify",
+					Version:   "1.7.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "golang.org/x/sys",
+					Version:   "0.0.0-20220715151400-c0bba94af5f8",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "gopkg.in/yaml.v3",
+					Version:   "3.0.0-20200313102051-9f266ea9e77c",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "stdlib",
+					Version:   "1.16",
+					Locations: []string{"testdata/indirect-1.16.mod"},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.mod

@@ -0,0 +1,5 @@
+module test
+
+go 1.16
+
+require github.com/sirupsen/logrus v1.9.3

extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.sum

@@ -0,0 +1,15 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.mod

@@ -0,0 +1,7 @@
+module test
+
+go 1.23
+
+require github.com/sirupsen/logrus v1.9.3
+
+require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect

extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.sum

@@ -0,0 +1 @@
+Wrongly formatted, this file should not be used!!!