feat: add PyPI clients for transitive extraction

clients/datasource/internal/pypi/pypi.go

@@ -0,0 +1,40 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pypi defines the structures to parse PyPI JSON API response.
+package pypi
+
+// Response defines the response of PyPI JSON API.
+type Response struct {
+	Info     Info     `json:"info"`
+	Releases Releases `json:"releases"`
+}
+
+// Info holds the info in the response of PyPI JSON API.
+type Info struct {
+	RequiresDist []string `json:"requires_dist"`
+}
+
+// Release is a map of version to its release information.
+type Releases map[string][]Release
+
+// Release holds the information about a release.
+type Release struct {
+	Digests Digests `json:"digests"`
+}
+
+// Digests holds the information of digests of a release.
+type Digests struct {
+	SHA256 string `json:"sha256"`
+}

clients/datasource/pypi_registry.go

@@ -0,0 +1,96 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datasource
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/google/osv-scalibr/clients/datasource/internal/pypi"
+)
+
+// PyPIAPI holds the base of the URL of PyPI JSON API.
+const PyPIAPI = "https://pypi.org/pypi/"
+
+// PyPIRegistryAPIClient defines a client to fetch metadata from a PyPI registry.
+type PyPIRegistryAPIClient struct {
+	registry string
+
+	// Cache fields
+	mu             *sync.Mutex
+	cacheTimestamp *time.Time // If set, this means we loaded from a cache
+	responses      *RequestCache[string, pypi.Response]
+}
+
+// NewPyPIRegistryAPIClient returns a new PyPIRegistryAPIClient.
+func NewPyPIRegistryAPIClient(registry string) *PyPIRegistryAPIClient {
+	return &PyPIRegistryAPIClient{
+		registry:  registry,
+		mu:        &sync.Mutex{},
+		responses: NewRequestCache[string, pypi.Response](),
+	}
+}
+
+func (p *PyPIRegistryAPIClient) GetPackageInfo(ctx context.Context, name string) (pypi.Response, error) {
+	path, err := url.JoinPath(p.registry, name, "json")
+	if err != nil {
+		return pypi.Response{}, err
+	}
+	return p.get(ctx, path)
+}
+
+func (p *PyPIRegistryAPIClient) GetVersionInfo(ctx context.Context, name, version string) (pypi.Response, error) {
+	path, err := url.JoinPath(p.registry, name, version, "json")
+	if err != nil {
+		return pypi.Response{}, err
+	}
+	return p.get(ctx, path)
+}
+
+func (p *PyPIRegistryAPIClient) get(ctx context.Context, url string) (pypi.Response, error) {
+	return p.responses.Get(url, func() (pypi.Response, error) {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			return pypi.Response{}, err
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return pypi.Response{}, fmt.Errorf("%w: PyPI registry query failed: %w", errAPIFailed, err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			return pypi.Response{}, errors.New(resp.Status)
+		}
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return pypi.Response{}, err
+		}
+
+		var result pypi.Response
+		err = json.Unmarshal(body, &result)
+
+		return result, err
+	})
+}

clients/datasource/pypi_registry_cache.go

@@ -0,0 +1,65 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datasource
+
+import (
+	"time"
+
+	"github.com/google/osv-scalibr/clients/datasource/internal/pypi"
+)
+
+type pypiRegistryCache struct {
+	Timestamp *time.Time
+	Responses map[string]pypi.Response // url -> response
+}
+
+// GobEncode encodes cache to bytes.
+func (p *PyPIRegistryAPIClient) GobEncode() ([]byte, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.cacheTimestamp == nil {
+		now := time.Now().UTC()
+		p.cacheTimestamp = &now
+	}
+
+	cache := pypiRegistryCache{
+		Timestamp: p.cacheTimestamp,
+		Responses: p.responses.GetMap(),
+	}
+
+	return gobMarshal(&cache)
+}
+
+// GobDecode encodes bytes to cache.
+func (p *PyPIRegistryAPIClient) GobDecode(b []byte) error {
+	var cache pypiRegistryCache
+	if err := gobUnmarshal(b, &cache); err != nil {
+		return err
+	}
+
+	if cache.Timestamp != nil && time.Since(*cache.Timestamp) >= cacheExpiry {
+		// Cache expired
+		return nil
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.cacheTimestamp = cache.Timestamp
+	p.responses.SetMap(cache.Responses)
+
+	return nil
+}

clients/datasource/pypi_registry_test.go

@@ -0,0 +1,123 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datasource_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/osv-scalibr/clients/clienttest"
+	"github.com/google/osv-scalibr/clients/datasource"
+	"github.com/google/osv-scalibr/clients/datasource/internal/pypi"
+)
+
+func TestGetPackageInfo(t *testing.T) {
+	srv := clienttest.NewMockHTTPServer(t)
+	client := datasource.NewPyPIRegistryAPIClient(srv.URL)
+	srv.SetResponse(t, "abc/json", []byte(`
+	{
+		"releases": {
+			"1.0.0": [
+				{
+					"digests": {
+						"sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
+					}
+				}
+			],
+			"2.0.0": [
+				{
+					"digests": {
+						"sha256": "7259c75c05335b71328905296c425026859546059b02a6144e59049755452292"
+					}
+				},
+				{
+					"digests": {
+						"sha256": "47250e501570778c18251e06497f5664a7538d61895a97576f30a47d21c430e7"
+					}
+				}
+			],
+			"3.0.0": [
+				{
+					"digests": {
+						"sha256": "93f53801a2434547b74457e7c53d3663a898492f25492af7e721a9557b4b10b0"
+					}
+				}
+			]
+		}
+	}
+	`))
+
+	got, err := client.GetPackageInfo(context.Background(), "abc")
+	if err != nil {
+		t.Fatalf("failed to get PyPI package info %s: %v", "abc", err)
+	}
+	want := pypi.Response{
+		Releases: pypi.Releases{
+			"1.0.0": []pypi.Release{
+				{Digests: pypi.Digests{SHA256: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"}},
+			},
+			"2.0.0": []pypi.Release{
+				{Digests: pypi.Digests{SHA256: "7259c75c05335b71328905296c425026859546059b02a6144e59049755452292"}},
+				{Digests: pypi.Digests{SHA256: "47250e501570778c18251e06497f5664a7538d61895a97576f30a47d21c430e7"}},
+			},
+			"3.0.0": []pypi.Release{
+				{Digests: pypi.Digests{SHA256: "93f53801a2434547b74457e7c53d3663a898492f25492af7e721a9557b4b10b0"}},
+			},
+		},
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("GetPackageInfo(%s) mismatch (-want +got):\n%s", "abc", diff)
+	}
+}
+
+func TestGetVersionInfo(t *testing.T) {
+	srv := clienttest.NewMockHTTPServer(t)
+	client := datasource.NewPyPIRegistryAPIClient(srv.URL)
+	srv.SetResponse(t, "abc/1.0.0/json", []byte(`
+	{
+		"info": {
+			"requires_dist": [
+				"charset-normalizer (<4,>=2)",
+				"idna (<4,>=2.5)",
+				"urllib3 (<3,>=1.21.1)",
+				"certifi (>=2017.4.17)",
+				"PySocks (!=1.5.7,>=1.5.6) ; extra == 'socks'",
+				"chardet (<6,>=3.0.2) ; extra == 'use_chardet_on_py3'"
+			]
+		}
+	}
+	`))
+
+	got, err := client.GetVersionInfo(context.Background(), "abc", "1.0.0")
+	if err != nil {
+		t.Fatalf("failed to get PyPI version info %s %s: %v", "abc", "1.0.0", err)
+	}
+	want := pypi.Response{
+		Info: pypi.Info{
+			RequiresDist: []string{
+				"charset-normalizer (<4,>=2)",
+				"idna (<4,>=2.5)",
+				"urllib3 (<3,>=1.21.1)",
+				"certifi (>=2017.4.17)",
+				"PySocks (!=1.5.7,>=1.5.6) ; extra == 'socks'",
+				"chardet (<6,>=3.0.2) ; extra == 'use_chardet_on_py3'",
+			},
+		},
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("GetVersionInfo(%s, %s) mismatch (-want +got):\n%s", "abc", "1.0.0", diff)
+	}
+}