pass and init proxy settings in dulwich and pygit2 (#378)

* pass and init proxy settings in dulwich and pygit2

* add tests, fix more methods in dulwich

* lint

* debug tests

* drop pytest socket

* fix windows path to gitconfig

Author: shcheklein

.gitignore

@@ -138,3 +138,4 @@ dmypy.json
 cython_debug/
 
 .DS_Store
+.vscode/

pyproject.toml

@@ -48,6 +48,7 @@ tests = [
     "pytest-mock",
     "pytest-sugar",
     "pytest-test-utils>=0.1.0,<0.2",
+    "proxy.py",
 ]
 dev = [
     "mypy==1.11.2",
@@ -73,6 +74,7 @@ markers = [
 ]
 asyncio_mode = "auto"
 
+
 [tool.coverage.run]
 branch = true
 source = ["scmrepo", "tests"]

src/scmrepo/git/backend/dulwich/__init__.py

@@ -16,6 +16,7 @@
     Union,
 )
 
+from dulwich.config import ConfigFile, StackedConfig
 from funcy import cached_property, reraise
 
 from scmrepo.exceptions import AuthError, CloneError, InvalidRemote, RevError, SCMError
@@ -27,7 +28,7 @@
 
 if TYPE_CHECKING:
     from dulwich.client import SSHVendor
-    from dulwich.config import ConfigFile, StackedConfig
+    from dulwich.config import ConfigFile
     from dulwich.repo import Repo
 
     from scmrepo.git.objects import GitCommit
@@ -579,7 +580,8 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None, **kwargs):
 
         try:
             _remote, location = get_remote_repo(self.repo, url)
-            client, path = get_transport_and_path(location, **kwargs)
+            _config = kwargs.pop("config", StackedConfig.default())
+            client, path = get_transport_and_path(location, config=_config, **kwargs)
         except Exception as exc:
             raise InvalidRemote(url) from exc
 
@@ -616,7 +618,8 @@ def push_refspecs(  # noqa: C901
 
         try:
             _remote, location = get_remote_repo(self.repo, url)
-            client, path = get_transport_and_path(location, **kwargs)
+            _config = kwargs.pop("config", StackedConfig.default())
+            client, path = get_transport_and_path(location, config=_config, **kwargs)
         except Exception as exc:
             raise SCMError(f"'{url}' is not a valid Git remote or URL") from exc
 
@@ -723,7 +726,8 @@ def determine_wants(
 
         with reraise(Exception, SCMError(f"'{url}' is not a valid Git remote or URL")):
             _remote, location = get_remote_repo(self.repo, url)
-            client, path = get_transport_and_path(location, **kwargs)
+            _config = kwargs.pop("config", StackedConfig.default())
+            client, path = get_transport_and_path(location, config=_config, **kwargs)
 
         with reraise(
             (NotGitRepository, KeyError),
@@ -909,6 +913,7 @@ def validate_git_remote(self, url: str, **kwargs):
 
         try:
             _, location = get_remote_repo(self.repo, url)
+            _config = kwargs.pop("config", StackedConfig.default())
             client, path = get_transport_and_path(location, **kwargs)
         except Exception as exc:
             raise InvalidRemote(url) from exc

src/scmrepo/git/backend/pygit2/__init__.py

@@ -701,15 +701,13 @@ def _default_status(
                     remote_refs: dict[str, Oid] = (
                         {
                             head["name"]: head["oid"]
-                            for head in remote.ls_remotes(callbacks=cb)
+                            for head in remote.ls_remotes(callbacks=cb, proxy=True)
                         }
                         if not force
                         else {}
                     )
                     remote.fetch(
-                        refspecs=refspecs,
-                        callbacks=cb,
-                        message="fetch",
+                        refspecs=refspecs, callbacks=cb, message="fetch", proxy=True
                     )
 
             result: dict[str, SyncStatus] = {}

tests/test_git.py

@@ -1,27 +1,41 @@
 import os
 import shutil
+from pathlib import Path
 from typing import Any, Optional
 
 import pytest
 from asyncssh import SFTPClient
 from asyncssh.connection import SSHClientConnection
 from dulwich.client import LocalGitClient
 from git import Repo as GitPythonRepo
+from proxy import TestCase as ProxyTestCase
 from pygit2 import GitError
 from pygit2.remotes import Remote
 from pytest_mock import MockerFixture
 from pytest_test_utils import TempDirFactory, TmpDir
 from pytest_test_utils.matchers import Matcher
 
-from scmrepo.exceptions import InvalidRemote, MergeConflictError, RevError, SCMError
-from scmrepo.git import Git
+from scmrepo.exceptions import (
+    InvalidRemote,
+    MergeConflictError,
+    RevError,
+    SCMError,
+)
+from scmrepo.git import Git, GitBackends
 from scmrepo.git.objects import GitTag
 
 from .conftest import backends
 
 # pylint: disable=redefined-outer-name,unused-argument,protected-access
 
 
+BAD_PROXY_CONFIG = """[http]
+proxy = "http://bad-proxy.dvc.org"
+[https]
+proxy = "http://bad-proxy.dvc.org"
+"""
+
+
 @pytest.fixture
 def submodule_dir(tmp_dir: TmpDir, scm: Git):
     scm.commit("init")
@@ -941,6 +955,84 @@ def test_clone(
         assert fobj.read().strip() == "foo"
 
 
+@pytest.fixture
+def proxy_server():
+    class _ProxyServer(ProxyTestCase):
+        pass
+
+    _ProxyServer.setUpClass()
+    yield f"http://{_ProxyServer.PROXY.flags.hostname}:{_ProxyServer.PROXY.flags.port}"
+    _ProxyServer.tearDownClass()
+
+
+def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir):
+    url = "https://github.com/iterative/dvcyaml-schema"
+
+    p = (
+        Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"])
+        / ".gitconfig"
+    )
+    p.write_text(BAD_PROXY_CONFIG)
+    with pytest.raises(Exception):  # noqa: PT011, B017
+        git.clone(url, "dir")
+
+    mock_config_content = f"""[http]\n
+proxy = {proxy_server}
+[https]
+proxy = {proxy_server}
+"""
+
+    p.write_text(mock_config_content)
+    git.clone(url, "dir")
+
+
+def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: TmpDir):
+    url = "https://github.com/iterative/dvcyaml-schema"
+    git = GitBackends.DEFAULT["dulwich"](".")
+
+    p = (
+        Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"])
+        / ".gitconfig"
+    )
+    p.write_text(BAD_PROXY_CONFIG)
+    with pytest.raises(Exception):  # noqa: PT011, B017
+        list(git.iter_remote_refs(url))
+
+    mock_config_content = f"""[http]
+proxy = {proxy_server}
+[https]
+proxy = {proxy_server}
+"""
+
+    p.write_text(mock_config_content)
+    res = list(git.iter_remote_refs(url))
+    assert res
+
+
+@pytest.mark.skip_git_backend("gitpython")
+def test_fetch_refspecs_proxy_server(
+    proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir
+):
+    url = "https://github.com/iterative/dvcyaml-schema"
+
+    p = (
+        Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"])
+        / ".gitconfig"
+    )
+    p.write_text(BAD_PROXY_CONFIG)
+    with pytest.raises(Exception):  # noqa: PT011, B017
+        git.fetch_refspecs(url, ["refs/heads/master:refs/heads/master"])
+
+    mock_config_content = f"""[http]
+proxy = {proxy_server}
+[https]
+proxy = {proxy_server}
+"""
+
+    p.write_text(mock_config_content)
+    git.fetch_refspecs(url, "refs/heads/master:refs/heads/master")
+
+
 @pytest.mark.skip_git_backend("pygit2")
 def test_fetch(tmp_dir: TmpDir, scm: Git, git: Git, tmp_dir_factory: TempDirFactory):
     tmp_dir.gen("foo", "foo")