Merge branch 'dev' into windbg

Author: peace-maker

.github/workflows/ci.yml

@@ -19,6 +19,13 @@ jobs:
         git fetch origin
         git log --oneline --graph -10
 
+    - name: Install RPyC for gdb
+      run: |
+        # The version packaged in python3-rpyc is too old on Ubuntu 22.04
+        sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
+        /usr/bin/python -m pip install rpyc
+        gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
+
     - name: Cache for pip
       uses: actions/cache@v4
       id: cache-pip
@@ -62,7 +69,6 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          python3-rpyc \
           gdb gdbserver socat \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
@@ -112,6 +118,10 @@ jobs:
 
     - name: Coverage doctests
       run: |
+        # Python version installed using setup-python interferes with gdb's python
+        # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules
+        # like _socket. This is a workaround.
+        unset LD_LIBRARY_PATH
         PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples

CHANGELOG.md

@@ -9,9 +9,10 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.13.0](#4130-dev)  | `dev`    |
-| [4.12.0](#4120-beta)  | `beta`   |
-| [4.11.1](#4111-stable)  | `stable` | Nov 14, 2023
+| [4.14.0](#4140-dev)  | `dev`   |
+| [4.13.0](#4130-beta)  | `beta`    |
+| [4.12.0](#4120-stable)  | `stable`   | Feb 22, 2024
+| [4.11.1](#4111)  |          | Nov 14, 2023
 | [4.11.0](#4110)  |          | Sep 15, 2023
 | [4.10.0](#4100)  |          | May 21, 2023
 | [4.9.0](#490)    |          | Dec 29, 2022
@@ -68,7 +69,19 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.13.0 (`dev`)
+## 4.14.0 (`dev`)
+
+- [#2360][2360] Add offline parameter for `search_by_hash` series function
+- [#2356][2356] Add local libc database provider for libcdb
+- [#2374][2374] libcdb.unstrip_libc: debug symbols are fetched only if not present
+- [#2327][2327] Add basic support to debug processes on Windows
+
+[2360]: https://github.com/Gallopsled/pwntools/pull/2360
+[2356]: https://github.com/Gallopsled/pwntools/pull/2356
+[2374]: https://github.com/Gallopsled/pwntools/pull/2374
+[2327]: https://github.com/Gallopsled/pwntools/pull/2327
+
+## 4.13.0 (`beta`)
 
 - [#2242][2242] Term module revamp: activating special handling of terminal only when necessary
 - [#2277][2277] elf: Resolve more relocations into GOT entries
@@ -97,7 +110,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2341][2341] Launch GDB correctly in iTerm on Mac
 - [#2268][2268] Add a `flatten` argument to `ssh.libs`
 - [#2347][2347] Fix/workaround Unicorn Engine 1GB limit that calls exit()
-- [#2327][2327] Add basic support to debug processes on Windows
+- [#2233][2233] Fix gdb.debug: exe parameter now respected, allow empty argv
 
 [2242]: https://github.com/Gallopsled/pwntools/pull/2242
 [2277]: https://github.com/Gallopsled/pwntools/pull/2277
@@ -123,9 +136,9 @@ The table below shows which release corresponds to each branch, and what date th
 [2341]: https://github.com/Gallopsled/pwntools/pull/2341
 [2268]: https://github.com/Gallopsled/pwntools/pull/2268
 [2347]: https://github.com/Gallopsled/pwntools/pull/2347
-[2327]: https://github.com/Gallopsled/pwntools/pull/2327
+[2233]: https://github.com/Gallopsled/pwntools/pull/2233
 
-## 4.12.0 (`beta`)
+## 4.12.0 (`stable`)
 
 - [#2202][2202] Fix `remote` and `listen` in sagemath
 - [#2117][2117] Add -p (--prefix) and -s (--separator) arguments to `hex` command
@@ -134,6 +147,8 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2212][2212] Add `--libc libc.so` argument to `pwn template` command
 - [#2257][2257] Allow creation of custom templates for `pwn template` command
 - [#2225][2225] Allow empty argv in ssh.process()
+- [#2349][2349] Fix term.readline omitting a trailing \n
+- [#2352][2352] add `RETURN_CONST` as an allowed `_const_code` in safeeval
 
 [2202]: https://github.com/Gallopsled/pwntools/pull/2202
 [2117]: https://github.com/Gallopsled/pwntools/pull/2117
@@ -142,8 +157,10 @@ The table below shows which release corresponds to each branch, and what date th
 [2212]: https://github.com/Gallopsled/pwntools/pull/2212
 [2257]: https://github.com/Gallopsled/pwntools/pull/2257
 [2225]: https://github.com/Gallopsled/pwntools/pull/2225
+[2349]: https://github.com/Gallopsled/pwntools/pull/2349
+[2352]: https://github.com/Gallopsled/pwntools/pull/2352
 
-## 4.11.1 (`stable`)
+## 4.11.1
 
 - [#2271][2271] FIX: Generated shebang with path to python invalid if path contains spaces
 - [#2272][2272] Fix `tube.clean_and_log` not logging buffered data
@@ -1124,4 +1141,4 @@ are mentioned here.
 - Added a lots of shellcodes
 - Stuff we forgot
 - Lots of documentation fixes
-- Lots of bugfixes
+- Lots of bugfixes

docs/requirements.txt

@@ -8,7 +8,8 @@ isort
 mako>=1.0.0
 paramiko>=1.15.2
 pip>=6.0.8
-pyelftools>=0.2.3
+pyelftools>=0.29, <0.30; python_version<'3'
+pyelftools>=0.29; python_version>='3'
 pygments>=2.0
 pypandoc
 pyserial>=2.7

docs/source/conf.py

@@ -71,6 +71,7 @@ def filter(self, record):
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+import six
 import pwnlib.update
 import pwnlib.util.fiddling
 import logging
@@ -97,8 +98,8 @@ def __setattr__(self, name, value):
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
 local_doctest = os.environ.get('USER') == 'pwntools'
-branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 skip_android = True
+is_python2 = six.PY2
 '''
 
 autoclass_content = 'both'

docs/source/index.rst

@@ -83,7 +83,6 @@ Each of the ``pwntools`` modules is documented here.
    :hidden:
 
    testexample
-   rop/call
 
 .. only:: not dash
 

docs/source/install.rst

@@ -51,13 +51,11 @@ Command-Line Tools
 
 When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
 
-However, if you run as an unprivileged user, you may see a warning message that looks like this:
+However, if you run as an unprivileged user, you may see a warning message that looks like this::
 
-.. code-block::
-
-      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
-      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
-      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+    WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+    elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+    unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
 
 Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
 

extra/docker/base/Dockerfile

@@ -40,10 +40,10 @@ RUN apt-get update \
         patchelf \
     && locale-gen en_US.UTF-8 \
     && update-locale LANG=en_US.UTF-8 \
-    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --upgrade pip setuptools wheel \
-    && python2.7 -m pip install --upgrade pwntools \
-    && python3 -m pip install --upgrade pip \
-    && python3 -m pip install --upgrade pwntools \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --no-cache-dir --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --no-cache-dir --upgrade pwntools \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/buster/Dockerfile

@@ -5,4 +5,4 @@ RUN apt-get -y dist-upgrade
 RUN apt-get -y install python3 python3-pip
 RUN apt-get -y install git wget unzip
 
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@dev

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwnlib/args.py

@@ -159,6 +159,11 @@ def STDERR(v):
     """Sends logging to ``stderr`` by default, instead of ``stdout``"""
     context.log_console = sys.stderr
 
+def LOCAL_LIBCDB(v):
+    """Sets path to local libc-database via ``context.local_libcdb``, e.g. 
+    ``LOCAL_LIBCDB='/path/to/libc-databse'``"""
+    context.local_libcdb = v
+
 hooks = {
     'LOG_LEVEL': LOG_LEVEL,
     'LOG_FILE': LOG_FILE,
@@ -170,6 +175,7 @@ def STDERR(v):
     'NOASLR': NOASLR,
     'NOPTRACE': NOPTRACE,
     'STDERR': STDERR,
+    'LOCAL_LIBCDB': LOCAL_LIBCDB,
 }
 
 def initialize():

pwnlib/commandline/template.py

@@ -47,10 +47,11 @@ def detect_missing_binaries(args):
         else:
             if os.access(filename, os.X_OK):
                 other_files.append(filename)
-    if len(other_files) == 1:
-        exe = other_files[0]
-    elif len(other_files) > 1:
-        log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
+    if not exe:
+        if len(other_files) == 1:
+            exe = other_files[0]
+        elif len(other_files) > 1:
+            log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
 
     if exe != args.exe:
         log.success("Found challenge binary %r", exe)

pwnlib/context/__init__.py

@@ -360,6 +360,7 @@ class ContextType(object):
         'endian': 'little',
         'gdbinit': "",
         'kernel': None,
+        'local_libcdb': "/var/lib/libc-database",
         'log_level': logging.INFO,
         'log_file': _devnull(),
         'log_console': sys.stdout,
@@ -830,6 +831,7 @@ def bits(self, bits):
         The default value is ``32``, but changes according to :attr:`arch`.
 
         Examples:
+
             >>> context.clear()
             >>> context.bits == 32
             True
@@ -1070,6 +1072,32 @@ def log_console(self, stream):
             stream = open(stream, 'wt')
         return stream
 
+    @_validator
+    def local_libcdb(self, path):
+        """ 
+        Sets path to local libc-database, get more information for libc-database:
+        https://github.com/niklasb/libc-database
+
+        Works in :attr:`pwnlib.libcdb` when searching by local database provider.
+
+        The default value is ``/var/lib/libc-database``.
+
+        Sets `context.local_libcdb` to empty string or `None` will turn off local libc-database integration.
+
+        Examples:
+
+            >>> context.local_libcdb = pwnlib.data.elf.path
+            >>> context.local_libcdb = 'foobar'
+            Traceback (most recent call last):
+            ...
+            AttributeError: 'foobar' does not exist, please download libc-database first
+        """
+
+        if not os.path.isdir(path):
+            raise AttributeError("'%s' does not exist, please download libc-database first" % path)
+
+        return path
+
     @property
     def mask(self):
         return (1 << self.bits) - 1

pwnlib/elf/elf.py

@@ -1346,6 +1346,7 @@ def vaddr_to_offset(self, address):
             or :const:`None`.
 
         Examples:
+
             >>> bash = ELF(which('bash'))
             >>> bash.vaddr_to_offset(bash.address)
             0
@@ -1496,6 +1497,7 @@ def write(self, address, data):
             that it stays in the same segment.
 
         Examples:
+
           >>> bash = ELF(which('bash'))
           >>> bash.read(bash.address+1, 3)
           b'ELF'
@@ -2387,6 +2389,7 @@ def set_interpreter(exepath, interpreter_path):
             A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
 
         Example:
+
             >>> tmpdir = tempfile.mkdtemp()
             >>> ls_path = os.path.join(tmpdir, 'ls')
             >>> _ = shutil.copy(which('ls'), ls_path)