Merge branch 'Gallopsled:dev' into cx-multi-shellcraft-cmd

Author: cocoa-xu

.github/workflows/ci.yml

@@ -19,6 +19,13 @@ jobs:
         git fetch origin
         git log --oneline --graph -10
 
+    - name: Install RPyC for gdb
+      run: |
+        # The version packaged in python3-rpyc is too old on Ubuntu 22.04
+        sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
+        /usr/bin/python -m pip install rpyc
+        gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
+
     - name: Cache for pip
       uses: actions/cache@v4
       id: cache-pip
@@ -62,7 +69,6 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          python3-rpyc \
           gdb gdbserver socat \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
@@ -72,6 +78,7 @@ jobs:
           binutils-powerpc-linux-gnu \
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
+          binutils-riscv64-linux-gnu \
           gcc-multilib \
           libc6-dbg \
           elfutils \
@@ -112,6 +119,10 @@ jobs:
 
     - name: Coverage doctests
       run: |
+        # Python version installed using setup-python interferes with gdb's python
+        # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules
+        # like _socket. This is a workaround.
+        unset LD_LIBRARY_PATH
         PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples

CHANGELOG.md

@@ -9,9 +9,11 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.13.0](#4130-dev)  | `dev`    |
-| [4.12.0](#4120-beta)  | `beta`   |
-| [4.11.1](#4111-stable)  | `stable` | Nov 14, 2023
+| [4.14.0](#4140-dev)  | `dev`   |
+| [4.13.0](#4130-beta)  | `beta`    |
+| [4.12.1](#4121)  |          |
+| [4.12.0](#4120-stable)  | `stable`   | Feb 22, 2024
+| [4.11.1](#4111)  |          | Nov 14, 2023
 | [4.11.0](#4110)  |          | Sep 15, 2023
 | [4.10.0](#4100)  |          | May 21, 2023
 | [4.9.0](#490)    |          | Dec 29, 2022
@@ -68,7 +70,29 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.13.0 (`dev`)
+## 4.14.0 (`dev`)
+
+- [#2360][2360] Add offline parameter for `search_by_hash` series function
+- [#2356][2356] Add local libc database provider for libcdb
+- [#2374][2374] libcdb.unstrip_libc: debug symbols are fetched only if not present
+- [#2327][2327] Add basic support to debug processes on Windows
+- [#2322][2322] Add basic RISCV64 shellcraft support
+- [#2330][2330] Change `context.newline` when setting `context.os` to `"windows"`
+- [#2389][2389] Fix passing bytes to `context.log_file` and `crc.BitPolynom`
+- [#2391][2391] Fix error message when passing invalid kwargs to `xor`
+- [#2376][2376] Return buffered data on first EOF in tube.readline()
+
+[2360]: https://github.com/Gallopsled/pwntools/pull/2360
+[2356]: https://github.com/Gallopsled/pwntools/pull/2356
+[2374]: https://github.com/Gallopsled/pwntools/pull/2374
+[2327]: https://github.com/Gallopsled/pwntools/pull/2327
+[2322]: https://github.com/Gallopsled/pwntools/pull/2322
+[2330]: https://github.com/Gallopsled/pwntools/pull/2330
+[2389]: https://github.com/Gallopsled/pwntools/pull/2389
+[2391]: https://github.com/Gallopsled/pwntools/pull/2391
+[2376]: https://github.com/Gallopsled/pwntools/pull/2376
+
+## 4.13.0 (`beta`)
 
 - [#2242][2242] Term module revamp: activating special handling of terminal only when necessary
 - [#2277][2277] elf: Resolve more relocations into GOT entries
@@ -97,6 +121,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2341][2341] Launch GDB correctly in iTerm on Mac
 - [#2268][2268] Add a `flatten` argument to `ssh.libs`
 - [#2347][2347] Fix/workaround Unicorn Engine 1GB limit that calls exit()
+- [#2233][2233] Fix gdb.debug: exe parameter now respected, allow empty argv
 
 [2242]: https://github.com/Gallopsled/pwntools/pull/2242
 [2277]: https://github.com/Gallopsled/pwntools/pull/2277
@@ -122,8 +147,17 @@ The table below shows which release corresponds to each branch, and what date th
 [2341]: https://github.com/Gallopsled/pwntools/pull/2341
 [2268]: https://github.com/Gallopsled/pwntools/pull/2268
 [2347]: https://github.com/Gallopsled/pwntools/pull/2347
+[2233]: https://github.com/Gallopsled/pwntools/pull/2233
 
-## 4.12.0 (`beta`)
+## 4.12.1
+
+- [#2373][2373] Fix displaying bright color variation in terminal output
+- [#2378][2378] Don't go though a shell in `gdb.debug`
+
+[2373]: https://github.com/Gallopsled/pwntools/pull/2373
+[2378]: https://github.com/Gallopsled/pwntools/pull/2378
+
+## 4.12.0 (`stable`)
 
 - [#2202][2202] Fix `remote` and `listen` in sagemath
 - [#2117][2117] Add -p (--prefix) and -s (--separator) arguments to `hex` command
@@ -132,6 +166,8 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2212][2212] Add `--libc libc.so` argument to `pwn template` command
 - [#2257][2257] Allow creation of custom templates for `pwn template` command
 - [#2225][2225] Allow empty argv in ssh.process()
+- [#2349][2349] Fix term.readline omitting a trailing \n
+- [#2352][2352] add `RETURN_CONST` as an allowed `_const_code` in safeeval
 
 [2202]: https://github.com/Gallopsled/pwntools/pull/2202
 [2117]: https://github.com/Gallopsled/pwntools/pull/2117
@@ -140,8 +176,10 @@ The table below shows which release corresponds to each branch, and what date th
 [2212]: https://github.com/Gallopsled/pwntools/pull/2212
 [2257]: https://github.com/Gallopsled/pwntools/pull/2257
 [2225]: https://github.com/Gallopsled/pwntools/pull/2225
+[2349]: https://github.com/Gallopsled/pwntools/pull/2349
+[2352]: https://github.com/Gallopsled/pwntools/pull/2352
 
-## 4.11.1 (`stable`)
+## 4.11.1
 
 - [#2271][2271] FIX: Generated shebang with path to python invalid if path contains spaces
 - [#2272][2272] Fix `tube.clean_and_log` not logging buffered data
@@ -1122,4 +1160,4 @@ are mentioned here.
 - Added a lots of shellcodes
 - Stuff we forgot
 - Lots of documentation fixes
-- Lots of bugfixes
+- Lots of bugfixes

docs/requirements.txt

@@ -8,7 +8,8 @@ isort
 mako>=1.0.0
 paramiko>=1.15.2
 pip>=6.0.8
-pyelftools>=0.2.3
+pyelftools>=0.29, <0.30; python_version<'3'
+pyelftools>=0.29; python_version>='3'
 pygments>=2.0
 pypandoc
 pyserial>=2.7

docs/source/conf.py

@@ -71,6 +71,7 @@ def filter(self, record):
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+import six
 import pwnlib.update
 import pwnlib.util.fiddling
 import logging
@@ -97,8 +98,8 @@ def __setattr__(self, name, value):
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
 local_doctest = os.environ.get('USER') == 'pwntools'
-branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 skip_android = True
+is_python2 = six.PY2
 '''
 
 autoclass_content = 'both'

docs/source/index.rst

@@ -77,12 +77,12 @@ Each of the ``pwntools`` modules is documented here.
    update
    useragents
    util/*
+   windbg
 
 .. toctree::
    :hidden:
 
    testexample
-   rop/call
 
 .. only:: not dash
 

docs/source/install.rst

@@ -15,6 +15,9 @@ following system libraries installed.
 
    install/*
 
+
+Note: For Mac OS X you will need to have cmake ``brew install cmake`` and pkg-config ``brew install pkg-config`` installed.
+
 Released Version
 -----------------
 
@@ -51,13 +54,11 @@ Command-Line Tools
 
 When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
 
-However, if you run as an unprivileged user, you may see a warning message that looks like this:
-
-.. code-block::
+However, if you run as an unprivileged user, you may see a warning message that looks like this::
 
-      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
-      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
-      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+    WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+    elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+    unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
 
 Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
 

docs/source/install/binutils.rst

@@ -32,14 +32,15 @@ Mac OS X
 ^^^^^^^^^^^^^^^^
 
 Mac OS X is just as easy, but requires building binutils from source.
-However, we've made ``homebrew`` recipes to make this a single command.
+However, we've made ``homebrew`` recipes to make this just two commands.
 After installing `brew <https://brew.sh>`__, grab the appropriate
 recipe from our `binutils
 repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
+    $ wget https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
+    $ brew install ./binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/shellcraft/riscv64.rst

@@ -0,0 +1,19 @@
+.. testsetup:: *
+
+   from pwn import *
+   context.clear(arch='riscv64')
+
+:mod:`pwnlib.shellcraft.riscv64` --- Shellcode for RISCV64
+===========================================================
+
+:mod:`pwnlib.shellcraft.riscv64`
+-------------------------------
+
+.. automodule:: pwnlib.shellcraft.riscv64
+   :members:
+
+:mod:`pwnlib.shellcraft.riscv64.linux`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.riscv64.linux
+   :members:

docs/source/windbg.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+    from pwn import *
+
+:mod:`pwnlib.windbg` --- Working with WinDbg
+======================================
+
+.. automodule:: pwnlib.windbg
+   :members:

extra/docker/base/Dockerfile

@@ -40,10 +40,10 @@ RUN apt-get update \
         patchelf \
     && locale-gen en_US.UTF-8 \
     && update-locale LANG=en_US.UTF-8 \
-    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --upgrade pip setuptools wheel \
-    && python2.7 -m pip install --upgrade pwntools \
-    && python3 -m pip install --upgrade pip \
-    && python3 -m pip install --upgrade pwntools \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --no-cache-dir --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --no-cache-dir --upgrade pwntools \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/buster/Dockerfile

@@ -5,4 +5,4 @@ RUN apt-get -y dist-upgrade
 RUN apt-get -y install python3 python3-pip
 RUN apt-get -y install git wget unzip
 
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@dev

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable \
-    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwnlib/__init__.py

@@ -36,6 +36,7 @@
     'util',
     'update',
     'version',
+    'windbg',
 ]
 
 from . import args

pwnlib/args.py

@@ -159,6 +159,11 @@ def STDERR(v):
     """Sends logging to ``stderr`` by default, instead of ``stdout``"""
     context.log_console = sys.stderr
 
+def LOCAL_LIBCDB(v):
+    """Sets path to local libc-database via ``context.local_libcdb``, e.g. 
+    ``LOCAL_LIBCDB='/path/to/libc-databse'``"""
+    context.local_libcdb = v
+
 hooks = {
     'LOG_LEVEL': LOG_LEVEL,
     'LOG_FILE': LOG_FILE,
@@ -170,6 +175,7 @@ def STDERR(v):
     'NOASLR': NOASLR,
     'NOPTRACE': NOPTRACE,
     'STDERR': STDERR,
+    'LOCAL_LIBCDB': LOCAL_LIBCDB,
 }
 
 def initialize():

pwnlib/commandline/template.py

@@ -47,10 +47,11 @@ def detect_missing_binaries(args):
         else:
             if os.access(filename, os.X_OK):
                 other_files.append(filename)
-    if len(other_files) == 1:
-        exe = other_files[0]
-    elif len(other_files) > 1:
-        log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
+    if not exe:
+        if len(other_files) == 1:
+            exe = other_files[0]
+        elif len(other_files) > 1:
+            log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
 
     if exe != args.exe:
         log.success("Found challenge binary %r", exe)