Fix gdb and other doctests (#2355)

* Fix gdb.debug doctests

* Accept bytes in ssh.which

But still return a `str` return for backwards-compatibility.

* CI: Make rpyc visible in gdb's python

* Fix not running all doctests

They appear to have to be a free standing block of `>>>` with an empty line above.

* Remove unnecessary text type check in ssh.which

Author: peace-maker

.github/workflows/ci.yml

@@ -112,6 +112,10 @@ jobs:
 
     - name: Coverage doctests
       run: |
+        # Python version installed using setup-python interferes with gdb's python
+        # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules
+        # like _socket. This is a workaround.
+        unset LD_LIBRARY_PATH
         PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples

docs/source/conf.py

@@ -71,6 +71,7 @@ def filter(self, record):
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+import six
 import pwnlib.update
 import pwnlib.util.fiddling
 import logging
@@ -98,6 +99,7 @@ def __setattr__(self, name, value):
 travis_ci = os.environ.get('USER') == 'travis'
 local_doctest = os.environ.get('USER') == 'pwntools'
 skip_android = True
+is_python2 = six.PY2
 '''
 
 autoclass_content = 'both'

docs/source/index.rst

@@ -82,7 +82,6 @@ Each of the ``pwntools`` modules is documented here.
    :hidden:
 
    testexample
-   rop/call
 
 .. only:: not dash
 

docs/source/install.rst

@@ -51,13 +51,11 @@ Command-Line Tools
 
 When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
 
-However, if you run as an unprivileged user, you may see a warning message that looks like this:
+However, if you run as an unprivileged user, you may see a warning message that looks like this::
 
-.. code-block::
-
-      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
-      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
-      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+    WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+    elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+    unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
 
 Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
 

pwnlib/context/__init__.py

@@ -830,6 +830,7 @@ def bits(self, bits):
         The default value is ``32``, but changes according to :attr:`arch`.
 
         Examples:
+
             >>> context.clear()
             >>> context.bits == 32
             True

pwnlib/elf/elf.py

@@ -1346,6 +1346,7 @@ def vaddr_to_offset(self, address):
             or :const:`None`.
 
         Examples:
+
             >>> bash = ELF(which('bash'))
             >>> bash.vaddr_to_offset(bash.address)
             0
@@ -1496,6 +1497,7 @@ def write(self, address, data):
             that it stays in the same segment.
 
         Examples:
+
           >>> bash = ELF(which('bash'))
           >>> bash.read(bash.address+1, 3)
           b'ELF'
@@ -2387,6 +2389,7 @@ def set_interpreter(exepath, interpreter_path):
             A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
 
         Example:
+
             >>> tmpdir = tempfile.mkdtemp()
             >>> ls_path = os.path.join(tmpdir, 'ls')
             >>> _ = shutil.copy(which('ls'), ls_path)

pwnlib/fmtstr.py

@@ -129,6 +129,7 @@ def normalize_writes(writes):
     such that all values are raw bytes and consecutive writes are merged to a single key.
 
     Examples:
+
         >>> context.clear(endian="little", bits=32)
         >>> normalize_writes({0x0: [p32(0xdeadbeef)], 0x4: p32(0xf00dface), 0x10: 0x41414141})
         [(0, b'\xef\xbe\xad\xde\xce\xfa\r\xf0'), (16, b'AAAA')]
@@ -215,6 +216,7 @@ def compute_padding(self, counter):
         given the current format string write counter (how many bytes have been written until now).
 
         Examples:
+
             >>> hex(pwnlib.fmtstr.AtomWrite(0x0, 0x2, 0x2345).compute_padding(0x1111))
             '0x1234'
             >>> hex(pwnlib.fmtstr.AtomWrite(0x0, 0x2, 0xaa00).compute_padding(0xaabb))
@@ -246,6 +248,7 @@ def union(self, other):
         Combine adjacent writes into a single write.
 
         Example:
+
             >>> context.clear(endian = "little")
             >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
             AtomWrite(start=0, size=2, integer=0x201, mask=0x77ff)
@@ -285,11 +288,13 @@ def make_atoms_simple(address, data, badbytes=frozenset()):
 
     This function is simple and does not try to minimize the number of atoms. For example, if there are no
     bad bytes, it simply returns one atom for each byte:
+
         >>> pwnlib.fmtstr.make_atoms_simple(0x0, b"abc", set())
         [AtomWrite(start=0, size=1, integer=0x61, mask=0xff), AtomWrite(start=1, size=1, integer=0x62, mask=0xff), AtomWrite(start=2, size=1, integer=0x63, mask=0xff)]
     
     If there are bad bytes, it will try to bypass by skipping addresses containing bad bytes, otherwise a
     RuntimeError will be raised:
+
         >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'abc', b'\x62')
         [AtomWrite(start=97, size=2, integer=0x6261, mask=0xffff), AtomWrite(start=99, size=1, integer=0x63, mask=0xff)]
         >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'a'*0x10, b'\x62\x63\x64\x65\x66\x67\x68')
@@ -325,6 +330,7 @@ def merge_atoms_writesize(atoms, maxsize):
     This function simply merges adjacent atoms as long as the merged atom's size is not larger than ``maxsize``.
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> merge_atoms_writesize([AtomWrite(0, 1, 1), AtomWrite(1, 1, 1), AtomWrite(2, 1, 2)], 2)
         [AtomWrite(start=0, size=2, integer=0x101, mask=0xffff), AtomWrite(start=2, size=1, integer=0x2, mask=0xff)]
@@ -364,6 +370,7 @@ def find_min_hamming_in_range_step(prev, step, carry, strict):
         A tuple (score, value, mask) where score equals the number of matching bytes between the returned value and target.
 
     Examples:
+
         >>> initial = {(0,0): (0,0,0), (0,1): None, (1,0): None, (1,1): None}
         >>> pwnlib.fmtstr.find_min_hamming_in_range_step(initial, (0, 0xFF, 0x1), 0, 0)
         (1, 1, 255)
@@ -419,6 +426,7 @@ def find_min_hamming_in_range(maxbytes, lower, upper, target):
         target(int): the target value that should be approximated
 
     Examples:
+
         >>> pp = lambda svm: (svm[0], hex(svm[1]), hex(svm[2]))
         >>> pp(pwnlib.fmtstr.find_min_hamming_in_range(1, 0x0, 0x100, 0xaa))
         (1, '0xaa', '0xff')
@@ -470,6 +478,7 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
         overflows(int): how many extra overflows (of size sz) to tolerate to reduce the number of atoms
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> merge_atoms_overlapping([AtomWrite(0, 1, 1), AtomWrite(1, 1, 1)], 2, 8, 0, 1)
         [AtomWrite(start=0, size=2, integer=0x101, mask=0xffff)]
@@ -557,13 +566,15 @@ def overlapping_atoms(atoms):
     Finds pairs of atoms that write to the same address.
 
     Basic examples:
+
         >>> from pwnlib.fmtstr import *
         >>> list(overlapping_atoms([AtomWrite(0, 2, 0), AtomWrite(2, 10, 1)])) # no overlaps
         []
         >>> list(overlapping_atoms([AtomWrite(0, 2, 0), AtomWrite(1, 2, 1)])) # single overlap
         [(AtomWrite(start=0, size=2, integer=0x0, mask=0xffff), AtomWrite(start=1, size=2, integer=0x1, mask=0xffff))]
 
     When there are transitive overlaps, only the largest overlap is returned. For example:
+
         >>> list(overlapping_atoms([AtomWrite(0, 3, 0), AtomWrite(1, 4, 1), AtomWrite(2, 4, 1)]))
         [(AtomWrite(start=0, size=3, integer=0x0, mask=0xffffff), AtomWrite(start=1, size=4, integer=0x1, mask=0xffffffff)), (AtomWrite(start=1, size=4, integer=0x1, mask=0xffffffff), AtomWrite(start=2, size=4, integer=0x1, mask=0xffffffff))]
 
@@ -629,6 +640,7 @@ def sort_atoms(atoms, numbwritten):
         numbwritten(int): the value at which the counter starts
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> sort_atoms([AtomWrite(0, 1, 0xff), AtomWrite(1, 1, 0xfe)], 0) # the example described above
         [AtomWrite(start=1, size=1, integer=0xfe, mask=0xff), AtomWrite(start=0, size=1, integer=0xff, mask=0xff)]
@@ -694,6 +706,7 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4, no_dol
         no_dollars(bool) : flag to generete the payload with or w/o $ notation 
 
     Examples:
+
         >>> pwnlib.fmtstr.make_payload_dollar(1, [pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0xff)])
         (b'%255c%1$hhn', b'\x00\x00\x00\x00')
     '''
@@ -840,6 +853,7 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         The payload in order to do needed writes
 
     Examples:
+
         >>> context.clear(arch = 'amd64')
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%4$llnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00'

pwnlib/gdb.py

@@ -141,14 +141,11 @@
 from __future__ import absolute_import
 from __future__ import division
 
-from contextlib import contextmanager
 import os
-import sys
 import platform
 import psutil
 import random
 import re
-import shlex
 import six
 import six.moves
 import socket
@@ -512,27 +509,30 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         >>> io.close()
         
         Start a new process with modified argv[0]
-        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'], executable="/bin/sh")
+
+        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'], gdbscript='continue', exe="/bin/sh")
         >>> io.sendline(b"echo $0")
         >>> io.recvline()
-        b'$ \xde\xad\xbe\xef\n'
+        b'\xde\xad\xbe\xef\n'
         >>> io.close()
 
         Demonstrate that LD_PRELOAD is respected
+
         >>> io = process(["grep", "libc.so.6", "/proc/self/maps"])
         >>> real_libc_path = io.recvline().split()[-1]
+        >>> io.close()
         >>> import shutil
-        >>> shutil.copy(real_libc_path, "./libc.so.6") # make a copy of libc to demonstrate that it is loaded
-        >>> io = gdb.debug(["grep", "libc.so.6", "/proc/self/maps"], env={"LD_PRELOAD": "./libc.so.6"})
-        >>> io.recvline().split()[-1]
-        b"./libc.so.6"
-        >>> os.remove("./libc.so.6") # cleanup
+        >>> local_path = shutil.copy(real_libc_path, "./local-libc.so") # make a copy of libc to demonstrate that it is loaded
+        >>> io = gdb.debug(["grep", "local-libc.so", "/proc/self/maps"], gdbscript="continue", env={"LD_PRELOAD": "./local-libc.so"})
+        >>> io.recvline().split()[-1] # doctest: +ELLIPSIS
+        b'.../local-libc.so'
+        >>> os.remove("./local-libc.so") # cleanup
 
 
     Using GDB Python API:
 
-    .. doctest
-       :skipif: six.PY2
+    .. doctest::
+       :skipif: is_python2
 
         Debug a new process
 
@@ -577,18 +577,21 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         >>> io.sendline(b"echo hello")
 
         Interact with the process
+
         >>> io.interactive() # doctest: +SKIP
         >>> io.close()
 
         Using a modified args[0] on a remote process
-        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'],  gdbscript='continue', exe="/bin/sh",  ssh=shell)
+
+        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'], gdbscript='continue', exe="/bin/sh", ssh=shell)
         >>> io.sendline(b"echo $0")
         >>> io.recvline()
         b'$ \xde\xad\xbe\xef\n'
         >>> io.close()
 
         Using an empty args[0] on a remote process
-        >>> io = gdb.debug(args=[],  gdbscript='continue', exe="/bin/sh",  ssh=shell)
+
+        >>> io = gdb.debug(args=[], gdbscript='continue', exe="/bin/sh", ssh=shell)
         >>> io.sendline(b"echo $0")
         >>> io.recvline()
         b'$ \n'
@@ -681,7 +684,10 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     garbage = gdbserver.recvline(timeout=1)
 
     # Some versions of gdbserver output an additional message
-    garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=2)
+    try:
+        garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=2)
+    except EOFError:
+        pass
 
     return gdbserver
 
@@ -944,8 +950,8 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
         Using GDB Python API:
 
-        .. doctest
-           :skipif: six.PY2
+        .. doctest::
+           :skipif: is_python2
 
             >>> io = process('bash')
 

pwnlib/libcdb.py

@@ -249,6 +249,7 @@ def unstrip_libc(filename):
         :const:`True` if binary was unstripped, :const:`False` otherwise.
 
     Examples:
+
         >>> filename = search_by_build_id('69389d485a9793dbe873f0ea2c93e02efaa9aa3d', unstrip=False)
         >>> libc = ELF(filename)
         >>> 'main_arena' in libc.symbols
@@ -432,6 +433,7 @@ def download_libraries(libc_path, unstrip=True):
         The path to the cached directory containing the downloaded libraries.
 
     Example:
+
         >>> libc_path = ELF(which('ls'), checksec=False).libc.path
         >>> lib_path = download_libraries(libc_path)
         >>> lib_path is not None
@@ -545,6 +547,7 @@ def search_by_symbol_offsets(symbols, select_index=None, unstrip=True, return_as
         is returned instead.
 
     Examples:
+
         >>> filename = search_by_symbol_offsets({'puts': 0x420, 'printf': 0xc90}, select_index=1)
         >>> libc = ELF(filename)
         >>> libc.sym.system == 0x52290
@@ -597,6 +600,7 @@ def search_by_build_id(hex_encoded_id, unstrip=True):
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_build_id('fe136e485814fee2268cf19e5c124ed0f73f4400')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -622,6 +626,7 @@ def search_by_md5(hex_encoded_id, unstrip=True):
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -647,6 +652,7 @@ def search_by_sha1(hex_encoded_id, unstrip=True):
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_sha1('34471e355a5e71400b9d65e78d2cd6ce7fc49de5')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -673,6 +679,7 @@ def search_by_sha256(hex_encoded_id, unstrip=True):
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_sha256('5e877a8272da934812d2d1f9ee94f73c77c790cbc5d8251f5322389fc9667f21')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'

pwnlib/rop/rop.py

@@ -1499,6 +1499,7 @@ def ret2csu(self, edi=Padding('edi'), rsi=Padding('rsi'),
                 .dynamic section. .got.plt entries are a good target. Required
                 for PIE binaries.
         Test:
+
             >>> context.clear(binary=pwnlib.data.elf.ret2dlresolve.get("amd64"))
             >>> r = ROP(context.binary)
             >>> r.ret2csu(1, 2, 3, 4, 5, 6, 7, 8, 9)

pwnlib/shellcraft/templates/aarch64/pushstr_array.asm

@@ -17,6 +17,7 @@ Arguments:
         ends with exactly one NULL byte.
 
 Example:
+
     >>> assembly = shellcraft.execve("/bin/sh", ["sh", "-c", "echo Hello string $WORLD"], {"WORLD": "World!"})
     >>> ELF.from_assembly(assembly).process().recvall()
     b'Hello string World!\n'

pwnlib/shellcraft/templates/arm/ret.asm

@@ -5,6 +5,7 @@ Args:
     return_value: Value to return
 
 Examples:
+
     >>> with context.local(arch='arm'):
     ...     print(enhex(asm(shellcraft.ret())))
     ...     print(enhex(asm(shellcraft.ret(0))))

pwnlib/shellcraft/templates/i386/xor.asm

@@ -19,6 +19,7 @@ Args:
                  the number of bytes to XOR.
 
 Example:
+
     >>> sc  = shellcraft.read(0, 'esp', 32)
     >>> sc += shellcraft.xor(0xdeadbeef, 'esp', 32)
     >>> sc += shellcraft.write(1, 'esp', 32)

pwnlib/shellcraft/templates/thumb/linux/findpeer.asm

@@ -8,6 +8,7 @@
     against the peer port. Resulting socket is left in r6.
 
     Example:
+
         >>> enhex(asm(shellcraft.findpeer(1337)))
         '6ff00006ee4606f101064ff001074fea072707f11f07f54630461fb401a96a4601df0130efdd01994fea11414ff039024fea022202f105029142e4d1'
 </%docstring>