From e2583456ec9beb55065d2b578a2598dff6277c0e Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Tue, 23 Apr 2024 11:28:16 +0200 Subject: [PATCH 1/5] Add `tube.upload_manually` Upload data in chunks when having a tube connected to a shell. This is useful when doing kernel or qemu challenges where you can't use the ssh tube's file upload features. --- pwnlib/tubes/tube.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/pwnlib/tubes/tube.py b/pwnlib/tubes/tube.py index a14e2d286..42b35e2b3 100644 --- a/pwnlib/tubes/tube.py +++ b/pwnlib/tubes/tube.py @@ -21,6 +21,8 @@ from pwnlib.log import Logger from pwnlib.timeout import Timeout from pwnlib.tubes.buffer import Buffer +from pwnlib.util import fiddling +from pwnlib.util import iters from pwnlib.util import misc from pwnlib.util import packing @@ -1077,6 +1079,92 @@ def clean_and_log(self, timeout = 0.05): with context.local(log_level='debug'): return cached_data + self.clean(timeout) + def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_size = 0x200, chmod_flags = 'u+x', compression='auto', end_marker = 'PWNTOOLS_DONE'): + """upload_manually(data, target_path = './payload', prompt = b'$', chunk_size = 0x200, chmod_flags = 'u+x', compression='auto', end_marker = 'PWNTOOLS_DONE') + + Upload a file manually using base64 encoding and compression. + This can be used when the tube is connected to a shell. + + The file is uploaded in base64-encoded chunks by appending to a file + and then decompressing it: + + ``` + loop: + echo | base64 -d >> . + -d . + chmod + ``` + + It is assumed that a `base64` command is available on the target system. + When ``compression`` is ``auto`` the best compression utility available is chosen. + + Arguments: + + data(bytes): The data to upload. + target_path(str): The path to upload the data to. + prompt(bytes): The shell prompt to wait for. + chunk_size(int): The size of each chunk to upload. + chmod_flags(str): The flags to use with chmod. ``""`` to ignore. + compression(str): The compression to use. ``auto`` to automatically choose the best compression or ``gzip`` or ``xz``. + end_marker(str): The marker to use to detect the end of the output. Only used when prompt is not set. + + """ + echo_end = "" + if not prompt: + echo_end = "; echo {}".format(end_marker) + end_markerb = end_marker.encode() + else: + end_markerb = prompt + + # Detect available compression utility, fallback to uncompressed upload. + compression_mode = None + possible_compression = ['gzip'] + if six.PY3: + possible_compression.insert(0, 'xz') + if not prompt: + self.sendline("echo {}".format(end_marker).encode()) + if compression == 'auto': + for utility in possible_compression: + self.sendlineafter(end_markerb, "command -v {} && echo YEP || echo NOPE;{}".format(utility, echo_end).encode()) + result = self.recvuntil([b'YEP', b'NOPE']) + if b'YEP' in result: + compression_mode = utility + break + elif compression in possible_compression: + compression_mode = compression + else: + self.error('Invalid compression mode: %s, has to be one of %s', compression, possible_compression) + + self.debug('Manually uploading using compression mode: %s', compression_mode) + + if compression_mode == 'xz': + import lzma + data = lzma.compress(data, format=lzma.FORMAT_XZ, preset=9) + compressed_path = target_path + '.xz' + elif compression_mode == 'gzip': + import gzip + data = gzip.compress(data, compresslevel=9) + compressed_path = target_path + '.gz' + else: + compressed_path = target_path + + # Upload data in `chunk_size` chunks. Assume base64 is available. + with self.progress('Uploading payload') as p: + for idx, chunk in enumerate(iters.group(chunk_size, data)): + if idx == 0: + self.sendlineafter(end_markerb, "echo {} | base64 -d > {}{}".format(fiddling.b64e(chunk), compressed_path, echo_end).encode()) + else: + self.sendlineafter(end_markerb, "echo {} | base64 -d >> {}{}".format(fiddling.b64e(chunk), compressed_path, echo_end).encode()) + p.status('{}/{}'.format(idx, len(data)//chunk_size)) + + # Decompress the file and set the permissions. + if compression_mode is not None: + self.sendlineafter(end_markerb, '{} -d {}{}'.format(compression_mode, compressed_path, echo_end).encode()) + if chmod_flags: + self.sendlineafter(end_markerb, 'chmod {} {}{}'.format(chmod_flags, target_path, echo_end).encode()) + if not prompt: + self.recvuntil(end_markerb) + def connect_input(self, other): """connect_input(other) From 3ea58d7f7810054d7023d4793f1199aa16ba4a5e Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Sat, 26 Oct 2024 13:29:50 +0200 Subject: [PATCH 2/5] Add tests and fix corner cases --- pwnlib/tubes/tube.py | 55 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/pwnlib/tubes/tube.py b/pwnlib/tubes/tube.py index 42b35e2b3..527bb3161 100644 --- a/pwnlib/tubes/tube.py +++ b/pwnlib/tubes/tube.py @@ -1091,12 +1091,14 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ ``` loop: echo | base64 -d >> . - -d . + -d -f . chmod ``` It is assumed that a `base64` command is available on the target system. - When ``compression`` is ``auto`` the best compression utility available is chosen. + When ``compression`` is ``auto`` the best compression utility available + between ``gzip`` and ``xz`` is chosen with a fallback to uncompressed + upload. Arguments: @@ -1108,6 +1110,28 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ compression(str): The compression to use. ``auto`` to automatically choose the best compression or ``gzip`` or ``xz``. end_marker(str): The marker to use to detect the end of the output. Only used when prompt is not set. + Examples: + + >>> l = listen() + >>> l.spawn_process('/bin/sh') + >>> r = remote('127.0.0.1', l.lport) + >>> r.upload_manually(b'some\xca\xfedata\n', prompt=b'', chmod_flags='') + >>> r.sendline(b'cat ./payload') + >>> r.recvline() + b'some\xca\xfedata\n' + + >>> r.upload_manually(cyclic(0x1000), target_path='./cyclic_pattern', prompt=b'', chunk_size=0x10, compression='gzip') + >>> r.sendline(b'sha256sum ./cyclic_pattern') + >>> r.recvlineS(keepends=False).startswith(sha256sumhex(cyclic(0x1000))) + True + + >>> blob = ELF.from_assembly(shellcraft.echo('Hello world!\n') + shellcraft.exit(0)) + >>> r.upload_manually(blob.data, prompt=b'') + >>> r.sendline(b'./payload') + >>> r.recvline() + b'Hello world!\n' + >>> r.close() + >>> l.close() """ echo_end = "" if not prompt: @@ -1125,7 +1149,7 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ self.sendline("echo {}".format(end_marker).encode()) if compression == 'auto': for utility in possible_compression: - self.sendlineafter(end_markerb, "command -v {} && echo YEP || echo NOPE;{}".format(utility, echo_end).encode()) + self.sendlineafter(end_markerb, "command -v {} && echo YEP || echo NOPE{}".format(utility, echo_end).encode()) result = self.recvuntil([b'YEP', b'NOPE']) if b'YEP' in result: compression_mode = utility @@ -1137,33 +1161,44 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ self.debug('Manually uploading using compression mode: %s', compression_mode) + compressed_data = b'' if compression_mode == 'xz': import lzma - data = lzma.compress(data, format=lzma.FORMAT_XZ, preset=9) + compressed_data = lzma.compress(data, format=lzma.FORMAT_XZ, preset=9) compressed_path = target_path + '.xz' elif compression_mode == 'gzip': import gzip - data = gzip.compress(data, compresslevel=9) + compressed_data = gzip.compress(data, compresslevel=9) compressed_path = target_path + '.gz' else: compressed_path = target_path + + # Don't compress if it doesn't reduce the size. + if len(compressed_data) >= len(data): + compression_mode = None + compressed_path = target_path + else: + data = compressed_data # Upload data in `chunk_size` chunks. Assume base64 is available. with self.progress('Uploading payload') as p: for idx, chunk in enumerate(iters.group(chunk_size, data)): + if None in chunk: + chunk = chunk[:chunk.index(None)] if idx == 0: - self.sendlineafter(end_markerb, "echo {} | base64 -d > {}{}".format(fiddling.b64e(chunk), compressed_path, echo_end).encode()) + self.sendlineafter(end_markerb, "echo {} | base64 -d > {}{}".format(fiddling.b64e(bytes(chunk)), compressed_path, echo_end).encode()) else: - self.sendlineafter(end_markerb, "echo {} | base64 -d >> {}{}".format(fiddling.b64e(chunk), compressed_path, echo_end).encode()) - p.status('{}/{}'.format(idx, len(data)//chunk_size)) + self.sendlineafter(end_markerb, "echo {} | base64 -d >> {}{}".format(fiddling.b64e(bytes(chunk)), compressed_path, echo_end).encode()) + p.status('{}/{} {}'.format(idx+1, len(data)//chunk_size+1, misc.size(idx*chunk_size + len(chunk)))) + p.success(misc.size(len(data))) # Decompress the file and set the permissions. if compression_mode is not None: - self.sendlineafter(end_markerb, '{} -d {}{}'.format(compression_mode, compressed_path, echo_end).encode()) + self.sendlineafter(end_markerb, '{} -d -f {}{}'.format(compression_mode, compressed_path, echo_end).encode()) if chmod_flags: self.sendlineafter(end_markerb, 'chmod {} {}{}'.format(chmod_flags, target_path, echo_end).encode()) if not prompt: - self.recvuntil(end_markerb) + self.recvuntil(end_markerb + b'\n') def connect_input(self, other): """connect_input(other) From 0c0731a62ad07298c76610cd16b456f4787c0163 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Sat, 26 Oct 2024 13:33:09 +0200 Subject: [PATCH 3/5] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 553b16e99..4288607f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The table below shows which release corresponds to each branch, and what date th - [#2482][2482] Throw error when using `sni` and setting `server_hostname` manually in `remote` - [#2478][2478] libcdb-cli: add `--offline-only`, refactor unstrip and add fetch parser for download libc-database - [#2484][2484] Allow to disable caching +- [#2410][2410] Add `tube.upload_manually` to upload files in chunks [2471]: https://github.com/Gallopsled/pwntools/pull/2471 [2358]: https://github.com/Gallopsled/pwntools/pull/2358 @@ -96,6 +97,7 @@ The table below shows which release corresponds to each branch, and what date th [2482]: https://github.com/Gallopsled/pwntools/pull/2482 [2478]: https://github.com/Gallopsled/pwntools/pull/2478 [2484]: https://github.com/Gallopsled/pwntools/pull/2484 +[2410]: https://github.com/Gallopsled/pwntools/pull/2410 ## 4.14.0 (`beta`) From 4644b4bb428a1a95ebf6ffae3386ee9b99dd5322 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Sat, 26 Oct 2024 13:41:51 +0200 Subject: [PATCH 4/5] Fix character escaping in tests --- pwnlib/tubes/tube.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pwnlib/tubes/tube.py b/pwnlib/tubes/tube.py index 527bb3161..8168ad2fd 100644 --- a/pwnlib/tubes/tube.py +++ b/pwnlib/tubes/tube.py @@ -1115,21 +1115,21 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ >>> l = listen() >>> l.spawn_process('/bin/sh') >>> r = remote('127.0.0.1', l.lport) - >>> r.upload_manually(b'some\xca\xfedata\n', prompt=b'', chmod_flags='') + >>> r.upload_manually(b'some\\xca\\xfedata\\n', prompt=b'', chmod_flags='') >>> r.sendline(b'cat ./payload') >>> r.recvline() - b'some\xca\xfedata\n' + b'some\\xca\\xfedata\\n' >>> r.upload_manually(cyclic(0x1000), target_path='./cyclic_pattern', prompt=b'', chunk_size=0x10, compression='gzip') >>> r.sendline(b'sha256sum ./cyclic_pattern') >>> r.recvlineS(keepends=False).startswith(sha256sumhex(cyclic(0x1000))) True - >>> blob = ELF.from_assembly(shellcraft.echo('Hello world!\n') + shellcraft.exit(0)) + >>> blob = ELF.from_assembly(shellcraft.echo('Hello world!\\n') + shellcraft.exit(0)) >>> r.upload_manually(blob.data, prompt=b'') >>> r.sendline(b'./payload') >>> r.recvline() - b'Hello world!\n' + b'Hello world!\\n' >>> r.close() >>> l.close() """ From f5f8b338023bf2e89ec9bd70ad8cffcb303a0705 Mon Sep 17 00:00:00 2001 From: Peace-Maker Date: Sat, 26 Oct 2024 14:02:05 +0200 Subject: [PATCH 5/5] Fix gzip compression on Python 2 --- pwnlib/tubes/tube.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pwnlib/tubes/tube.py b/pwnlib/tubes/tube.py index 8168ad2fd..84798314f 100644 --- a/pwnlib/tubes/tube.py +++ b/pwnlib/tubes/tube.py @@ -1087,7 +1087,7 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ The file is uploaded in base64-encoded chunks by appending to a file and then decompressing it: - + ``` loop: echo | base64 -d >> . @@ -1168,11 +1168,15 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ compressed_path = target_path + '.xz' elif compression_mode == 'gzip': import gzip - compressed_data = gzip.compress(data, compresslevel=9) + from six import BytesIO + f = BytesIO() + with gzip.GzipFile(fileobj=f, mode='wb', compresslevel=9) as g: + g.write(data) + compressed_data = f.getvalue() compressed_path = target_path + '.gz' else: compressed_path = target_path - + # Don't compress if it doesn't reduce the size. if len(compressed_data) >= len(data): compression_mode = None @@ -1186,12 +1190,12 @@ def upload_manually(self, data, target_path = './payload', prompt = b'$', chunk_ if None in chunk: chunk = chunk[:chunk.index(None)] if idx == 0: - self.sendlineafter(end_markerb, "echo {} | base64 -d > {}{}".format(fiddling.b64e(bytes(chunk)), compressed_path, echo_end).encode()) + self.sendlineafter(end_markerb, "echo {} | base64 -d > {}{}".format(fiddling.b64e(bytearray(chunk)), compressed_path, echo_end).encode()) else: - self.sendlineafter(end_markerb, "echo {} | base64 -d >> {}{}".format(fiddling.b64e(bytes(chunk)), compressed_path, echo_end).encode()) + self.sendlineafter(end_markerb, "echo {} | base64 -d >> {}{}".format(fiddling.b64e(bytearray(chunk)), compressed_path, echo_end).encode()) p.status('{}/{} {}'.format(idx+1, len(data)//chunk_size+1, misc.size(idx*chunk_size + len(chunk)))) p.success(misc.size(len(data))) - + # Decompress the file and set the permissions. if compression_mode is not None: self.sendlineafter(end_markerb, '{} -d -f {}{}'.format(compression_mode, compressed_path, echo_end).encode())