WIP: Introduce augmented CLI for qemu

fuzzers/full_system/qemu_baremetal/Justfile

@@ -28,6 +28,11 @@ build flavor="breakpoint": target_dir
         --target-dir {{TARGET_DIR}}
 
 run flavor="breakpoint": (target flavor) (build flavor)
+    #!/bin/bash
+
+    export KERNEL={{ KERNEL }}
+    export TARGET_DIR={{ TARGET_DIR }}
+
     {{BUILD_DIR / "qemu_baremetal"}} \
         -icount shift=auto,align=off,sleep=off \
         -machine mps2-an385 \
@@ -66,4 +71,4 @@ test_flavor flavor: (target flavor) (build flavor)
 test: (test_flavor "low_level") (test_flavor "breakpoint") (test_flavor "sync_exit")
 
 clean:
-    cargo clean
+    cargo clean

fuzzers/full_system/qemu_linux_kernel/Cargo.toml

@@ -19,7 +19,7 @@ lto = "fat"
 codegen-units = 1
 
 [dependencies]
-libafl = { path = "../../../libafl" }
+libafl = { path = "../../../libafl" , features = ["errors_backtrace"] }
 libafl_bolts = { path = "../../../libafl_bolts" }
 libafl_qemu = { path = "../../../libafl_qemu", default-features = false, features = [
   "x86_64",

fuzzers/full_system/qemu_linux_kernel/Justfile

@@ -24,9 +24,12 @@ update_files api="": target_dir linux_builder_dir (build api)
 
     cp {{ BUILD_DIR }}/include/* "{{ LINUX_BUILDER_DIR }}/setup/"
 
-target api="": linux_builder_dir update_files
+target api="": linux_builder_dir (update_files api)
     {{LINUX_BUILDER_DIR}}/build.sh
 
+update api="": (update_files api)
+    {{LINUX_BUILDER_DIR}}/update.sh
+
 build api="":
     cargo build \
       --profile {{ PROFILE }} \
@@ -45,18 +48,14 @@ run api="": (build api)
       LIBAFL_QEMU_BIOS_DIR={{ LIBAFL_QEMU_DIR_DEFAULT }}/build/qemu-bundle/usr/local/share/qemu
     fi
 
-    qemu-img create -f qcow2 -o backing_file={{ LINUX_BUILDER_OUT }}/OVMF_CODE.4m.fd -F raw {{ LINUX_BUILDER_OUT }}/OVMF_CODE.4m.qcow2
-    qemu-img create -f qcow2 -o backing_file={{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.fd -F raw {{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.qcow2
-    qemu-img create -f qcow2 -o backing_file={{ LINUX_BUILDER_OUT }}/linux.qcow2 -F qcow2 {{ LINUX_BUILDER_OUT }}/linux.tmp.qcow2
-
     {{FUZZER}} \
       -accel tcg \
       -m 4G \
-      -drive if=pflash,format=qcow2,file="{{ LINUX_BUILDER_OUT }}/OVMF_CODE.4m.qcow2" `# OVMF code pflash` \
-      -drive if=pflash,format=qcow2,file="{{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.qcow2" `# OVMF vars pflash` \
+      -drive if=pflash,format=qcow2,readonly=on,file="{{ LINUX_BUILDER_OUT }}/OVMF_CODE.4m.qcow2" `# OVMF code pflash` \
+      -drive if=pflash,format=qcow2,file="lqemu(mdisk,{{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.fd)" `# OVMF vars pflash` \
       -device ahci,id=ahci,bus=pci.0,addr=4 \
       -device ide-hd,bus=ahci.0,drive=disk,bootindex=1 \
-      -blockdev driver=file,filename="{{ LINUX_BUILDER_OUT }}/linux.tmp.qcow2",node-name=storage `# Backend file of "disk"` \
+      -blockdev driver=file,filename="lqemu(mdisk,{{ LINUX_BUILDER_OUT }}/linux.qcow2)",node-name=storage `# Backend file of "disk"` \
       -blockdev driver=qcow2,file=storage,node-name=disk `# QCOW2 "disk"` \
       -L "${LIBAFL_QEMU_BIOS_DIR}" \
       -nographic \

fuzzers/full_system/qemu_linux_kernel/src/fuzzer.rs

@@ -37,20 +37,21 @@ use libafl_qemu::{
     StdEmulatorDriver,
 };
 use libafl_qemu::{
-    emu::Emulator,
-    executor::QemuExecutor,
-    modules::{
+    parameters::{config::{QemuConfig, RamSize}, AugmentedCli}, emu::Emulator, executor::QemuExecutor, modules::{
         cmplog::CmpLogObserver, edges::StdEdgeCoverageClassicModule,
         utils::filters::HasAddressFilterTuple, CmpLogModule, EmulatorModuleTuple,
-    },
-    FastSnapshotManager, NopSnapshotManager, QemuInitError,
+    }, FastSnapshotManager, NopSnapshotManager, QemuInitError
 };
 use libafl_targets::{edges_map_mut_ptr, EDGES_MAP_DEFAULT_SIZE, MAX_EDGES_FOUND};
+use libafl_bolts::core_affinity::CoreId;
+use libafl::events::ClientDescription;
 
 #[cfg(feature = "nyx")]
 fn get_emulator<C, ET, I, S>(
     args: Vec<String>,
     modules: ET,
+    workdir: PathBuf,
+    core_id: CoreId,
 ) -> Result<
     Emulator<C, NyxCommandManager<S>, NyxEmulatorDriver, ET, I, S, NopSnapshotManager>,
     QemuInitError,
@@ -61,7 +62,7 @@ where
     S: Unpin,
 {
     Emulator::empty()
-        .qemu_parameters(args)
+        .qemu_parameters(AugmentedCli::new(args, workdir, core_id))
         .modules(modules)
         .driver(NyxEmulatorDriver::builder().build())
         .command_manager(NyxCommandManager::default())
@@ -73,6 +74,8 @@ where
 fn get_emulator<C, ET, I, S>(
     args: Vec<String>,
     mut modules: ET,
+    workdir: PathBuf,
+    core_id: CoreId,
 ) -> Result<
     Emulator<C, StdCommandManager<S>, StdEmulatorDriver, ET, I, S, FastSnapshotManager>,
     QemuInitError,
@@ -86,7 +89,7 @@ where
     modules.allow_address_range_all(&LINUX_PROCESS_ADDRESS_RANGE);
 
     Emulator::builder()
-        .qemu_parameters(args)
+        .qemu_parameters(AugmentedCli::new(args, workdir, core_id))
         .modules(modules)
         .build()
 }
@@ -117,11 +120,12 @@ pub fn fuzz() {
     // Hardcoded parameters
     let timeout = Duration::from_secs(60000);
     let broker_port = 1337;
-    let cores = Cores::from_cmdline("1").unwrap();
+    let cores = Cores::from(vec![1,2]);
     let corpus_dirs = [PathBuf::from("./corpus")];
     let objective_dir = PathBuf::from("./crashes");
+    let workdir = PathBuf::from("./workdir");
 
-    let mut run_client = |state: Option<_>, mut mgr, _client_description| {
+    let mut run_client = |state: Option<_>, mut mgr, client_description: ClientDescription| {
         // Initialize QEMU
         let args: Vec<String> = env::args().collect();
 
@@ -143,7 +147,7 @@ pub fn fuzz() {
             CmpLogModule::default(),
         );
 
-        let emu = get_emulator(args, modules)?;
+        let emu = get_emulator(args, modules, workdir.clone(), client_description.core_id())?;
 
         let devices = emu.list_devices();
         println!("Devices = {:?}", devices);

libafl_qemu/Cargo.toml

@@ -133,6 +133,8 @@ memmap2 = "0.9.5"
 getset = "0.1.3"
 # Document all features of this crate (for `cargo doc`)
 document-features = { workspace = true, optional = true }
+tempfile = "3.17.1"
+regex = { workspace = true }
 
 [build-dependencies]
 libafl_qemu_build = { workspace = true, default-features = true }

libafl_qemu/src/emu/builder.rs

@@ -9,7 +9,7 @@ use crate::{
     Emulator, NopEmulatorDriver, NopSnapshotManager, QemuInitError, QemuParams, StdEmulatorDriver,
     StdSnapshotManager,
     command::{NopCommandManager, StdCommandManager},
-    config::QemuConfigBuilder,
+    parameters::config::QemuConfigBuilder,
     modules::{EmulatorModule, EmulatorModuleTuple},
 };
 #[cfg(doc)]

libafl_qemu/src/emu/mod.rs

@@ -44,7 +44,7 @@ mod systemmode;
 #[cfg(feature = "systemmode")]
 pub use systemmode::*;
 
-use crate::config::QemuConfigBuilder;
+use crate::parameters::config::QemuConfigBuilder;
 
 #[derive(Clone, Copy)]
 pub enum GuestAddrKind {

libafl_qemu/src/modules/calls.rs

@@ -1,6 +1,6 @@
 use core::{cell::UnsafeCell, fmt::Debug};
 
-use capstone::prelude::*;
+use capstone::{Capstone, InsnDetail, arch::BuildsCapstone};
 use libafl::{
     executors::ExitKind,
     inputs::Input,
@@ -303,9 +303,9 @@ where
 
             #[cfg(cpu_target = "arm")]
             h.cs.set_mode(if pc & 1 == 1 {
-                arch::arm::ArchMode::Thumb.into()
+                capstone::arch::arm::ArchMode::Thumb.into()
             } else {
-                arch::arm::ArchMode::Arm.into()
+                capstone::arch::arm::ArchMode::Arm.into()
             })
             .unwrap();
         }

libafl_qemu/src/qemu/drive.rs

@@ -0,0 +1,75 @@
+use std::io;
+use std::io::ErrorKind;
+use std::path::PathBuf;
+use std::process::{Command, ExitStatus};
+use libafl_bolts::core_affinity::CoreId;
+
+pub enum QemuDiskKind {
+    Qcow2,
+    Raw,
+}
+
+pub struct MulticoreDrive {
+    input: PathBuf,
+    output_dir: PathBuf,
+    kind: QemuDiskKind,
+}
+
+impl MulticoreDrive {
+    pub fn new(input: PathBuf, output_dir: PathBuf) -> Self {
+        let kind = if let Some(ext) = input.extension() {
+            if ext.to_str().unwrap() == "qcow2" {
+                QemuDiskKind::Qcow2
+            } else {
+                QemuDiskKind::Raw
+            }
+        } else {
+            QemuDiskKind::Raw
+        };
+
+        Self {
+            input,
+            output_dir,
+            kind,
+        }
+    }
+
+    pub fn push(&mut self, core_id: &CoreId) -> Result<PathBuf, io::Error> {
+        let input_fmt = match &self.kind {
+            QemuDiskKind::Qcow2 => {
+                "qcow2"
+            }
+            QemuDiskKind::Raw => {
+                "raw"
+            }
+        };
+
+        if !self.input.exists() {
+            return Err(io::Error::new(ErrorKind::NotFound, "The input file does not exist."))
+        }
+
+        let input_fname = self.input.file_name().unwrap();
+        let output_partial_path = self.output_dir.join(input_fname.to_str().unwrap());
+        let output_f = PathBuf::from(format!("{}.{}", output_partial_path.display(), core_id.0));
+
+        let backing = format!("backing_file={}", self.input.display());
+
+        // qemu-img create -f qcow2 -o backing_file={{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.fd -F raw {{ LINUX_BUILDER_OUT }}/OVMF_VARS.4m.qcow2
+        let mut qemu_img = Command::new("qemu-img");
+        qemu_img.arg("create")
+            .args(["-f", "qcow2"])
+            .args(["-o", backing.as_str()])
+            .args(["-F", input_fmt])
+            .arg(&output_f);
+
+        let mut res = qemu_img.spawn()?;
+
+        let ret = res.wait()?;
+
+        if ret == ExitStatus::default() {
+            Ok(output_f)
+        } else {
+            Err(io::Error::new(io::ErrorKind::Other, "qemu-img failed."))
+        }
+    }
+}

libafl_qemu/src/qemu/mod.rs

@@ -32,11 +32,10 @@ use libafl_qemu_sys::{
 use libafl_qemu_sys::{libafl_qemu_remove_hw_breakpoint, libafl_qemu_set_hw_breakpoint};
 use num_traits::Num;
 use strum::IntoEnumIterator;
+use crate::{parameters::AugmentedCli, GuestAddrKind, GuestReg, Regs};
 
-use crate::{GuestAddrKind, GuestReg, Regs};
-
-pub mod config;
-use config::QemuConfig;
+pub mod parameters;
+pub use parameters::{QemuParams, QemuConfig};
 
 pub mod error;
 pub use error::{
@@ -57,6 +56,9 @@ mod hooks;
 pub use hooks::*;
 use libafl_bolts::{AsSliceMut, vec_init};
 
+mod drive;
+pub use drive::*;
+
 static mut QEMU_IS_INITIALIZED: bool = false;
 static mut QEMU_IS_RUNNING: bool = false;
 
@@ -113,13 +115,6 @@ pub struct Qemu {
     _private: (),
 }
 
-#[derive(Clone, Debug)]
-pub enum QemuParams {
-    // QemuConfig is quite big, at least 240 bytes so we use a Box
-    Config(Box<QemuConfig>),
-    Cli(Vec<String>),
-}
-
 #[derive(Debug, Clone)]
 pub struct QemuMemoryChunk {
     addr: GuestAddrKind,
@@ -195,6 +190,12 @@ impl From<QemuConfig> for QemuParams {
     }
 }
 
+impl From<AugmentedCli> for QemuParams {
+    fn from(cli: AugmentedCli) -> Self {
+        QemuParams::AugmentedCli(cli)
+    }
+}
+
 // impl TryFrom<QemuConfigBuilder> for QemuParams {
 //     type Error = QemuInitError;
 //
@@ -243,6 +244,7 @@ impl QemuParams {
                 .map(ToString::to_string)
                 .collect(),
             QemuParams::Cli(cli) => cli.clone(),
+            QemuParams::AugmentedCli(augmented_cli) => augmented_cli.parse(),
         }
     }
 }
@@ -556,7 +558,7 @@ impl Qemu {
                     .map_err(|_| unreachable!("QEMU_CONFIG was already set but Qemu was not init!"))
                     .expect("Could not set QEMU Config.");
             }
-            QemuParams::Cli(_) => {}
+            QemuParams::Cli(_) | QemuParams::AugmentedCli(_) => {}
         }
 
         let args = params.to_cli();

libafl_qemu/src/qemu/parameters/augmented_cli.rs

@@ -0,0 +1,65 @@
+use std::path::PathBuf;
+use hashbrown::hash_map::Entry;
+use hashbrown::HashMap;
+use libafl_bolts::core_affinity::CoreId;
+use regex::{Captures, Regex};
+use crate::MulticoreDrive;
+use std::fs;
+
+#[derive(Debug, Clone)]
+pub struct AugmentedCli {
+    base_cli: Vec<String>,
+    workdir: PathBuf,
+    core_id: CoreId,
+}
+
+impl AugmentedCli {
+    pub fn new(base_cli: Vec<String>, workdir: PathBuf, core_id: CoreId) -> Self {
+        Self {
+            base_cli,
+            workdir,
+            core_id
+        }
+    }
+
+    pub fn parse(&self) -> Vec<String> {
+        let mdisk_re = Regex::new(r"lqemu\(mdisk,(?<path>.*)\)").unwrap();
+        let mdisk_path = self.workdir.join("disks");
+
+        println!("removing {mdisk_path:?}");
+
+        if mdisk_path.exists() {
+            fs::remove_dir_all(&mdisk_path).unwrap();
+        }
+
+        fs::create_dir_all(&mdisk_path).unwrap();
+
+        // old_path -> new_path
+        let mut replacements: HashMap<PathBuf, PathBuf> = HashMap::new();
+
+        for hay in &self.base_cli {
+            for caps in mdisk_re.captures_iter(hay) {
+                let (_, [mdisk_in_path]) = caps.extract();
+
+                match replacements.entry(PathBuf::from(&mdisk_in_path)) {
+                    Entry::Occupied(_) => {}
+                    Entry::Vacant(new_entry) => {
+                        let mut multicore_disk = MulticoreDrive::new(PathBuf::from(mdisk_in_path), mdisk_path.clone());
+                        multicore_disk.push(&self.core_id).unwrap();
+                        new_entry.insert(multicore_disk.push(&self.core_id).unwrap());
+                    }
+                }
+            }
+        }
+
+        let mut new_cli = self.base_cli.clone();
+        for s in &mut new_cli {
+            *s = mdisk_re.replace_all(s, |caps: &Captures| {
+                let path = PathBuf::from(&caps["path"]);
+                replacements.get(&path).unwrap().as_os_str().to_str().unwrap().to_string()
+            }).to_string();
+        }
+
+        new_cli
+    }
+}

libafl_qemu/src/qemu/parameters/mod.rs

@@ -0,0 +1,13 @@
+pub mod config;
+pub use config::QemuConfig;
+
+pub mod augmented_cli;
+pub use augmented_cli::AugmentedCli;
+
+#[derive(Clone, Debug)]
+pub enum QemuParams {
+    // QemuConfig is quite big, at least 240 bytes so we use a Box
+    Config(Box<QemuConfig>),
+    Cli(Vec<String>),
+    AugmentedCli(AugmentedCli),
+}