Add responder-tools as project in workspace with nzmap

also move filter from xtask to responder-tools and rename to build_filter
nzmap spins up n sharded zmap instances

.gitignore

@@ -19,4 +19,4 @@ Cargo.lock
 data/
 
 # Artifacts
-bin/
+/bin/

Cargo.toml

@@ -1,2 +1,2 @@
 [workspace]
-members = ["responder", "responder-common", "xtask"]
+members = ["responder", "responder-common", "responder-tools", "xtask"]

responder-tools/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "responder-tools"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1"
+clap = { version = "3.1", features = ["derive"] }
+aya-tool = { git = "https://github.com/aya-rs/aya" }
+responder-common = { path = "../responder-common", features=["user"] }
+env_logger = "0.9"
+log = "0.4"
+rayon = "1.7.0"

xtask/src/filter.rs → responder-tools/src/bin/build_filter.rs

@@ -6,14 +6,21 @@ use log::info;
 use responder_common::bloom_filter;
 
 #[derive(Debug, Parser)]
-pub struct Options {
+struct Options {
     pub ip_path: PathBuf,
     pub out_path: Option<PathBuf>,
     #[clap(short, long)]
     pub force: bool,
 }
 
-pub fn build_filter(opts: Options) -> Result<(), anyhow::Error> {
+fn main() -> Result<(), anyhow::Error> {
+    env_logger::init();
+    let opts = Options::parse();
+
+    return build_filter(opts);
+}
+
+fn build_filter(opts: Options) -> Result<(), anyhow::Error> {
     let out_path = opts.out_path.unwrap_or_else(|| {
         let mut ext = opts.ip_path.extension().unwrap_or_default().to_os_string();
         ext.push(".bfb");

responder-tools/src/bin/nzmap.rs

@@ -0,0 +1,66 @@
+use std::{time::{UNIX_EPOCH, SystemTime}, process::Command};
+
+use anyhow::{Context, anyhow};
+use clap::Parser;
+use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+
+
+
+#[derive(Debug, Parser)]
+struct Options {
+    #[clap(short = 'n', long, default_value="1")]
+    pub count: u32,
+
+    #[clap(short = 'e', long)]
+    pub seed: Option<String>,
+
+    #[clap(name = "args", last = true)]
+    pub run_args: Vec<String>,
+
+}
+
+fn main() -> Result<(), anyhow::Error> {
+    env_logger::init();
+    let opts = Options::parse();
+    return spawn_zmaps(opts);
+}
+
+fn spawn_zmap(args: &[String], i: u32) -> Result<(), anyhow::Error> {
+    let mut args = Vec::from(args);
+    args.push(format!("--shard={}", i));
+    args.push(format!("--cores=0,{},1",i+2));
+    args.push(format!("-o/tmp/ips{}.csv", i));
+    let result = Command::new("zmap")
+        .args(args)
+        .spawn()?
+        .wait()?;
+    if result.success() {
+        return Ok(());
+    } else if let Some(code) = result.code() {
+        return Err(anyhow!("code: {}", code));
+    } else {
+        return Err(anyhow!(""));
+    }
+}
+
+
+fn spawn_zmaps(opts: Options) -> Result<(), anyhow::Error> {
+    let seed = if let Some(seed) = opts.seed {
+        seed
+    } else {
+        SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .context("Attempted to obtain UNIX timestamp for seed")?
+            .as_millis()
+            .to_string()
+    };
+    let mut args = opts.run_args.clone();
+    args.push(format!("--seed={}", seed));
+    args.push(format!("--shards={}", opts.count));
+
+    (0..opts.count).into_par_iter().map(|i| {
+        spawn_zmap(&args, i).context(format!("Shard {} encountered error", i))
+    }).reduce(|| Ok(()), Result::and)?;
+
+    Ok(())
+}

xtask/Cargo.toml

@@ -7,6 +7,5 @@ edition = "2021"
 anyhow = "1"
 clap = { version = "3.1", features = ["derive"] }
 aya-tool = { git = "https://github.com/aya-rs/aya" }
-responder-common = { path = "../responder-common", features=["user"] }
 env_logger = "0.9"
 log = "0.4"

xtask/src/build.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::PathBuf, process::Command};
+use std::{fs, path::{PathBuf, Path}, process::Command};
 
 use anyhow::Context as _;
 use clap::Parser;
@@ -29,27 +29,22 @@ fn build_user(_opts: &Options) -> Result<(), anyhow::Error> {
     Ok(())
 }
 
-pub fn build(opts: Options) -> Result<(), anyhow::Error> {
-    fs::create_dir_all(&opts.output_folder).context(format!(
-        "Error creating dirs for path: {}",
-        opts.output_folder
-            .to_str()
-            .unwrap_or("[path not valid unicode]")
-    ))?;
-
-    build_ebpf(build_ebpf::Options {
-        target: opts.bpf_target,
-        release: true,
-    })
-    .context("Error while building eBPF program")?;
+fn build_tools(_opts: &Options) -> Result<(), anyhow::Error> {
+    let status = Command::new("cargo")
+        .args(["build", "--release"])
+        .status()
+        .expect("failed to build responder tools");
+    if !status.success() {
+        return Err(anyhow::anyhow!(
+            "cargo build --release failed to run: {}",
+            status.to_string()
+        ));
+    }
+    Ok(())
+}
 
-    for entry in fs::read_dir(
-        [r"target", opts.bpf_target.to_string().as_str(), r"release"]
-            .iter()
-            .collect::<PathBuf>(),
-    )? {
-        let ebpf_bin = opts.output_folder.join(r"ebpf");
-        fs::create_dir_all(&ebpf_bin)?;
+fn copy_execs_from_dir<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> Result<(), anyhow::Error>{
+    for entry in fs::read_dir(from)? {
         if let Ok(entry) = entry {
             let path = entry.path();
             if path.is_file()
@@ -60,7 +55,7 @@ pub fn build(opts: Options) -> Result<(), anyhow::Error> {
                     .map(|s| s.contains("."))
                     .unwrap_or(true)
             {
-                let to = ebpf_bin.join(path.file_name().unwrap());
+                let to = to.as_ref().join(path.file_name().unwrap());
                 fs::copy(&path, &to).context(format!(
                     "Error copying from {} to {}",
                     path.to_str().unwrap_or("[path not valid unicode]"),
@@ -69,6 +64,32 @@ pub fn build(opts: Options) -> Result<(), anyhow::Error> {
             }
         }
     }
+    Ok(())
+}
+
+
+pub fn build(opts: Options) -> Result<(), anyhow::Error> {
+    fs::create_dir_all(&opts.output_folder).context(format!(
+        "Error creating dirs for path: {}",
+        opts.output_folder
+            .to_str()
+            .unwrap_or("[path not valid unicode]")
+    ))?;
+
+    build_ebpf(build_ebpf::Options {
+        target: opts.bpf_target,
+        release: true,
+    })
+    .context("Error while building eBPF program")?;
+
+    let ebpf_bin = opts.output_folder.join(r"ebpf");
+    fs::create_dir_all(&ebpf_bin)?;
+    copy_execs_from_dir(
+        [r"target", opts.bpf_target.to_string().as_str(), r"release"]
+            .iter()
+            .collect::<PathBuf>(),
+        ebpf_bin
+    )?;
 
     build_user(&opts).context("Error while building userspace application")?;
     fs::copy(
@@ -76,5 +97,11 @@ pub fn build(opts: Options) -> Result<(), anyhow::Error> {
         opts.output_folder.join("responder"),
     )?;
 
+    build_tools(&opts).context("Error while building tool applications")?;
+
+    let tools_bin = opts.output_folder.join(r"tools");
+    fs::create_dir_all(&tools_bin)?;
+    copy_execs_from_dir("target/release", tools_bin)?;
+
     Ok(())
 }

xtask/src/main.rs

@@ -2,7 +2,6 @@ mod build_ebpf;
 mod build;
 mod run;
 mod codegen;
-mod filter;
 
 use std::process::exit;
 
@@ -20,7 +19,6 @@ pub struct Options {
 enum Command {
     BuildEbpf(build_ebpf::Options),
     BuildArtifacts(build::Options),
-    BuildFilterFile(filter::Options),
     Run(run::Options),
     CodeGen,
 }
@@ -36,7 +34,6 @@ fn main() {
         BuildArtifacts(opts) => build::build(opts),
         Run(opts) => run::run(opts),
         CodeGen => codegen::generate(),
-        BuildFilterFile(opts) => filter::build_filter(opts),
     };
 
     if let Err(e) = ret {