Repository: kpcyrd/sn0int (language: rust)

Issue:
Enhancement request: Mark modules already installed in some way during mod search
As the module list grows, it would be nice to be able to tell if a module is installed already or not.

[featured] shows for the modules that are installed by the `quickstart` command.  It would be nice if we could somehow tell if a module is also installed already
maybe just [installed] ?


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 07b15615..d273cde1 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,18 @@ For everything else please have a look at the [detailed list][1].
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
   - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
@@ -95,9 +106,9 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
-- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
-- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
-- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
diff --git a/contrib/html-toc2md.pl b/contrib/html-toc2md.pl
index 7c823a7e..9b30618f 100755
--- a/contrib/html-toc2md.pl
+++ b/contrib/html-toc2md.pl
@@ -24,11 +24,11 @@
 # generate new toc
 while (my $line = <$t>) {
     if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
-        my $space = $1;
+        my $space = int($1)-1;
         my $section = $2;
         my $label = $3;
         $label =~ s/([\[\]])/\\$1/g;
-        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
     }
 }
 print;
diff --git a/docs/build.rst b/docs/build.rst
new file mode 100644
index 00000000..18ccd49c
--- /dev/null
+++ b/docs/build.rst
@@ -0,0 +1,80 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+None.
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+Note that you need a different command on windows:
+
+.. code-block:: bash
+
+    $ cargo build --release --features=sqlite-bundled
+
+After the build finished the binary is located at ``target/release/sn0int``.
diff --git a/docs/index.rst b/docs/index.rst
index 044178e4..d6ceb83f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -36,6 +36,7 @@ Getting Started
    :glob:
 
    install
+   build
    usage
    scripting
    database
diff --git a/docs/install.rst b/docs/install.rst
index f0f96d98..36b9aab7 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -40,32 +42,39 @@ Docker
 
 .. code-block:: bash
 
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
+On alpine edge, with enabled testing repositories:
+
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ apk add sn0int
 
 OpenBSD
 -------
 
+On -current:
+
 .. code-block:: bash
 
-    $ pkg_add git cargo sqlite3 geolite2-city geolite2-asn
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
+    emerge --ask net-analyzer/sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
 
 Make sure rust is installed and setup.
 
diff --git a/modules/dev/spyse-subdomains.lua b/modules/dev/spyse-subdomains.lua
new file mode 100644
index 00000000..cd0c7acd
--- /dev/null
+++ b/modules/dev/spyse-subdomains.lua
@@ -0,0 +1,49 @@
+-- Description: Request subdomains from spyse.com
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+-- Keyring-Access: spyse
+
+function run(arg)
+    local key = keyring('spyse')[1]
+    if not key then
+        return 'Missing required spyse access key'
+    end
+    local access_key = key['access_key']
+
+    local session = http_mksession()
+
+    local page=1
+    while true do
+        local req = http_request(session, 'GET', 'https://api.spyse.com/v1/subdomains', {
+            query={
+                api_token=key['access_key'],
+                domain=arg['value'],
+                page=''..page,
+            }
+        })
+        debug('sending request for page #' .. page)
+        local r = http_send(req)
+        if last_err() then return end
+        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+        local data = json_decode(r['text'])
+        if last_err() then return end
+
+        if #data['records'] == 0 then
+            break
+        end
+
+        for i=1, #data['records'] do
+            local record = data['records'][i]
+            local subdomain = record['domain']
+            -- info(record['ip']['ip'])
+            db_add('subdomain', {
+                domain_id=arg['id'],
+                value=subdomain,
+            })
+        end
+
+        page = page+1
+    end
+end
diff --git a/src/args.rs b/src/args.rs
index 2900dd14..f06d2f49 100644
--- a/src/args.rs
+++ b/src/args.rs
@@ -136,6 +136,9 @@ pub struct Install {
 
 #[derive(Debug, StructOpt)]
 pub struct Search {
+    /// Only show modules that aren't installed yet
+    #[structopt(long="new")]
+    pub new: bool,
     /// The search query
     pub query: String,
 }
diff --git a/src/cmd/mod_cmd.rs b/src/cmd/mod_cmd.rs
index 540dcb20..e5ba8602 100644
--- a/src/cmd/mod_cmd.rs
+++ b/src/cmd/mod_cmd.rs
@@ -44,6 +44,9 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct List {
+    /// Only show modules with a specific input source
+    #[structopt(long="source")]
+    pub source: Option<String>,
 }
 
 #[derive(Debug, StructOpt)]
@@ -86,8 +89,14 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let config = rl.config().clone();
 
     match args.subcommand {
-        SubCommand::List(_) => {
+        SubCommand::List(list) => {
             for module in rl.engine().list() {
+                if let Some(source) = &list.source {
+                    if !module.source_equals(&source) {
+                        continue;
+                    }
+                }
+
                 println!("{} ({})", module.canonical().green(),
                                     module.version().yellow());
                 println!("\t{}", module.description());
@@ -98,7 +107,7 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
             // trigger reload
             run(rl, &[String::from("mod"), String::from("reload")])?;
         },
-        SubCommand::Search(search) => registry::run_search(&search, &config)?,
+        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
         SubCommand::Reload(_) => {
             let current = rl.take_module()
                             .map(|m| m.canonical());
diff --git a/src/engine/mod.rs b/src/engine/mod.rs
index 09592fec..8af770e7 100644
--- a/src/engine/mod.rs
+++ b/src/engine/mod.rs
@@ -181,14 +181,22 @@ impl<'a> Engine<'a> {
     }
 
     pub fn get(&self, name: &str) -> Result<&Module> {
+        if let Some(module) = self.get_opt(name)? {
+            Ok(module)
+        } else {
+            bail!("Module not found")
+        }
+    }
+
+    pub fn get_opt(&self, name: &str) -> Result<Option<&Module>> {
         if let Some(modules) = self.modules.get(name) {
             if modules.len() != 1 {
                 bail!("Ambiguous name: {:?}", modules)
             } else {
-                Ok(&modules[0])
+                Ok(Some(&modules[0]))
             }
         } else {
-            bail!("Module not found")
+            Ok(None)
         }
     }
 
@@ -299,6 +307,13 @@ impl Module {
             self.author.cmp(&other.author)
         }
     }
+
+    pub fn source_equals(&self, other: &str) -> bool {
+        match self.source() {
+            Some(source) => source.group_as_str() == other,
+            None => other == "",
+        }
+    }
 }
 
 pub trait Reporter: Debug {
diff --git a/src/main.rs b/src/main.rs
index 86634204..ed105d2a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -9,7 +9,7 @@ use sn0int::cmd;
 use sn0int::complete;
 use sn0int::config::Config;
 use sn0int::errors::*;
-use sn0int::engine::{self, Module};
+use sn0int::engine::{self, Engine, Module};
 use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
 use sn0int::options::Opt;
 use sn0int::psl::Psl;
@@ -97,7 +97,10 @@ fn run() -> Result<()> {
         Some(SubCommand::New(new)) => run_new(&args, &new),
         Some(SubCommand::Publish(publish)) => registry::run_publish(&args, &publish, &config),
         Some(SubCommand::Install(install)) => registry::run_install(&install, &config),
-        Some(SubCommand::Search(search)) => registry::run_search(&search, &config),
+        Some(SubCommand::Search(search)) => {
+            let engine = Engine::new(false, &config)?;
+            registry::run_search(&engine, &search, &config)
+        },
         Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
         Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
         Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
diff --git a/src/registry.rs b/src/registry.rs
index 4493f637..9b5a6f36 100644
--- a/src/registry.rs
+++ b/src/registry.rs
@@ -3,6 +3,7 @@ use crate::args::{Args, Publish, Install, Search};
 use crate::api::Client;
 use crate::auth;
 use crate::config::Config;
+use crate::engine::Engine;
 use colored::Colorize;
 use separator::Separatable;
 use sn0int_common::metadata::Metadata;
@@ -86,7 +87,7 @@ pub fn run_install(install: &Install, config: &Config) -> Result<()> {
     }, false)
 }
 
-pub fn run_search(search: &Search, config: &Config) -> Result<()> {
+pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<()> {
     let client = Client::new(&config)?;
 
     let label = format!("Searching {:?}", search.query);
@@ -95,10 +96,18 @@ pub fn run_search(search: &Search, config: &Config) -> Result<()> {
     }, true)?;
 
     for module in &modules {
-        println!("{} ({}) - {} downloads {}", module.canonical().green(),
+        let canonical = module.canonical();
+        let installed = engine.get_opt(&canonical)?;
+
+        if search.new && installed.is_some() {
+            continue;
+        }
+
+        println!("{} ({}) - {} downloads{}{}", canonical.green(),
                             module.latest.yellow(),
                             module.downloads.separated_string(),
-                            (if module.featured { "[featured]" } else { "" }).cyan());
+                            (if module.featured { " [featured]" } else { "" }).cyan(),
+                            (if installed.is_some() { " [installed]" } else { "" }).green());
         println!("\t{}", module.description);
     }
 


Test patch: