Repository: kpcyrd/sn0int (language: rust)

Issue:
geoip lookups
There should be a way to do geoip lookups for ip addresses.


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 35e5b09a..76016d0f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -609,6 +609,16 @@ dependencies = [
  "synstructure 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "filetime"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.6"
@@ -1061,6 +1071,16 @@ name = "matches"
 version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "maxminddb"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "memchr"
 version = "2.1.0"
@@ -1997,8 +2017,10 @@ dependencies = [
  "hlua-badtouch 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "kuchiki 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libflate 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "maxminddb 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "opener 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "publicsuffix 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2015,6 +2037,7 @@ dependencies = [
  "sn0int-common 0.2.0",
  "structopt 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "syscallz 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2212,6 +2235,17 @@ dependencies = [
  "seccomp-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "tar"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.0.4"
@@ -2763,6 +2797,14 @@ dependencies = [
  "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "xattr"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "yansi"
 version = "0.4.0"
@@ -2833,6 +2875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum error-chain 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46"
 "checksum failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6dd377bcc1b1b7ce911967e3ec24fa19c3224394ec05b54aa7b083d498341ac7"
 "checksum failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "64c2d913fe8ed3b6c6518eedf4538255b989945c14c2a7d5cbff62a5e2120596"
+"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
 "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
@@ -2884,6 +2927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43"
 "checksum markup5ever 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a87c4100d614080c8ab43334fb028ebe387f273fb61ed4ff0eae9189b94b6be8"
 "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
+"checksum maxminddb 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f1511a09d61db34ee71f3639421300acc5fb7787431d879473acd096b6c5990"
 "checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b"
 "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
 "checksum migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8cf7c8c4f83fa9f47440c0b4af99973502de55e6e7b875f693bd263e03f93e7e"
@@ -3004,6 +3048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
 "checksum synstructure 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec37f4fab4bafaf6b5621c1d54e6aa5d4d059a8f84929e87abfdd7f9f04c6db2"
 "checksum syscallz 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ba6c26a802c1b51eb4a9c88a37000c6b7e443994d6b53dfcdf85b69dbec045"
+"checksum tar 0.4.17 (registry+https://github.com/rust-lang/crates.io-index)" = "83b0d14b53dbfd62681933fadd651e815f99e6084b649e049ab99296e05ab3de"
 "checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b"
 "checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508"
 "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f"
@@ -3068,4 +3113,5 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum winreg 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a27a759395c1195c4cc5cda607ef6f8f6498f64e78f7900f5de0a127a424704a"
 "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
+"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
 "checksum yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d60c3b48c9cdec42fb06b3b84b5b087405e1fa1c644a1af3930e4dfafe93de48"
diff --git a/Cargo.toml b/Cargo.toml
index 965a124b..ed5baa95 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -49,6 +49,9 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
 toml = "0.4"
+maxminddb = "0.10.0"
+tar = "0.4.17"
+libflate = "0.1.14"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
diff --git a/ci/run.sh b/ci/run.sh
index 75a814f4..c3e3885d 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -7,6 +7,8 @@ case "$1" in
         ;;
     test)
         ci/run.sh build
+        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
+        cargo run --example geoip-download -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
diff --git a/examples/geoip-download.rs b/examples/geoip-download.rs
new file mode 100644
index 00000000..a4e493cf
--- /dev/null
+++ b/examples/geoip-download.rs
@@ -0,0 +1,44 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::paths;
+use std::fs;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    let path = paths::cache_dir()?.join(&args.target);
+    if args.extract_only {
+        let body = fs::read(&args.url)?;
+        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
+    } else {
+        sn0int::geoip::GeoIP::download(path, &args.filter, &args.url)?;
+    }
+    Ok(())
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}
diff --git a/examples/geoip.rs b/examples/geoip.rs
new file mode 100644
index 00000000..9b860380
--- /dev/null
+++ b/examples/geoip.rs
@@ -0,0 +1,30 @@
+extern crate sn0int;
+extern crate env_logger;
+
+use std::env;
+use sn0int::errors::*;
+
+
+fn run() -> Result<()> {
+    let geoip = sn0int::geoip::GeoIP::open_or_download()?;
+
+    for arg in env::args().skip(1) {
+        let ip = arg.parse()?;
+        let lookup = geoip.lookup(ip)?;
+        println!("{:#?}", lookup);
+    }
+
+    Ok(())
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}
diff --git a/migrations/2018-10-22-141819_geoip/down.sql b/migrations/2018-10-22-141819_geoip/down.sql
new file mode 100644
index 00000000..55d11f01
--- /dev/null
+++ b/migrations/2018-10-22-141819_geoip/down.sql
@@ -0,0 +1,21 @@
+PRAGMA foreign_keys=off;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped)
+  SELECT id, family, value, unscoped
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;
diff --git a/migrations/2018-10-22-141819_geoip/up.sql b/migrations/2018-10-22-141819_geoip/up.sql
new file mode 100644
index 00000000..e60d2666
--- /dev/null
+++ b/migrations/2018-10-22-141819_geoip/up.sql
@@ -0,0 +1,7 @@
+ALTER TABLE ipaddrs ADD COLUMN continent VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN continent_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN city VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN latitude FLOAT;
+ALTER TABLE ipaddrs ADD COLUMN longitude FLOAT;
diff --git a/modules/dev/geoip.lua b/modules/dev/geoip.lua
new file mode 100644
index 00000000..8749d0c8
--- /dev/null
+++ b/modules/dev/geoip.lua
@@ -0,0 +1,38 @@
+-- Description: Query certificate transparency logs to discover subdomains
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    lookup = geoip_lookup(arg['value'])
+    if last_err() then return end
+
+    fields = {
+        'continent',
+        'continent_code',
+        'country',
+        'country_code',
+        'city',
+        'latitude',
+        'longitude',
+    }
+
+    update = {}
+    updated = false
+
+    i = 1
+    while i <= #fields do
+        f = fields[i]
+
+        if lookup[f] ~= arg[f] then
+            update[f] = lookup[f]
+            updated = true
+        end
+
+        i = i+1
+    end
+
+    if updated then
+        db_update('ipaddr', arg, update)
+    end
+end
diff --git a/src/archive.rs b/src/archive.rs
new file mode 100644
index 00000000..c5f0890e
--- /dev/null
+++ b/src/archive.rs
@@ -0,0 +1,40 @@
+use errors::*;
+use tar;
+use libflate::gzip;
+use std::io;
+use std::fs::File;
+use std::path::Path;
+
+
+pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
+    let file = gzip::Decoder::new(read)?;
+    let mut ar = tar::Archive::new(file);
+
+    for entry in ar.entries()? {
+        let mut entry = entry?;
+        let file_name = {
+            let path = entry.path()?;
+            path.file_name()
+                .ok_or(format_err!("Invalid path in archive"))?
+                .to_str()
+                .ok_or(format_err!("Filename is invalid utf8"))?
+                .to_owned()
+        };
+
+        debug!("Found in archive: {:?}", file_name);
+
+        if filter == file_name {
+            debug!("Extracting to {:?}", target.as_ref());
+
+            let mut target = File::create(target)?;
+            let n = io::copy(&mut entry, &mut target)?;
+            debug!("Wrote {:?} bytes", n);
+
+            return Ok(());
+        }
+
+        debug!("Skipping file");
+    }
+
+    bail!("Nothing in archive matched filter")
+}
diff --git a/src/db.rs b/src/db.rs
index 2082ad85..b42b53b0 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -31,6 +31,8 @@ impl Database {
                 .context("Failed to connect to database")?;
             migrations::run(&db)
                 .context("Failed to run migrations")?;
+            db.execute("PRAGMA foreign_keys = ON")
+                .context("Failed to enforce foreign keys")?;
             Ok(db)
         }, false)?;
 
@@ -61,6 +63,13 @@ impl Database {
             Insert::IpAddr(object) => self.insert_ipaddr_struct(&NewIpAddr {
                 family: &object.family,
                 value: &object.value,
+                continent: object.continent.as_ref(),
+                continent_code: object.continent_code.as_ref(),
+                country: object.country.as_ref(),
+                country_code: object.country_code.as_ref(),
+                city: object.city.as_ref(),
+                longitude: object.longitude,
+                latitude: object.latitude,
             }),
             Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
                 subdomain_id: object.subdomain_id,
@@ -144,6 +153,13 @@ impl Database {
         let new_ipaddr = NewIpAddr {
             family: &family,
             value: &ipaddr,
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            longitude: None,
+            latitude: None,
         };
 
         self.insert_ipaddr_struct(&new_ipaddr)
@@ -227,27 +243,39 @@ impl Database {
     pub fn update_generic(&self, object: &Update) -> Result<i32> {
         match object {
             Update::Subdomain(object) => self.update_subdomain(object),
+            Update::IpAddr(object) => self.update_ipaddr(object),
             Update::Url(object) => self.update_url(object),
             Update::Email(object) => self.update_email(object),
         }
     }
 
     pub fn update_subdomain(&self, subdomain: &SubdomainUpdate) -> Result<i32> {
-        diesel::update(subdomains::table)
+        use schema::subdomains::columns::*;
+        diesel::update(subdomains::table.filter(id.eq(subdomain.id)))
             .set(subdomain)
             .execute(&self.db)?;
         Ok(subdomain.id)
     }
 
+    pub fn update_ipaddr(&self, ipaddr: &IpAddrUpdate) -> Result<i32> {
+        use schema::ipaddrs::columns::*;
+        diesel::update(ipaddrs::table.filter(id.eq(ipaddr.id)))
+            .set(ipaddr)
+            .execute(&self.db)?;
+        Ok(ipaddr.id)
+    }
+
     pub fn update_url(&self, url: &UrlUpdate) -> Result<i32> {
-        diesel::update(urls::table)
+        use schema::urls::columns::*;
+        diesel::update(urls::table.filter(id.eq(url.id)))
             .set(url)
             .execute(&self.db)?;
         Ok(url.id)
     }
 
     pub fn update_email(&self, email: &EmailUpdate) -> Result<i32> {
-        diesel::update(emails::table)
+        use schema::emails::columns::*;
+        diesel::update(emails::table.filter(id.eq(email.id)))
             .set(email)
             .execute(&self.db)?;
         Ok(email.id)
diff --git a/src/engine/ctx.rs b/src/engine/ctx.rs
index d6ccf665..b9228282 100644
--- a/src/engine/ctx.rs
+++ b/src/engine/ctx.rs
@@ -1,6 +1,7 @@
 use errors::*;
 
 use engine::Reporter;
+use geoip::GeoIP;
 use hlua::{self, AnyLuaValue};
 use models::{Insert, Update};
 use psl::Psl;
@@ -59,6 +60,8 @@ pub trait State {
 
     fn psl(&self) -> Arc<Psl>;
 
+    fn geoip(&self) -> Arc<GeoIP>;
+
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
@@ -73,6 +76,7 @@ pub struct LuaState {
     http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
     dns_config: Arc<DnsConfig>,
     psl: Arc<Psl>,
+    geoip: Arc<GeoIP>,
 }
 
 impl State for LuaState {
@@ -124,6 +128,10 @@ impl State for LuaState {
         self.psl.clone()
     }
 
+    fn geoip(&self) -> Arc<GeoIP> {
+        self.geoip.clone()
+    }
+
     fn http_mksession(&self) -> String {
         let mut mtx = self.http_sessions.lock().unwrap();
         let (id, session) = HttpSession::new();
@@ -151,7 +159,7 @@ pub struct Script {
     code: String,
 }
 
-fn ctx<'a>(dns_config: DnsConfig, psl: Psl) -> (hlua::Lua<'a>, Arc<LuaState>) {
+fn ctx<'a>(dns_config: DnsConfig, psl: Psl, geoip: GeoIP) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
@@ -162,6 +170,7 @@ fn ctx<'a>(dns_config: DnsConfig, psl: Psl) -> (hlua::Lua<'a>, Arc<LuaState>) {
 
         dns_config: Arc::new(dns_config),
         psl: Arc::new(psl),
+        geoip: Arc::new(geoip),
     });
 
     runtime::clear_err(&mut lua, state.clone());
@@ -169,6 +178,7 @@ fn ctx<'a>(dns_config: DnsConfig, psl: Psl) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::db_update(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::geoip_lookup(&mut lua, state.clone());
     runtime::html_select(&mut lua, state.clone());
     runtime::html_select_list(&mut lua, state.clone());
     runtime::http_mksession(&mut lua, state.clone());
@@ -215,8 +225,13 @@ impl Script {
         })
     }
 
-    pub fn run(&self, dns_config: DnsConfig, psl: Psl, tx: Arc<Mutex<Box<Reporter>>>, arg: AnyLuaValue) -> Result<()> {
-        let (mut lua, state) = ctx(dns_config, psl);
+    pub fn run(&self, dns_config: DnsConfig,
+                      psl: Psl,
+                      geoip: GeoIP,
+                      tx: Arc<Mutex<Box<Reporter>>>,
+                      arg: AnyLuaValue
+    ) -> Result<()> {
+        let (mut lua, state) = ctx(dns_config, psl, geoip);
 
         debug!("Initializing lua module");
         lua.execute::<()>(&self.code)?;
@@ -253,6 +268,7 @@ impl Script {
 com
 // ===END ICANN DOMAINS===
 "#)?;
-        self.run(dns_config, psl, DummyReporter::new(), AnyLuaValue::LuaNil)
+        let geoip = GeoIP::open_or_download()?;
+        self.run(dns_config, psl, geoip, DummyReporter::new(), AnyLuaValue::LuaNil)
     }
 }
diff --git a/src/engine/isolation.rs b/src/engine/isolation.rs
index dc42f9e1..3fd6d69b 100644
--- a/src/engine/isolation.rs
+++ b/src/engine/isolation.rs
@@ -2,6 +2,7 @@ use errors::*;
 use channel;
 use chrootable_https::dns::DnsConfig;
 use engine::{Module, Event, Reporter};
+use geoip::GeoIP;
 use psl::Psl;
 use serde_json;
 
@@ -185,13 +186,14 @@ pub fn spawn_module(module: Module, tx: channel::Sender<(Event, Option<mpsc::Sen
     Ok(())
 }
 
-pub fn run_worker() -> Result<()> {
+pub fn run_worker(geoip: GeoIP) -> Result<()> {
     let mut reporter = StdioReporter::setup();
     let start = reporter.recv_start()?;
 
     let mtx: Arc<Mutex<Box<Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
     let result = start.module.run(start.dns_config,
                                   &start.psl,
+                                  geoip,
                                   mtx.clone(),
                                   start.arg.into());
     let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
diff --git a/src/engine/mod.rs b/src/engine/mod.rs
index ac061d9b..132b7507 100644
--- a/src/engine/mod.rs
+++ b/src/engine/mod.rs
@@ -1,5 +1,6 @@
 use errors::*;
 
+use geoip::GeoIP;
 use json::LuaJsonValue;
 use serde_json;
 use std::fs;
@@ -169,11 +170,11 @@ impl Module {
         &self.source
     }
 
-    pub fn run(&self, dns_config: DnsConfig, psl: &str, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, dns_config: DnsConfig, psl: &str, geoip: GeoIP, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Loading public suffix list");
         let psl = Psl::from_str(&psl)?;
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(dns_config, psl, reporter, arg.into())
+        self.script.run(dns_config, psl, geoip, reporter, arg.into())
     }
 
     fn cmp_canonical(&self, other: &Module) -> Ordering {
diff --git a/src/geoip.rs b/src/geoip.rs
new file mode 100644
index 00000000..53bebbef
--- /dev/null
+++ b/src/geoip.rs
@@ -0,0 +1,218 @@
+use errors::*;
+use archive;
+use chrootable_https::Client;
+use maxminddb::{self, geoip2};
+use std::fmt;
+use std::fs::File;
+use std::net::IpAddr;
+use std::collections::BTreeMap;
+use std::path::Path;
+use paths;
+use worker;
+
+pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
+
+
+fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<String> {
+    let names = match names {
+        Some(names) => names,
+        _ => return None,
+    };
+
+    names.get("en")
+        .map(|x| x.to_owned())
+}
+
+#[derive(Debug, Serialize)]
+pub struct Lookup {
+    continent: Option<String>,
+    continent_code: Option<String>,
+    country: Option<String>,
+    country_code: Option<String>,
+    city: Option<String>,
+    latitude: Option<f64>,
+    longitude: Option<f64>,
+}
+
+impl From<geoip2::City> for Lookup {
+    fn from(lookup: geoip2::City) -> Lookup {
+        // parse maxminddb lookup
+        let continent = match lookup.continent {
+            Some(continent) => Continent::from_maxmind(continent),
+            _ => None,
+        };
+        let country = match lookup.country {
+            Some(country) => Country::from_maxmind(country),
+            _ => None,
+        };
+        let city = match lookup.city {
+            Some(city) => from_geoip_model_names(city.names),
+            _ => None,
+        };
+        let location = match lookup.location {
+            Some(location) => Location::from_maxmind(location),
+            _ => None,
+        };
+
+        // flatten datastructure
+        let (continent, continent_code) = match continent {
+            Some(x) => (Some(x.name), Some(x.code)),
+            _ => (None, None),
+        };
+        let (country, country_code) = match country {
+            Some(x) => (Some(x.name), Some(x.code)),
+            _ => (None, None),
+        };
+        let (latitude, longitude) = match location {
+            Some(x) => (Some(x.latitude), Some(x.longitude)),
+            _ => (None, None),
+        };
+
+        // return result
+        Lookup {
+            continent,
+            continent_code,
+            country,
+            country_code,
+            city,
+            latitude,
+            longitude,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Continent {
+    code: String,
+    name: String,
+}
+
+impl Continent {
+    fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
+        let code = match continent.code {
+            Some(code) => code,
+            _ => return None,
+        };
+        let name = match from_geoip_model_names(continent.names) {
+            Some(name) => name,
+            _ => return None,
+        };
+
+        Some(Continent {
+            code,
+            name,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Country {
+    code: String,
+    name: String,
+}
+
+impl Country {
+    fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
+        let code = match country.iso_code {
+            Some(code) => code,
+            _ => return None,
+        };
+        let name = match from_geoip_model_names(country.names) {
+            Some(name) => name,
+            _ => return None,
+        };
+
+        Some(Country {
+            code,
+            name,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Location {
+    latitude: f64,
+    longitude: f64,
+}
+
+impl Location {
+    fn from_maxmind(location: geoip2::model::Location) -> Option<Self> {
+        let latitude = match location.latitude {
+            Some(latitude) => latitude,
+            _ => return None,
+        };
+        let longitude = match location.longitude {
+            Some(longitude) => longitude,
+            _ => return None,
+        };
+
+        Some(Location {
+            latitude,
+            longitude,
+        })
+    }
+}
+
+pub struct GeoIP {
+    reader: maxminddb::Reader,
+}
+
+impl fmt::Debug for GeoIP {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        write!(w, "GeoIP {{ ... }}")
+    }
+}
+
+impl GeoIP {
+    pub fn open(path: &str) -> Result<GeoIP> {
+        let reader = maxminddb::Reader::open(path)
+            .context("Failed to open geoip database")?;
+
+        Ok(GeoIP {
+            reader,
+        })
+    }
+
+    pub fn open_or_download() -> Result<GeoIP> {
+        let path = paths::cache_dir()?.join("GeoLite2-City.mmdb");
+
+        if File::open(&path).is_err() {
+            worker::spawn_fn("Downloading GeoIP city database", || {
+                GeoIP::download(&path, "GeoLite2-City.mmdb", GEOIP_CITY_URL)
+            }, false)?;
+        };
+
+        let path = path.to_str().ok_or(format_err!("Failed to decode path"))?;
+        GeoIP::open(&path)
+    }
+
+    pub fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
+        debug!("Downloading {:?}...", url);
+        let client = Client::with_system_resolver()?;
+        let resp = client.get(url)?;
+        debug!("Downloaded {} bytes", resp.body.len());
+        archive::extract(&mut &resp.body[..], filter, path)?;
+        Ok(())
+    }
+
+    pub fn lookup(&self, ip: IpAddr) -> Result<Lookup> {
+        let city: geoip2::City = self.reader.lookup(ip)?;
+        debug!("GeoIP result: {:?}", city);
+        Ok(Lookup::from(city))
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_geoip_lookup() {
+        let ip = "1.1.1.1".parse().unwrap();
+        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
+        println!("{:#?}", lookup);
+        assert_eq!(lookup.city, None);
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 71033f95..0377ba20 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -26,6 +26,9 @@ extern crate separator;
 extern crate sloppy_rfc4880;
 extern crate regex;
 extern crate toml;
+extern crate maxminddb;
+extern crate tar;
+extern crate libflate;
 extern crate serde;
 extern crate serde_json;
 extern crate serde_urlencoded;
@@ -38,6 +41,7 @@ extern crate serde_urlencoded;
 #[macro_use] extern crate crossbeam_channel as channel;
 
 pub mod api;
+pub mod archive;
 pub mod args;
 pub mod auth;
 pub mod cmd;
@@ -46,6 +50,7 @@ pub mod config;
 pub mod db;
 pub mod errors;
 pub mod engine;
+pub mod geoip;
 pub mod html;
 pub mod json;
 pub mod migrations;
diff --git a/src/main.rs b/src/main.rs
index 7d7f993e..a54a08e6 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,6 +11,8 @@ use sn0int::complete;
 use sn0int::config::Config;
 use sn0int::errors::*;
 use sn0int::engine::{self, Module};
+use sn0int::geoip::GeoIP;
+use sn0int::paths;
 use sn0int::registry;
 use sn0int::sandbox;
 use sn0int::shell;
@@ -43,9 +45,16 @@ fn run_run(gargs: &Args, args: &args::Run, config: Config) -> Result<()> {
 }
 
 fn run_sandbox() -> Result<()> {
+    // TODO: this file should be processed after the sandbox is up
+    let geoip_path = paths::cache_dir()?
+        .join("GeoLite2-City.mmdb");
+    let geoip_path = geoip_path.to_str()
+        .ok_or(format_err!("Failed to decode geoip city path"))?;
+    let geoip = GeoIP::open(&geoip_path)?;
+
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker()
+    engine::isolation::run_worker(geoip)
 }
 
 fn run() -> Result<()> {
diff --git a/src/models/ipaddr.rs b/src/models/ipaddr.rs
index bd7c307e..65ca1950 100644
--- a/src/models/ipaddr.rs
+++ b/src/models/ipaddr.rs
@@ -6,13 +6,20 @@ use std::net;
 use std::result;
 
 
-#[derive(Identifiable, Queryable, Associations, Serialize, PartialEq, Debug)]
+#[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="ipaddrs"]
 pub struct IpAddr {
     pub id: i32,
     pub family: String,
     pub value: String,
     pub unscoped: bool,
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
 }
 
 impl fmt::Display for IpAddr {
@@ -135,6 +142,9 @@ pub struct DetailedIpAddr {
     value: net::IpAddr,
     subdomains: Vec<PrintableSubdomain>,
     unscoped: bool,
+    continent: Option<String>,
+    country: Option<String>,
+    city: Option<String>,
 }
 
 impl fmt::Display for DetailedIpAddr {
@@ -142,11 +152,40 @@ impl fmt::Display for DetailedIpAddr {
         if !self.unscoped {
             write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{}\x1b[0m", self.id, self.value)?;
 
+            if let Some(ref continent) = self.continent {
+                write!(w, " [{}", continent)?;
+
+                if let Some(ref country) = self.country {
+                    write!(w, " / {}", country)?;
+                }
+
+                if let Some(ref city) = self.city {
+                    write!(w, " / {}", city)?;
+                }
+
+                write!(w, "]")?;
+            }
+
             for subdomain in &self.subdomains {
                 write!(w, "\n\t\x1b[33m{}\x1b[0m", subdomain)?;
             }
         } else {
-            write!(w, "\x1b[90m#{}, {}\x1b[0m", self.id, self.value)?;
+            write!(w, "\x1b[90m#{}, {}", self.id, self.value)?;
+
+            if let Some(ref continent) = self.continent {
+                write!(w, " [{}", continent)?;
+
+                if let Some(ref country) = self.country {
+                    write!(w, " / {}", country)?;
+                }
+
+                if let Some(ref city) = self.city {
+                    write!(w, " / {}", city)?;
+                }
+
+                write!(w, "]")?;
+            }
+            write!(w, "\x1b[0m");
 
             for subdomain in &self.subdomains {
                 write!(w, "\n\t\x1b[90m{}\x1b[0m", subdomain)?;
@@ -170,6 +209,9 @@ impl Detailed for IpAddr {
             value: self.value.parse()?,
             subdomains,
             unscoped: self.unscoped,
+            continent: self.continent.clone(),
+            country: self.country.clone(),
+            city: self.city.clone(),
         })
     }
 }
@@ -179,6 +221,13 @@ impl Detailed for IpAddr {
 pub struct NewIpAddr<'a> {
     pub family: &'a str,
     pub value: &'a str,
+    pub continent: Option<&'a String>,
+    pub continent_code: Option<&'a String>,
+    pub country: Option<&'a String>,
+    pub country_code: Option<&'a String>,
+    pub city: Option<&'a String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
 }
 
 #[derive(Debug, Insertable, Serialize, Deserialize)]
@@ -186,6 +235,56 @@ pub struct NewIpAddr<'a> {
 pub struct NewIpAddrOwned {
     pub family: String,
     pub value: String,
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+}
+
+#[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
+#[table_name="ipaddrs"]
+pub struct IpAddrUpdate {
+    pub id: i32,
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+}
+
+impl fmt::Display for IpAddrUpdate {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        let mut updates = Vec::new();
+
+        if let Some(ref continent) = self.continent {
+            updates.push(format!("continent => {:?}", continent));
+        }
+        if let Some(ref continent_code) = self.continent_code {
+            updates.push(format!("continent_code => {:?}", continent_code));
+        }
+        if let Some(ref country) = self.country {
+            updates.push(format!("country => {:?}", country));
+        }
+        if let Some(ref country_code) = self.country_code {
+            updates.push(format!("country_code => {:?}", country_code));
+        }
+        if let Some(ref city) = self.city {
+            updates.push(format!("city => {:?}", city));
+        }
+        if let Some(ref latitude) = self.latitude {
+            updates.push(format!("latitude => {:?}", latitude));
+        }
+        if let Some(ref longitude) = self.longitude {
+            updates.push(format!("longitude => {:?}", longitude));
+        }
+
+        write!(w, "{}", updates.join(", "))
+    }
 }
 
 impl Printable<PrintableIpAddr> for NewIpAddrOwned {
diff --git a/src/models/mod.rs b/src/models/mod.rs
index e55e1633..6e2de55f 100644
--- a/src/models/mod.rs
+++ b/src/models/mod.rs
@@ -30,6 +30,7 @@ impl Insert {
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Update {
     Subdomain(SubdomainUpdate),
+    IpAddr(IpAddrUpdate),
     Url(UrlUpdate),
     Email(EmailUpdate),
 }
@@ -38,6 +39,7 @@ impl fmt::Display for Update {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Update::Subdomain(update) => write!(w, "{}", update),
+            Update::IpAddr(update) => write!(w, "{}", update),
             Update::Url(update) => write!(w, "{}", update),
             Update::Email(update) => write!(w, "{}", update),
         }
diff --git a/src/models/url.rs b/src/models/url.rs
index 7f8fa9ae..88196af0 100644
--- a/src/models/url.rs
+++ b/src/models/url.rs
@@ -113,13 +113,13 @@ impl fmt::Display for UrlUpdate {
         let mut updates = Vec::new();
 
         if let Some(online) = self.online {
-            updates.push(format!("online => {:?}, ", online));
+            updates.push(format!("online => {:?}", online));
         }
         if let Some(status) = self.status {
-            updates.push(format!("status => {:?}, ", status));
+            updates.push(format!("status => {:?}", status));
         }
         if let Some(ref body) = self.body {
-            updates.push(format!("body => [{} bytes], ", body.len()));
+            updates.push(format!("body => [{} bytes]", body.len()));
         }
 
         write!(w, "{}", updates.join(", "))
diff --git a/src/runtime/db.rs b/src/runtime/db.rs
index ff4e0482..f0342159 100644
--- a/src/runtime/db.rs
+++ b/src/runtime/db.rs
@@ -85,7 +85,9 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<State>) {
             Family::Subdomain => structs::from_lua::<Domain>(object)
                     .map(|x| (x.id, x.to_string()))
                     .map_err(|e| state.set_error(e)),
-            Family::IpAddr => bail!("IpAddr doesn't have mutable fields"),
+            Family::IpAddr => structs::from_lua::<IpAddr>(object)
+                    .map(|x| (x.id, x.to_string()))
+                    .map_err(|e| state.set_error(e)),
             Family::SubdomainIpAddr => bail!("Unsupported operation"),
             Family::Url => structs::from_lua::<Url>(object)
                     .map(|x| (x.id, x.to_string()))
@@ -102,20 +104,15 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<State>) {
 
         let update = match family {
             Family::Domain => bail!("Domain doesn't have mutable fields"),
-            Family::Subdomain => {
-                Update::Subdomain(structs::from_lua::<SubdomainUpdate>(update)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::IpAddr => bail!("IpAddr doesn't have mutable fields"),
+            Family::Subdomain => Update::Subdomain(structs::from_lua::<SubdomainUpdate>(update)
+                .map_err(|e| state.set_error(e))?),
+            Family::IpAddr => Update::IpAddr(structs::from_lua::<IpAddrUpdate>(update)
+                .map_err(|e| state.set_error(e))?),
             Family::SubdomainIpAddr => bail!("Unsupported operation"),
-            Family::Url => {
-                Update::Url(structs::from_lua::<UrlUpdate>(update)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Email => {
-                Update::Email(structs::from_lua::<EmailUpdate>(update)
-                    .map_err(|e| state.set_error(e))?)
-            },
+            Family::Url => Update::Url(structs::from_lua::<UrlUpdate>(update)
+                .map_err(|e| state.set_error(e))?),
+            Family::Email => Update::Email(structs::from_lua::<EmailUpdate>(update)
+                .map_err(|e| state.set_error(e))?),
         };
 
         state.db_update(object, update)
diff --git a/src/runtime/geoip.rs b/src/runtime/geoip.rs
new file mode 100644
index 00000000..e9256038
--- /dev/null
+++ b/src/runtime/geoip.rs
@@ -0,0 +1,42 @@
+use errors::*;
+
+use engine::ctx::State;
+use json::LuaJsonValue;
+use hlua::{self, AnyLuaValue};
+use serde_json;
+use std::sync::Arc;
+use std::net::IpAddr;
+
+
+pub fn geoip_lookup(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("geoip_lookup", hlua::function1(move |ip: String| -> Result<AnyLuaValue> {
+        let ip = ip.parse::<IpAddr>()
+            .context("Failed to parse IP")
+            .map_err(|err| state.set_error(err.into()))?;
+
+        let lookup = state.geoip().lookup(ip)
+            .map_err(|err| state.set_error(err))?;
+
+        let lookup = serde_json::to_value(lookup)
+            .map_err(|e| state.set_error(e.into()))?;
+
+        Ok(LuaJsonValue::from(lookup).into())
+    }))
+}
+
+
+#[cfg(test)]
+mod tests {
+    use engine::ctx::Script;
+
+    #[test]
+    fn verify_geoip() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = geoip_lookup('1.1.1.1')
+            print(x)
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}
diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs
index ad30d5ba..3182d716 100644
--- a/src/runtime/mod.rs
+++ b/src/runtime/mod.rs
@@ -8,6 +8,7 @@ macro_rules! import_fns {
 import_fns!(db);
 import_fns!(dns);
 import_fns!(error);
+import_fns!(geoip);
 import_fns!(http);
 import_fns!(html);
 import_fns!(json);
diff --git a/src/schema.rs b/src/schema.rs
index 4f7bf2a2..2ee7f53d 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -21,6 +21,13 @@ table! {
         family -> Text,
         value -> Text,
         unscoped -> Bool,
+        continent -> Nullable<Text>,
+        continent_code -> Nullable<Text>,
+        country -> Nullable<Text>,
+        country_code -> Nullable<Text>,
+        city -> Nullable<Text>,
+        latitude -> Nullable<Float>,
+        longitude -> Nullable<Float>,
     }
 }
 
diff --git a/src/shell.rs b/src/shell.rs
index 2f53f808..cb21dfed 100644
--- a/src/shell.rs
+++ b/src/shell.rs
@@ -8,6 +8,7 @@ use colored::Colorize;
 use ctrlc;
 use db::Database;
 use engine::{Engine, Module};
+use geoip::GeoIP;
 use rustyline::error::ReadlineError;
 use rustyline::{self, CompletionType, EditMode, Editor};
 use shellwords;
@@ -317,6 +318,7 @@ pub fn init(args: &Args, config: Config) -> Result<Readline> {
 
     let db = Database::establish(workspace)?;
     let psl = Psl::open_or_download()?;
+    let _geoip = GeoIP::open_or_download()?;
     let engine = Engine::new()?;
     let rl = Readline::new(config, db, psl, engine);
 


Test patch: