Repository: kpcyrd/sn0int (language: rust)

Issue:
subdomain can be a tld according to psl
The following script fails with an error:
```
-- Description: TODO your description here
-- Version: 0.1.0
-- License: GPL-3.0

function run()
    info(psl_domain_from_dns_name('a.prod.fastly.net'))
end
```

This is because `a.prod.fastly.net` is on the public suffix list and is therefore equal with `com`. `psl_domain_from_dns_name` and the `domains` table are psl+1. I'm considering changing the rules slightly so that it's still supposed to be psl+1, but also accepts psl entries.

---

## Versions

- **sn0int --version:** 0.12.0


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index ac604ee0..38e8ff19 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2425,15 +2425,25 @@ dependencies = [
 
 [[package]]
 name = "rocket_failure"
-version = "0.1.2"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "rocket 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rocket_failure_errors 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rocket_failure_errors"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rocket_http"
 version = "0.4.2"
@@ -2784,7 +2794,6 @@ dependencies = [
  "dirs 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hlua-badtouch 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2799,7 +2808,6 @@ dependencies = [
  "maxminddb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "nude 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "opener 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pledge 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2839,7 +2847,7 @@ version = "0.7.0"
 dependencies = [
  "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "rocket_failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rocket_failure_errors 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2862,7 +2870,7 @@ dependencies = [
  "reqwest 0.9.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocket 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocket_contrib 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rocket_failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rocket_failure 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3982,7 +3990,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum rocket 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "42c1e9deb3ef4fa430d307bfccd4231434b707ca1328fae339c43ad1201cc6f7"
 "checksum rocket_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "79aa1366f9b2eccddc05971e17c5de7bb75a5431eb12c2b5c66545fd348647f4"
 "checksum rocket_contrib 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e0fa5c1392135adc0f96a02ba150ac4c765e27c58dbfd32aa40678e948f6e56f"
-"checksum rocket_failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b4be7ec53a7d81e2fb2bd48febb34aa9d52a8bc518ac4ee7cb68c05d9b32fb95"
+"checksum rocket_failure 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b62e0d82ae96e59ec6af6cf3dbfc6fcb71adcfbb10188909338dfeed1fe3ab50"
+"checksum rocket_failure_errors 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b125cdd87ffc0c5e453153bee0224aade4c13e57dc2d2663d077d42bd0060e2"
 "checksum rocket_http 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b1391457ee4e80b40d4b57fa5765c0f2836b20d73bcbee4e3f35d93cf3b80817"
 "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
diff --git a/Cargo.toml b/Cargo.toml
index 1b1896d2..d8e7b910 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,6 @@ libflate = "0.1.14"
 threadpool = "1.7"
 x509-parser = "0.5.0"
 der-parser = "2.0"
-nom = "4.1.1"
 atty = "0.2"
 bufstream = "0.1.4"
 tokio = "0.1.14"
@@ -72,7 +71,6 @@ bytesize = "1.0"
 ipnetwork = "0.14"
 
 digest = "0.8.0"
-hex = "0.3.1"
 bs58 = "0.2.2"
 blake2 = "0.8.0"
 md-5 = "0.8.0"
diff --git a/README.md b/README.md
index dd04f839..e3b092c7 100644
--- a/README.md
+++ b/README.md
@@ -153,6 +153,7 @@ For everything else please have a look at the [detailed list][1].
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
@@ -169,12 +170,15 @@ For everything else please have a look at the [detailed list][1].
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
   - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
   - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
   - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
   - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
   - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
diff --git a/docs/reference.rst b/docs/reference.rst
index 4298dc02..b9cbbdc2 100644
--- a/docs/reference.rst
+++ b/docs/reference.rst
@@ -375,7 +375,7 @@ This function may fail.
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
 
 http_send
 ---------
@@ -403,7 +403,26 @@ the following keys:
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch_json
+---------------
+
+This is a shorthand for http_send_, validating the status code and parsing the
+response body as json.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch_json(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+    data = json_decode(resp['text'])
+    if last_err() then return end
 
 img_load
 --------
@@ -630,6 +649,41 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+semver_match
+------------
+
+Compare a version to a version requirement. This can be used with
+sn0int_version_ to test for certain features or behavior.
+
+.. code-block:: lua
+
+    semver_match('=0.11.2', sn0int_version())
+    semver_match('>0.11.2', sn0int_version())
+    semver_match('<0.11.2', sn0int_version())
+    semver_match('~0.11.2', sn0int_version())
+    semver_match('^0.11.2', sn0int_version())
+    semver_match('0.11.2', sn0int_version()) -- synonym for ^0.11.2
+    semver_match('<=0.11.2', sn0int_version())
+    semver_match('>=0.11.2', sn0int_version())
+    semver_match('>=0.4.0, <=0.10.0', sn0int_version())
+
+set_err
+-------
+
+Manipulate the global error object. If you want to exit the main ``run``
+function with an error you can simply return a string, but those are difficult
+to propagate through functions. ``set_err`` specifically assigns an error to
+the global error object that are also used by all other rust functions.
+
+.. code-block:: lua
+
+    function foo()
+        set_err("something failed")
+    end
+
+    foo()
+    if last_err() then return end
+
 sha1
 ----
 
@@ -685,6 +739,16 @@ only used for debugging.
 
     sleep(1)
 
+sn0int_version
+--------------
+
+Get the current sn0int version string. This can be used with semver_match_ to
+test for certain features or behavior.
+
+.. code-block:: lua
+
+    info(sn0int_version())
+
 sock_connect
 ------------
 
diff --git a/sn0int-registry/Cargo.toml b/sn0int-registry/Cargo.toml
index f87ed291..f89d3d06 100644
--- a/sn0int-registry/Cargo.toml
+++ b/sn0int-registry/Cargo.toml
@@ -10,7 +10,8 @@ edition = "2018"
 [dependencies]
 sn0int-common = { version="0.7.0", path="sn0int-common" }
 rocket = { version = "0.4", default-features=false }
-rocket_failure = { version = "0.1.2", features = ["with-rocket"] }
+#rocket_failure = { path = "../../rocket_failure" }
+rocket_failure = { version = "0.2" }
 rocket_contrib = { version = "0.4.1", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
diff --git a/sn0int-registry/sn0int-common/Cargo.toml b/sn0int-registry/sn0int-common/Cargo.toml
index 306b1111..61fe79c9 100644
--- a/sn0int-registry/sn0int-common/Cargo.toml
+++ b/sn0int-registry/sn0int-common/Cargo.toml
@@ -10,7 +10,7 @@ edition = "2018"
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
-#rocket_failure = { path = "../../../rocket_failure" }
-rocket_failure = "0.1.1"
+#rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
+rocket_failure_errors = "0.2"
 failure = "0.1"
 nom = "4.0"
diff --git a/sn0int-registry/sn0int-common/src/lib.rs b/sn0int-registry/sn0int-common/src/lib.rs
index ac4cde6a..cd307566 100644
--- a/sn0int-registry/sn0int-common/src/lib.rs
+++ b/sn0int-registry/sn0int-common/src/lib.rs
@@ -9,7 +9,7 @@ pub mod metadata;
 pub mod id;
 pub use crate::id::*;
 
-pub use rocket_failure::StrictApiResponse as ApiResponse;
+pub use rocket_failure_errors::StrictApiResponse as ApiResponse;
 
 #[cfg(test)]
 mod tests {
diff --git a/src/cmd/add_cmd.rs b/src/cmd/add_cmd.rs
index 3cc8fd4f..4cca9d19 100644
--- a/src/cmd/add_cmd.rs
+++ b/src/cmd/add_cmd.rs
@@ -107,11 +107,11 @@ impl IntoInsert for AddDomain {
         };
 
         // ensure input is a valid domain
-        let parsed_domain = rl.psl()?.parse_domain(&domain)
+        let dns_name = rl.psl()?.parse_dns_name(&domain)
             .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
 
-        if Some(domain.as_str()) != parsed_domain.root() {
-            bail!("This is not a valid domain, might be a subdomain or tld");
+        if dns_name.fulldomain.is_some() {
+            bail!("Domain has an unexpected subdomain, add as a subdomain instead");
         }
 
         Ok(Insert::Domain(NewDomain {
@@ -136,12 +136,8 @@ impl IntoInsert for AddSubdomain {
         let dns_name = rl.psl()?.parse_dns_name(&subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
-        let domain = dns_name.domain()
-            .ok_or_else(|| format_err!("Dns Name seems invalid"))?
-            .to_string();
-
         let domain_id = match rl.db().insert_struct(NewDomain {
-            value: domain,
+            value: dns_name.root,
             unscoped: false,
         }, true)? {
             Some((_, domain_id)) => domain_id,
diff --git a/src/engine/ctx.rs b/src/engine/ctx.rs
index be7106c3..18b71c50 100644
--- a/src/engine/ctx.rs
+++ b/src/engine/ctx.rs
@@ -373,6 +373,7 @@ fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a
     runtime::http_mksession(&mut lua, state.clone());
     runtime::http_request(&mut lua, state.clone());
     runtime::http_send(&mut lua, state.clone());
+    runtime::http_fetch_json(&mut lua, state.clone());
     runtime::img_exif(&mut lua, state.clone());
     runtime::img_load(&mut lua, state.clone());
     runtime::img_nudity(&mut lua, state.clone());
@@ -390,6 +391,7 @@ fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a
     runtime::regex_find(&mut lua, state.clone());
     runtime::regex_find_all(&mut lua, state.clone());
     runtime::semver_match(&mut lua, state.clone());
+    runtime::set_err(&mut lua, state.clone());
     runtime::sha1(&mut lua, state.clone());
     runtime::sha2_256(&mut lua, state.clone());
     runtime::sha2_512(&mut lua, state.clone());
@@ -493,6 +495,9 @@ impl Script {
 // ===BEGIN ICANN DOMAINS===
 com
 // ===END ICANN DOMAINS===
+// ===BEGIN PRIVATE DOMAINS===
+a.prod.fastly.net
+// ===END PRIVATE DOMAINS===
 "#.into());
         let geoip = GeoIP::open_reader()?;
         let asn = AsnDB::open_reader()?;
diff --git a/src/json.rs b/src/json.rs
index fcd625ff..8f98491c 100644
--- a/src/json.rs
+++ b/src/json.rs
@@ -6,8 +6,8 @@ use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
 
 
-pub fn decode(x: &str) -> Result<AnyLuaValue> {
-    let v: Value = serde_json::from_str(&x)
+pub fn decode<T: AsRef<[u8]>>(x: T) -> Result<AnyLuaValue> {
+    let v: Value = serde_json::from_slice(x.as_ref())
                         .context("deserialize failed")?;
     let v: LuaJsonValue = v.into();
     Ok(v.into())
diff --git a/src/psl.rs b/src/psl.rs
index ece14ae4..0fe4d93e 100644
--- a/src/psl.rs
+++ b/src/psl.rs
@@ -6,10 +6,16 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use crate::paths;
 use crate::lazy::LazyInit;
-use publicsuffix::{self, Domain, DnsName};
 use crate::worker;
 
 
+#[derive(Debug, PartialEq)]
+pub struct DnsName {
+    pub fulldomain: Option<String>,
+    pub root: String,
+    pub suffix: String,
+}
+
 #[derive(Debug)]
 pub enum PslReader {
     Reader(File),
@@ -85,13 +91,137 @@ pub struct Psl {
 }
 
 impl Psl {
-    pub fn parse_domain(&self, domain: &str) -> Result<Domain> {
-        self.list.parse_domain(domain)
-            .map_err(|e| format_err!("Failed to parse domain: {}", e))
+    pub fn parse_dns_name(&self, name: &str) -> Result<DnsName> {
+        let dns_name = self.list.parse_dns_name(name)
+            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+
+        let (root, suffix) = if let Some(domain) = dns_name.domain() {
+            let root = domain.to_string();
+            if let Some(suffix) = domain.suffix() {
+                (root, suffix.to_string())
+            } else {
+                // XXX: not sure if this is reachable
+                (root.clone(), root)
+            }
+        } else {
+            // this is technically a tld, but support eg. a.prod.fastly.net anyway
+            // XXX: consider forcing domain to be in self.list.private()
+            (name.to_string(), name.to_string())
+        };
+
+        let fulldomain = if root.as_str() != name {
+            Some(name.to_string())
+        } else {
+            None
+        };
+
+        Ok(DnsName {
+            fulldomain,
+            root,
+            suffix,
+        })
     }
+}
 
-    pub fn parse_dns_name(&self, name: &str) -> Result<DnsName> {
-        self.list.parse_dns_name(name)
-            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn init() -> Arc<Psl> {
+        PslReader::String(r#"
+// ===BEGIN ICANN DOMAINS===
+com
+// ===END ICANN DOMAINS===
+// ===BEGIN PRIVATE DOMAINS===
+a.prod.fastly.net
+// ===END PRIVATE DOMAINS===
+"#.into()).initialize().unwrap()
+    }
+
+    #[test]
+    fn test_psl_example_com() {
+        let x = init().parse_dns_name("example.com").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: None,
+            root: "example.com".into(),
+            suffix: "com".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_www_example_com() {
+        let x = init().parse_dns_name("www.example.com").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: Some("www.example.com".into()),
+            root: "example.com".into(),
+            suffix: "com".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_com() {
+        let x = init().parse_dns_name("com").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: None,
+            root: "com".into(),
+            suffix: "com".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_a_b_c_d_e_f_g_com() {
+        let x = init().parse_dns_name("a.b.c.d.e.f.g.com").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: Some("a.b.c.d.e.f.g.com".into()),
+            root: "g.com".into(),
+            suffix: "com".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_empty() {
+        let x = init().parse_dns_name("").is_err();
+        assert!(x);
+    }
+
+    #[test]
+    fn test_psl_asdfinvalid() {
+        let x = init().parse_dns_name("asdfinvalid").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: None,
+            root: "asdfinvalid".into(),
+            suffix: "asdfinvalid".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_www_example_asdfinvalid() {
+        let x = init().parse_dns_name("www.example.asdfinvalid").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: Some("www.example.asdfinvalid".into()),
+            root: "example.asdfinvalid".into(),
+            suffix: "asdfinvalid".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_a_prod_fastly_net() {
+        let x = init().parse_dns_name("a.prod.fastly.net").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: None,
+            root: "a.prod.fastly.net".into(),
+            suffix: "a.prod.fastly.net".into(),
+        });
+    }
+
+    #[test]
+    fn test_psl_www_a_prod_fastly_net() {
+        let x = init().parse_dns_name("www.a.prod.fastly.net").expect("parse_dns_name");
+        assert_eq!(x, DnsName {
+            fulldomain: None,
+            root: "www.a.prod.fastly.net".into(),
+            suffix: "a.prod.fastly.net".into(),
+        });
     }
 }
diff --git a/src/runtime/error.rs b/src/runtime/error.rs
index 954b3605..fee94b62 100644
--- a/src/runtime/error.rs
+++ b/src/runtime/error.rs
@@ -17,3 +17,9 @@ pub fn last_err(lua: &mut hlua::Lua, state: Arc<State>) {
         }
     }))
 }
+
+pub fn set_err(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("set_err", hlua::function1(move |err: String| {
+        state.set_error(failure::err_msg(err));
+    }))
+}
diff --git a/src/runtime/http.rs b/src/runtime/http.rs
index c3483441..815934d5 100644
--- a/src/runtime/http.rs
+++ b/src/runtime/http.rs
@@ -2,6 +2,7 @@ use crate::errors::*;
 
 use crate::engine::ctx::State;
 use crate::hlua::{self, AnyLuaValue, AnyHashableLuaValue};
+use crate::json;
 use std::sync::Arc;
 use std::collections::HashMap;
 use crate::web::{RequestOptions, HttpRequest};
@@ -26,18 +27,34 @@ pub fn http_request(lua: &mut hlua::Lua, state: Arc<State>) {
 
 pub fn http_send(lua: &mut hlua::Lua, state: Arc<State>) {
     lua.set("http_send", hlua::function1(move |request: AnyLuaValue| -> Result<HashMap<AnyHashableLuaValue, AnyLuaValue>> {
-        let req = match HttpRequest::try_from(request)
-                                .context("invalid http request object") {
-            Ok(req) => req,
-            Err(err) => return Err(state.set_error(Error::from(err))),
-        };
+        let req = HttpRequest::try_from(request)
+            .context("invalid http request object")
+            .map_err(|err| state.set_error(err.into()))?;
 
-        req.send(state.as_ref())
+        req.send_lua(state.as_ref())
             .map_err(|err| state.set_error(err))
             .map(|resp| resp.into())
     }))
 }
 
+pub fn http_fetch_json(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("http_fetch_json", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
+        let req = HttpRequest::try_from(request)
+            .context("invalid http request object")
+            .map_err(|err| state.set_error(err.into()))?;
+
+        let resp = req.send(state.as_ref())
+            .map_err(|err| state.set_error(err))?;
+
+        if resp.status < 200 || resp.status > 299 {
+            return Err(state.set_error(format_err!("http status error: {}", resp.status)));
+        }
+
+        json::decode(&resp.body)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
 
 #[cfg(test)]
 mod tests {
@@ -146,4 +163,48 @@ mod tests {
         "#).expect("failed to load script");
         script.test().expect("Script failed");
     }
+
+    #[test]
+    #[ignore]
+    fn verify_fetch_ok() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            req = http_request(session, "GET", "https://httpbin.org/anything", {})
+            x = http_fetch_json(req)
+            if last_err() then return end
+            print(x)
+            if x['method'] ~= 'GET' then
+                return 'unexpected response'
+            end
+        end
+        "#).expect("failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    #[ignore]
+    fn verify_fetch_404() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            req = http_request(session, "GET", "https://httpbin.org/status/404", {})
+            x = http_fetch_json(req)
+        end
+        "#).expect("failed to load script");
+        script.test().err().expect("Script should have failed");
+    }
+
+    #[test]
+    #[ignore]
+    fn verify_fetch_invalid_json() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            req = http_request(session, "GET", "https://httpbin.org/html", {})
+            x = http_fetch_json(req)
+        end
+        "#).expect("failed to load script");
+        script.test().err().expect("Script should have failed");
+    }
 }
diff --git a/src/runtime/psl.rs b/src/runtime/psl.rs
index fa9d8371..20f1779f 100644
--- a/src/runtime/psl.rs
+++ b/src/runtime/psl.rs
@@ -11,12 +11,7 @@ pub fn psl_domain_from_dns_name(lua: &mut hlua::Lua, state: Arc<State>) {
         let dns_name = psl.parse_dns_name(&dns_name)
             .map_err(|err| state.set_error(err))?;
 
-        let domain = dns_name.domain()
-            .ok_or_else(|| format_err!("Failed to get domain from {:?}", dns_name))
-            .map_err(|err| state.set_error(err))?
-            .to_string();
-
-        Ok(domain)
+        Ok(dns_name.root)
     }))
 }
 
@@ -30,7 +25,7 @@ mod tests {
         function run()
             domain = psl_domain_from_dns_name("foo.example.com")
             if domain ~= "example.com" then 
-                return 'unexpected domain'
+                return 'unexpected domain: ' .. domain
             end
         end
         "#).expect("Failed to load script");
@@ -43,7 +38,20 @@ mod tests {
         function run()
             domain = psl_domain_from_dns_name("example.com")
             if domain ~= "example.com" then 
-                return 'unexpected domain'
+                return 'unexpected domain: ' .. domain
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_psl_lookup_tld() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            domain = psl_domain_from_dns_name("asdfinvalid")
+            if domain ~= "asdfinvalid" then
+                return 'unexpected domain: ' .. domain
             end
         end
         "#).expect("Failed to load script");
@@ -51,14 +59,12 @@ mod tests {
     }
 
     #[test]
-    fn verify_psl_lookup_invalid() {
+    fn verify_psl_lookup_fastly() {
         let script = Script::load_unchecked(r#"
         function run()
-            domain = psl_domain_from_dns_name("asdf")
-            if last_err() then
-                clear_err()
-            else
-                return "invalid value didn't cause error"
+            domain = psl_domain_from_dns_name("a.prod.fastly.net")
+            if domain ~= "a.prod.fastly.net" then
+                return 'unexpected domain: ' .. domain
             end
         end
         "#).expect("Failed to load script");
diff --git a/src/web.rs b/src/web.rs
index 872f7cef..cf12958a 100644
--- a/src/web.rs
+++ b/src/web.rs
@@ -1,4 +1,4 @@
-pub use chrootable_https::{Client, HttpClient, Resolver};
+pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
 use crate::blobs::Blob;
 use std::collections::{HashMap, HashSet};
@@ -130,7 +130,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &State) -> Result<LuaMap> {
+    pub fn send(&self, state: &State) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -215,6 +215,12 @@ impl HttpRequest {
             .with_timeout(self.timeout)
             .wait_for_response()?;
 
+        Ok(res)
+    }
+
+    pub fn send_lua(&self, state: &State) -> Result<LuaMap> {
+        let res = self.send(state)?;
+
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));


Test patch: