Repository: smicallef/spiderfoot
File: modules/sfp_sublist3r.py (Python)

Code before:
class sfp_sublist3r(SpiderFootPlugin):

    meta = {
        "name": "Sublist3r PassiveDNS",
        "summary": "Passive subdomain enumeration using Sublist3r's API",
        "useCases": ["Investigate", "Footprint", "Passive"],
        "categories": ["Passive DNS"],
        "dataSource": {
            "website": "https://api.sublist3r.com",
            "model": "FREE_NOAUTH_UNLIMITED",
            "description": "This is the API queried by the Sublist3r tool.",
        }
    }

    # Default options
    opts = {}

    # Option descriptions
    optdescs = {}

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.sf.debug("Setting up sfp_sublist3r")
        self.state = self.tempStorage()
        self.state.update({
            "events": [],
        })
        self.__dataSource__ = "DNS"

        self.opts.update(userOpts)

    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    def producedEvents(self):
        return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]

    def query(self, domain):
        url = f"https://api.sublist3r.com/search.php?domain={domain}"
        ret = []
        res = self.sf.fetchUrl(
            url,
            useragent=self.opts.get("_useragent", "Spiderfoot"),
            # mirror sublist3r's headers
            headers={
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.8",
                "Accept-Encoding": "gzip",
            }
        )
        try:

Review comment (suggestion):
I like the approach taken here but for now can you make this consistent with all other modules (using `self.results`)? We'll move to a better state management approach later on.

Code after:
class sfp_sublist3r(SpiderFootPlugin):

    meta = {
        "name": "Sublist3r PassiveDNS",
        "summary": "Passive subdomain enumeration using Sublist3r's API",
        "useCases": ["Investigate", "Footprint", "Passive"],
        "categories": ["Passive DNS"],
        "dataSource": {
            "website": "https://api.sublist3r.com",
            "model": "FREE_NOAUTH_UNLIMITED",
            "description": "This is the API queried by the Sublist3r tool.",
        }
    }

    # Default options
    opts = {}

    # Option descriptions
    optdescs = {}

    results = None

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.sf.debug("Setting up sfp_sublist3r")
        self.results = self.tempStorage()
        self.opts.update(userOpts)

    def watchedEvents(self):
        return ["DOMAIN_NAME", "INTERNET_NAME"]

    def producedEvents(self):
        return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]

    def query(self, domain):
        url = f"https://api.sublist3r.com/search.php?domain={domain}"
        ret = []
        res = self.sf.fetchUrl(
            url,
            useragent=self.opts.get("_useragent", "Spiderfoot"),
            # mirror sublist3r's headers
            headers={
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.8",
                "Accept-Encoding": "gzip",
            }
        )
        try:
            ret = [s.strip().lower() for s in json.loads(res["content"])]
        except json.decoder.JSONDecodeError as e:
            self.sf.error(f"Error decoding JSON response: {e}")