Repository: smicallef/spiderfoot
File: modules/sfp_talosintel.py (Python)

Code before:
            if target in blacklist:
                self.sf.debug(f"IP address {target} found in Talos Intelligence blacklist.")
                return True
        elif targetType == "netblock":
            netblock = IPNetwork(target)
            for ip in blacklist:
                if IPAddress(ip) in netblock:
                    self.sf.debug(f"IP address {ip} found within netblock/subnet {target} in Talos Intelligence blacklist.")
                    return True

        return False

    def retrieveBlacklist(self):
        blacklist = self.sf.cacheGet('talosintel', 24)

        if blacklist is not None:
            return self.parseBlacklist(blacklist)

        # https://talosintelligence.com/documents/ip-blacklist redirects to:
        # https://snort.org/downloads/ip-block-list
        res = self.sf.fetchUrl(
            "https://snort.org/downloads/ip-block-list",
            self.opts['_fetchtimeout'],
            useragent=self.opts['_useragent'],
        )

        if res['code'] != "200":
            self.sf.error(f"Unexpected HTTP response code {res['code']} from Talos Intelligence.")
            self.errorState = True
            return

        if res['content'] is None:
            self.sf.error("Received no content from Talos Intelligence")
            self.errorState = True
            return

        self.sf.cachePut("talosintel", res['content'])

        return self.parseBlacklist(res['content'])

    def parseBlacklist(self, blacklist):
        """Parse plaintext blacklist

        Args:
            blacklist (str): plaintext blacklist from Talos Intelligence

        Returns:
            list: list of blacklisted IP addresses
        """
        ips = list()


Review comment (suggestion):
`https://talosintelligence.com/documents/ip-blacklist` was giving me HTTP 503 errors. I'm not sure if it was intermittent or due to blocking the user agent or due to using HTTP/2.0. Using the direct URL works. This was also a (mostly) silent error, as there was no error checking for a non-200 response code.


Code after:
        if targetType == "ip":
            if target in blacklist:
                self.sf.debug(f"IP address {target} found in Talos Intelligence blacklist.")
                return True
        elif targetType == "netblock":
            netblock = IPNetwork(target)
            for ip in blacklist:
                if IPAddress(ip) in netblock:
                    self.sf.debug(f"IP address {ip} found within netblock/subnet {target} in Talos Intelligence blacklist.")
                    return True

        return False

    def retrieveBlacklist(self):
        blacklist = self.sf.cacheGet('talosintel', 24)

        if blacklist is not None:
            return self.parseBlacklist(blacklist)

        # https://talosintelligence.com/documents/ip-blacklist redirects to:
        # https://snort.org/downloads/ip-block-list
        res = self.sf.fetchUrl(
            "https://snort.org/downloads/ip-block-list",
            timeout=self.opts['_fetchtimeout'],
            useragent=self.opts['_useragent'],
        )

        if res['code'] != "200":
            self.sf.error(f"Unexpected HTTP response code {res['code']} from Talos Intelligence.")
            self.errorState = True
            return

        if res['content'] is None:
            self.sf.error("Received no content from Talos Intelligence")
            self.errorState = True
            return

        self.sf.cachePut("talosintel", res['content'])

        return self.parseBlacklist(res['content'])

    def parseBlacklist(self, blacklist):
        """Parse plaintext blacklist

        Args:
            blacklist (str): plaintext blacklist from Talos Intelligence

        Returns:
            list: list of blacklisted IP addresses
        """
        ips = list()