Repository: smicallef/spiderfoot
File: modules/sfp_haveibeenpwned.py (Python)

Code before:
        pasteData = self.queryPaste(eventData)
        if pasteData is None:
            return None

        sites = {
            "Pastebin": "https://pastebin.com/",
            "Pastie": "http://pastie.org/",
            "Slexy": "https://slexy.org/",
            "Ghostbin": "https://ghostbin.com/",
            "QuickLeak": "http://www.quickleak.net/",
            "JustPaste": "https://justpaste.it/",
        }

        for n in pasteData:
            try:
                source = n.get("Source")
                site = source
                if source in sites.keys():
                    site = sites[n.get("Source")]

                if not n.get("Source") is None:
                    e = SpiderFootEvent("LEAKSITE_URL", site, self.__name__, event)
                    self.notifyListeners(e)

                if not n.get("Title") is None:
                    e = SpiderFootEvent("LEAKSITE_CONTENT", n.get("Title"), self.__name__, event)
                    self.notifyListeners(e)

            except Exception as e:
                self.sf.debug(f"Unable to parse result from HaveIBeenPwned?: {e}")
                continue


# End of sfp_haveibeenpwned class


Review comment (suggestion):
Look at sfp_pastebin to see how `LEAKSITE_CONTENT` should be reported.

Code after:
        if pasteData is None:
            return None

        sites = {
            "Pastebin": "https://pastebin.com/",
            "Pastie": "http://pastie.org/p/",
            "Slexy": "https://slexy.org/view/",
            "Ghostbin": "https://ghostbin.com/paste/",
            "JustPaste": "https://justpaste.it/",
        }
        links = set()
        for n in pasteData:
            try:
                source = n.get("Source")
                site = source
                if source in sites.keys():
                    site = f"{sites[n.get('Source')]}{n.get('Id')}"
                    links.add(site)

            except Exception as e:
                self.sf.debug(f"Unable to parse result from HaveIBeenPwned?: {e}")
                continue

        for link in links:
            try:
                self.sf.debug("Found a link: " + link)

                if self.checkForStop():
                    return None

                res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])

                if res['content'] is None:
                    self.sf.debug(f"Ignoring {link} as no data returned")
                    continue

                # Sometimes pastes search results false positives
                if eventData.lower() not in str(res['content']).lower():
                    self.sf.debug("String not found in pastes content.")
                    continue

                if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) + r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
                    continue

                evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
                self.notifyListeners(evt1)

                evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
                self.notifyListeners(evt2)

            except Exception as e: