Repository: smicallef/spiderfoot
File: modules/sfp_stackoverflow.py (Python)

Code before:
                    ips.add(match)
            return list(ips)
        else:
            return None

    def extractIP6s(self, text):
        ips = set()

        matches = re.findall(r'(?:^|(?<=\s))(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?=\s|$)', text)

        if matches:
            for match in matches:
                if self.sf.validIP6(match) and not(self.sf.isValidLocalOrLoopbackIP(match)):
                    ips.add(match)
            return list(ips)
        else:
            return None

    def handleEvent(self, event):
        eventData = event.data

        if self.errorState:
            return

        if eventData in self.results:
            self.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        query_results = self.query(eventData, "excerpts")
        items = query_results.get('items')
        allEmails = []
        allUsernames = []
        allIP4s = []
        allIP6s = []

        if items is None:
            return
        # Iterate through all results from query, creating raw_rir_data events and extracting emails
        for item in items:
            if self.checkForStop():
                return

            body = item["body"]
            excerpt = item["excerpt"]
            question = item["question_id"]
            text = body + excerpt

            # create raw_rir_data event
            e = SpiderFootEvent('RAW_RIR_DATA',

Review comment (suggestion):
For all events reported, there should be some checking to avoid reporting the same event twice. For example, testing with `tesla.com` I'm seeing the same email address being reported multiple times.

Code after:
        ips = set()

        matches = re.findall(r'^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$', text)

        if matches:
            for match in matches:
                if self.sf.validIP(match) and not(self.sf.isValidLocalOrLoopbackIP(match)):
                    ips.add(match)
            return list(ips)
        else:
            return None

    def extractIP6s(self, text):
        ips = set()

        matches = re.findall(r'(?:^|(?<=\s))(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?=\s|$)', text)

        if matches:
            for match in matches:
                if self.sf.validIP6(match) and not(self.sf.isValidLocalOrLoopbackIP(match)):
                    ips.add(match)
            return list(ips)
        else:
            return None

    def handleEvent(self, event):
        eventData = event.data

        if self.errorState:
            return

        if eventData in self.results:
            self.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        query_results = self.query(eventData, "excerpts")
        items = query_results.get('items')
        allEmails = []
        allUsernames = []
        allIP4s = []
        allIP6s = []

        if items is None:
            return
        # Iterate through all results from query, creating raw_rir_data events and extracting emails
        for item in items:
            if self.checkForStop():
                return