Repository: smicallef/spiderfoot
File: modules/sfp_stackoverflow.py (Python)

Code before:
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    # What events this module produces
    def producedEvents(self):
        return ["RAW_RIR_DATA",
                "EMAILADDR",
                "AFFILIATE_EMAILADDR",
                "USERNAME",
                "IP_ADDRESS",
                "IPV6_ADDRESS",
                "HUMAN_NAME"
                ]

    def query(self, qry, qryType):
        # The Stackoverflow excerpts endpoint will search the site for mentions of a keyword and returns a snippet of relevant results
        if qryType == "excerpts":
            try:
                res = self.sf.fetchUrl(
                    f"https://api.stackexchange.com/2.3/search/excerpts?order=desc&q={qry}&site=stackoverflow",
                    timeout=self.opts['_fetchtimeout'],
                    useragent="SpiderFoot"
                )
            except Exception as e:
                self.error(f"Error querying StackExchange API: {e}")
                self.errorState = True
                return None

        # Questions profile endpoint, used to return displayname
        if qryType == "questions":
            try:
                res = self.sf.fetchUrl(
                    f"https://api.stackexchange.com/2.3/questions/{qry}?order=desc&sort=activity&site=stackoverflow",
                    timeout=self.opts['_fetchtimeout'],
                    useragent="SpiderFoot"
                )
            except Exception as e:
                self.error(f"Error querying StackExchange API: {e}")
                self.errorState = True
                return None


Review comment (suggestion):
In my testing I'm seeing throttling errors returned, indicated by a return code of `502`. This should be handled, with an error log reported. I don't know what the throttling limits are depending on whether an API key is provided, but perhaps have a `sleep` after each call to avoid throttling.

Code after:
    # Tracking the error state of the module
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    # What events this module produces
    def producedEvents(self):
        return ["RAW_RIR_DATA",
                "EMAILADDR",
                "AFFILIATE_EMAILADDR",
                "USERNAME",
                "IP_ADDRESS",
                "IPV6_ADDRESS",
                "HUMAN_NAME"
                ]

    def query(self, qry, qryType):
        # The Stackoverflow excerpts endpoint will search the site for mentions of a keyword and returns a snippet of relevant results
        if qryType == "excerpts":
            try:
                res = self.sf.fetchUrl(
                    f"https://api.stackexchange.com/2.3/search/excerpts?order=desc&q={qry}&site=stackoverflow",
                    timeout=self.opts['_fetchtimeout'],
                    useragent="SpiderFoot"
                )
                time.sleep(1)
            except Exception as e:
                self.error(f"Error querying StackExchange API: {e}")
                self.errorState = True
                return None

        # Questions profile endpoint, used to return displayname
        if qryType == "questions":
            try:
                res = self.sf.fetchUrl(
                    f"https://api.stackexchange.com/2.3/questions/{qry}?order=desc&sort=activity&site=stackoverflow",
                    timeout=self.opts['_fetchtimeout'],
                    useragent="SpiderFoot"
                )
                time.sleep(1)
            except Exception as e:
                self.error(f"Error querying StackExchange API: {e}")