Repository: smicallef/spiderfoot
File: modules/sfp_dehashed.py (Python)

Code before:
            'RAW_RIR_DATA'
        ]

    # Query Dehashed
    def query(self, event, per_page, start):
        if event.eventType == "EMAILADDR":
            queryString = f"https://api.dehashed.com/search?query=email:\"{event.data}\"&page={start}&size={self.opts['per_page']}"
        if event.eventType == "DOMAIN_NAME":
            queryString = f"https://api.dehashed.com/search?query=email:\"@{event.data}\"&page={start}&size={self.opts['per_page']}"

        token = (base64.b64encode(self.opts['api_key_username'].encode('utf8') + ":".encode('utf-8') + self.opts['api_key'].encode('utf-8'))).decode('utf-8')
        headers = {
            'Accept': 'application/json',
            'Authorization': f'Basic {token}'
        }

        res = self.sf.fetchUrl(queryString,
                               headers=headers,
                               timeout=15,
                               useragent=self.opts['_useragent'],
                               verify=False)

        time.sleep(self.opts['pause'])

        if res['code'] == "400":
            self.sf.error("Too many requests were performed in a small amount of time. Please wait a bit before querying the API.")
            return None

        if res['code'] == "401":
            self.sf.error("Invalid API credentials")
            self.errorState = True
            return None

        if res['code'] != "200":
            self.sf.error("Unable to fetch data from Dehashed.")
            self.errorState = True
            return None

        if res['content'] is None:
            self.sf.debug('No response from Dehashed')
            return None

        try:
            return json.loads(res['content'])
        except Exception as e:
            self.sf.debug(f"Error processing JSON response: {e}")
            return None

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType

Review comment (suggestion):
Not sure how useful such an error would be since it would occur mid-scan, and the SpiderFoot user wouldn't be able to do anything about it. Better if it backed off and tried again after a few seconds.

Code after:
            'RAW_RIR_DATA'
        ]

    # Query Dehashed
    def query(self, event, per_page, start):
        if event.eventType == "EMAILADDR":
            queryString = f"https://api.dehashed.com/search?query=email:\"{event.data}\"&page={start}&size={self.opts['per_page']}"
        if event.eventType == "DOMAIN_NAME":
            queryString = f"https://api.dehashed.com/search?query=email:\"@{event.data}\"&page={start}&size={self.opts['per_page']}"

        token = (base64.b64encode(self.opts['api_key_username'].encode('utf8') + ":".encode('utf-8') + self.opts['api_key'].encode('utf-8'))).decode('utf-8')
        headers = {
            'Accept': 'application/json',
            'Authorization': f'Basic {token}'
        }

        res = self.sf.fetchUrl(queryString,
                               headers=headers,
                               timeout=15,
                               useragent=self.opts['_useragent'],
                               verify=True)

        time.sleep(self.opts['pause'])

        if res['code'] == "400":
            self.sf.error("Too many requests were performed in a small amount of time. Please wait a bit before querying the API.")
            time.sleep(5)
            res = self.sf.fetchUrl(queryString, headers=headers, timeout=15, useragent=self.opts['_useragent'], verify=True)

        if res['code'] == "401":
            self.sf.error("Invalid API credentials")
            self.errorState = True
            return None

        if res['code'] != "200":
            self.sf.error("Unable to fetch data from Dehashed.")
            self.errorState = True
            return None

        if res['content'] is None:
            self.sf.debug('No response from Dehashed')
            return None

        try:
            return json.loads(res['content'])
        except Exception as e:
            self.sf.debug(f"Error processing JSON response: {e}")
            return None

    # Handle events sent to this module
    def handleEvent(self, event):