Repository: smicallef/spiderfoot
File: modules/sfp_censys.py (Python)

Code before:

        if eventData in self.results:
            self.sf.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        qrylist = list()
        if eventName.startswith("NETBLOCK_"):
            for ipaddr in IPNetwork(eventData):
                qrylist.append(str(ipaddr))
                self.results[str(ipaddr)] = True
        else:
            qrylist.append(eventData)

        for addr in qrylist:
            if self.checkForStop():
                return

            if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]:
                rec = self.queryIp(addr)

            try:
                rec = rec["result"]
            except Exception:
                rec = None

            if rec is None:
                continue

            self.sf.debug("Found results in Censys.io")

            # For netblocks, we need to create the IP address event so that
            # the threat intel event is more meaningful.
            if eventName.startswith('NETBLOCK_'):
                pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event)
                self.notifyListeners(pevent)
            else:
                pevent = event

            e = SpiderFootEvent("RAW_RIR_DATA", json.dumps(rec), self.__name__, pevent)
            self.notifyListeners(e)

            try:
                # Date format: 2021-09-22T16:46:47.623Z
                created_dt = datetime.strptime(rec.get('last_updated_at', "1970-01-01T00:00:00.000Z"), '%Y-%m-%dT%H:%M:%S.%fZ')
                created_ts = int(time.mktime(created_dt.timetuple()))
                age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])

                if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
                    self.sf.debug("Record found but too old, skipping.")

Review comment (suggestion):
This clobbers the existing `rec` variable.

Also `rec` doesn't make sense as a variable name.

Also, you don't need a try/except if you use `.get('result').

You could do:
```suggestion
            if not rec:
                continue

            results = rec.get('results')
            if not results:
                continue
```

This would also require changing every instance of `rec` to `results` in this module.


Code after:

        if eventData in self.results:
            self.sf.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        qrylist = list()
        if eventName.startswith("NETBLOCK_"):
            for ipaddr in IPNetwork(eventData):
                qrylist.append(str(ipaddr))
                self.results[str(ipaddr)] = True
        else:
            qrylist.append(eventData)

        for addr in qrylist:
            if self.checkForStop():
                return

            if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]:
                rec = self.queryIp(addr)

            rec = rec.get("result")

            if rec is None:
                continue

            self.sf.debug("Found results in Censys.io")

            # For netblocks, we need to create the IP address event so that
            # the threat intel event is more meaningful.
            if eventName.startswith('NETBLOCK_'):
                pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event)
                self.notifyListeners(pevent)
            else:
                pevent = event

            e = SpiderFootEvent("RAW_RIR_DATA", json.dumps(rec), self.__name__, pevent)
            self.notifyListeners(e)

            try:
                # Date format: 2021-09-22T16:46:47.623Z
                created_dt = datetime.strptime(rec.get('last_updated_at', "1970-01-01T00:00:00.000Z"), '%Y-%m-%dT%H:%M:%S.%fZ')
                created_ts = int(time.mktime(created_dt.timetuple()))
                age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])

                if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
                    self.sf.debug("Record found but too old, skipping.")
                    continue

                if 'location' in rec: