Language: Python
Label: safe
CWE: CWE-918 \u2014 Server-Side Request Forgery (SSRF)
The web server receives a URL or similar request from an upstream component and retrieves the contents of this URL, but it does not sufficiently ensure that the request is being sent to the expected destination.

Code:
    def select(self, query: ast.Select) -> pd.DataFrame:

        conditions = extract_comparison_conditions(query.where)
        urls = []
        for op, arg1, arg2 in conditions:

            if op == 'or':
                raise NotImplementedError(f'OR is not supported')

            if arg1 == 'url':
                url = arg2

                if op == '=':
                    urls = [str(url)]
                elif op == 'in':
                    if type(url) == str:
                        urls = [str(url)]
                    else:
                        urls = url
                else:
                    raise NotImplementedError(
                        f'url can be url = "someurl", you can also crawl multiple sites, as follows:'
                        f' url IN ("url1", "url2", ..)'
                    )

            else:
                pass

        if len(urls) == 0:
            raise NotImplementedError(
                f'You must specify what url you want to crawl, for example: SELECT * FROM crawl WHERE url IN ("someurl", ..)')

        if query.limit is None:
            raise NotImplementedError(f'You must specify a LIMIT which defines the number of pages to crawl')
        limit = query.limit.value

        if limit < 0:
            limit = 0

        config = Config()
        is_cloud = config.get("cloud", False)
        if is_cloud:
            urls = [
                url
                for url in urls
                if not is_private_url(url)
            ]

        result = get_all_websites(urls, limit, html=False)
        if len(result) > limit:
            result = result[:limit]
        # filter targets
        result = project_dataframe(result, query.targets, self.get_columns())
        return result