Task: URL and IP lookups through Greynoise and VirusTotal

n8n workflow JSON:
{"meta":{"instanceId":"8c8c5237b8e37b006a7adce87f4369350c58e41f3ca9de16196d3197f69eabcd","templateId":"1971"},"nodes":[{"id":"dbb98f7d-6737-4eaa-9a66-9779c042c575","name":"VirusTotal result","type":"n8n-nodes-base.httpRequest","position":[2430,1648],"parameters":{"url":"={{ $json.data.links.self }}","options":{},"authentication":"predefinedCredentialType","nodeCredentialType":"virusTotalApi"},"typeVersion":4.1},{"id":"fb71337b-ebd3-4331-9f18-ff953c6b068b","name":"DNS Lookup","type":"n8n-nodes-base.httpRequest","position":[1330,1028],"parameters":{"url":"=https://dns.google/resolve","options":{},"sendQuery":true,"queryParameters":{"parameters":[{"name":"name","value":"={{ $json.url.includes('://') ? $json.url.split('://')[1].split('/')[0] : $json.url }}"}]}},"typeVersion":4.1},{"id":"290c6e9c-31d1-4476-9beb-b72a795ecfbb","name":"Set IP From Lookup","type":"n8n-nodes-base.code","position":[1530,1028],"parameters":{"mode":"runOnceForEachItem","jsCode":"// Get the resolved IP address (last item in the Answer array)\nconst ip = $json.Answer.pop().data;\nconst item = {...$('Is IP?').item.json}\nitem.ip = ip\n\nreturn {json: item};"},"typeVersion":2},{"id":"2e25aa5e-479c-4e3b-b866-89f2bdbabbba","name":"Set IP","type":"n8n-nodes-base.set","position":[1390,828],"parameters":{"values":{"string":[{"name":"ip","value":"={{ $json.url }}"}]},"options":{}},"typeVersion":2},{"id":"69b89cd7-1456-4067-a9da-d81ef3f86097","name":"Merge VirusTotal & Greynoise results","type":"n8n-nodes-base.merge","position":[3610,948],"parameters":{"mode":"combine","options":{},"mergeByFields":{"values":[{"field1":"ip","field2":"ip"}]}},"typeVersion":2.1},{"id":"1011bb3b-3f75-40b8-a473-e07b70079b60","name":"Is IP?","type":"n8n-nodes-base.if","position":[1110,848],"parameters":{"conditions":{"string":[{"value1":"={{ $json.url }}","value2":"/^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/","operation":"regex"}]}},"typeVersion":1},{"id":"770b4056-1497-48ed-bcd7-ad6e7106cc7d","name":"Start VirusTotal Scan","type":"n8n-nodes-base.httpRequest","position":[1990,1648],"parameters":{"url":"https://www.virustotal.com/api/v3/urls","method":"POST","options":{},"sendBody":true,"contentType":"multipart-form-data","authentication":"predefinedCredentialType","bodyParameters":{"parameters":[{"name":"url","value":"={{ $json.url }}"}]},"nodeCredentialType":"virusTotalApi"},"typeVersion":4.1},{"id":"d5d31e4a-2f95-4151-af35-bb8129f2e5a3","name":"VirusTotal Summary","type":"n8n-nodes-base.set","position":[3230,1628],"parameters":{"values":{"string":[{"name":"virusTotalStats","value":"={{ $json.data.attributes.stats }}"},{"name":"blockList","value":"={{ $json.data.attributes.results.BlockList.result }}"},{"name":"openPhish","value":"={{ $json.data.attributes.results.OpenPhish.result }}"},{"name":"url","value":"={{ $('Merge').all()[$itemIndex].json.url }}"},{"name":"ip","value":"={{ $('Merge').all()[$itemIndex].json.ip }}"}]},"options":{"dotNotation":false},"keepOnlySet":true},"typeVersion":2},{"id":"467c795f-6f13-4d6d-a8cf-5cf9be2e7a77","name":"VirusTotal ready?","type":"n8n-nodes-base.if","position":[2790,1648],"parameters":{"conditions":{"string":[{"value1":"={{ $json.data.attributes.status }}","value2":"queued","operation":"notEqual"}]}},"typeVersion":1},{"id":"284728e4-dc74-4c37-890b-5305970960c0","name":"Wait 5s","type":"n8n-nodes-base.wait","position":[2230,1648],"webhookId":"18348e84-831d-4ea8-bb39-6ec847c72275","parameters":{"unit":"seconds","amount":5},"typeVersion":1},{"id":"76e1414a-d690-44df-a3b8-8dbb4a192720","name":"Webhook","type":"n8n-nodes-base.webhook","notes":"Example:\n\ncurl -X POST \"https://n8n.yourdomain.com/webhook-test/d5124bd8-aada-44da-8050-3070f303ad24\" \\\n                 -H \"Content-Type: application/json\" \\\n                 -d '{\"data\": [{\"url\": \"1.1.1.1\"}, {\"url\": \"88.204.59.2\"}, {\"url\": \"54.36.148.188\"}, {\"url\": \"facebook.com\"}], \"email\": \"user@domain.com\"}'","position":[450,1448],"webhookId":"d5124bd8-aada-44da-8050-3070f303ad24","parameters":{"path":"d5124bd8-aada-44da-8050-3070f303ad24","options":{},"httpMethod":"POST"},"notesInFlow":true,"typeVersion":1},{"id":"b3e188f3-0a39-4451-ab70-632282243f03","name":"Get List of URLs","type":"n8n-nodes-base.itemLists","position":[650,1448],"parameters":{"options":{},"fieldToSplitOut":"body.data"},"typeVersion":3},{"id":"360628b7-afc0-4444-a8c0-a85fae54b0e3","name":"Set Email","type":"n8n-nodes-base.set","position":[850,1448],"parameters":{"values":{"string":[{"name":"Email","value":"={{ $('Webhook').item.json.body.email }}"}]},"options":{}},"typeVersion":2},{"id":"6df9593b-5f9f-4b50-bddb-97dcb2017d6e","name":"Merge Greynoise results","type":"n8n-nodes-base.merge","position":[2370,728],"parameters":{"mode":"combine","options":{},"combinationMode":"mergeByPosition"},"typeVersion":2.1},{"id":"1957a675-7a5a-4ccd-b334-f2c4f9749f58","name":"Send Report Slack","type":"n8n-nodes-base.slack","position":[3850,1168],"parameters":{"text":"=Successfully scanned {{ $json.url }} {{$json.ip !== $json.url ? `(${$json.ip})`: '' }}\n\n\nVirusTotal Report ({{ $json.virusTotalStats.harmless + $json.virusTotalStats.malicious + $json.virusTotalStats.suspicious + $json.virusTotalStats.undetected}} scans)\n\n{{$json.virusTotalStats.harmless}} Harmless\n{{$json.virusTotalStats.malicious}} Malicious\n{{$json.virusTotalStats.suspicious}} Suspicious\n{{$json.virusTotalStats.undetected}} Undetected\n{{$json.virusTotalStats.timeout}} Timed out\n\nBlockList: {{ $json.blockList }}\nOpenPhish: {{ $json.openPhish }}\n\nSummary: {{ $json.virusTotalStats.suspicious + $json.virusTotalStats.malicious === 0 ? \"✅ Harmless\": \"🚨 Malicous\" }}\n\n\n\nGreynoise Report\n\nTrust Level: {{ $json.trust_level ?? \"Not trusted\"}}\nClassification: {{ $json.classification }}\n\nLocation: {{ $json.location || 'n/a' }}\nCategory: {{ $json.category }}\nTags: {{$json.tags.join(', ') || 'None'}}","select":"channel","channelId":{"__rl":true,"mode":"name","value":"#notifications"},"otherOptions":{}},"typeVersion":2.1},{"id":"4d64351f-0233-4859-afd2-fc31e3fc37cd","name":"Send Report Email","type":"n8n-nodes-base.gmail","position":[3850,948],"parameters":{"sendTo":"={{ $('Merge').first().json.Email }}","message":"=Successfully scanned {{ $json.url }} {{$json.ip !== $json.url ? `(${$json.ip})`: '' }}<br /><br /><br />\n\n\n<h3>VirusTotal Report ({{ $json.virusTotalStats.harmless + $json.virusTotalStats.malicious + $json.virusTotalStats.suspicious + $json.virusTotalStats.undetected}} scans)</h3><br /><br />\n\n{{$json.virusTotalStats.harmless}} Harmless<br />\n{{$json.virusTotalStats.malicious}} Malicious<br />\n{{$json.virusTotalStats.suspicious}} Suspicious<br />\n{{$json.virusTotalStats.undetected}} Undetected<br />\n{{$json.virusTotalStats.timeout}} Timed out<br /><br />\n\nBlockList: {{ $json.blockList }}<br />\nOpenPhish: {{ $json.openPhish }}<br /><br />\n\n<b>Summary: {{ $json.virusTotalStats.suspicious + $json.virusTotalStats.malicious === 0 ? \"✅ Harmless\": \"🚨 Malicous\" }}</b><br /><br /><br />\n\n\n\n<h3>Greynoise Report</h3><br /><br />\n\nTrust Level: {{ $json.trust_level ?? \"Not trusted\"}}<br />\nClassification: {{ $json.classification }}<br /><br />\n\nLocation: {{ $json.location || 'n/a' }}<br />\nCategory: {{ $json.category }}<br />\nTags: {{$json.tags.join(', ') || 'None'}}<br /><br /><br /><br />","options":{},"subject":"={{ $json.url }} Scan Report"},"typeVersion":2},{"id":"e4305eb1-8e57-49d0-97b7-391200bd0042","name":"Greynoise Summary","type":"n8n-nodes-base.set","position":[2650,728],"parameters":{"values":{"string":[{"name":"ip","value":"={{ $json.ip }}"},{"name":"classification","value":"={{ $json.classification || 'safe' }}"},{"name":"location","value":"={{ $json.metadata?.region ? `${$json.metadata?.region} ${$json.metadata?.country}` : '' }}"},{"name":"tags","value":"={{ $json.tags ?? [] }}"},{"name":"category","value":"={{ $json.category || 'n/a' }}"},{"name":"trustLevel","value":"={{ $json.trust_level }}"}]},"options":{"dotNotation":false},"keepOnlySet":true},"typeVersion":2},{"id":"c149b1f3-e447-4194-a94e-7d8e0bf38241","name":"Merge","type":"n8n-nodes-base.merge","position":[1750,848],"parameters":{},"typeVersion":2.1},{"id":"88c30a1d-c232-4da5-87c3-4d67234b6a29","name":"Combine looped items","type":"n8n-nodes-base.code","position":[3010,1628],"parameters":{"jsCode":"let results = [],\n  i = 0;\n\ndo {\n  try {\n    results = results.concat($(\"VirusTotal result\").all(0, i)\n                     .filter(node => node.json.data.attributes.status === 'completed')\n  );\n  } catch (error) {\n    return results;\n  }\n  i++;\n} while (true);"},"typeVersion":2},{"id":"839170f5-7c97-40fd-aeaa-ad57262a586e","name":"Filter","type":"n8n-nodes-base.filter","position":[2610,1648],"parameters":{"conditions":{"string":[{"value1":"={{ $json.data.attributes.status }}","value2":"completed","operation":"notEqual"}]}},"typeVersion":1,"alwaysOutputData":true},{"id":"d68db329-4628-44a8-8f97-b06cbf18e238","name":"Sticky Note","type":"n8n-nodes-base.stickyNote","position":[380,240],"parameters":{"width":651.1325602067182,"height":703.911103299255,"content":"## Form Input Overview\n\n- **Purpose**: \n  - Instead of forcing other departments to use a full threat platform, simplify the interaction with our Threat Intel workflow which allows other departments to submit items via URL-accessible forms.\n\n- **Form Access URLs**:\n  - **Execute Mode**: `https://n8n.domain.com/webhook/test/url-scan-form` - Use this to execute the workflow interactively within the n8n canvas. Hit the 'Execute Workflow' button to see real-time execution results.\n  - **Silent Mode**: `https://n8n.domain.com/webhook/url-scan-form` - Use this for background execution without canvas updates. Results will be logged silently and can be reviewed in the 'Executions' tab.\n\n## Details and Best Practices\nWhen using the form, ensure that all inputs match the required format, like valid URLs for scans, to prevent any workflow interruptions. Keep in mind these forms are not performing input sanitation so incorrectly entered values will trigger an error workflow. Should there be any issues upon form submission, such as an absence of a confirmation message, or if the workflow fails, you can find detailed error information in the 'Executions' tab. "},"typeVersion":1},{"id":"f9081f7d-35ab-489c-87bb-c2deba7515f9","name":"Sticky Note1","type":"n8n-nodes-base.stickyNote","position":[370,968],"parameters":{"width":653.8285114211884,"height":663.9676956356055,"content":"## API Integration\nWant to submit URLs and IPs automatically? Utilize the JSON structure below to upload multiple indicators simultaneously. The workflow leverages 'Item list' to parse the 'data' field, while 'Set Email' node appends the provided email to each URL.\n\n```json\n{\n    \"email\": \"johndoe@example.com\",\n    \"data\": [\n        {\n            \"url\": \"aztechsol.com\"\n        },\n        {\n            \"ip\": \"8.8.8.8\"\n        }\n    ]\n}\n```\n\n## Details and Best Practices\n- Webhook Usage: Send data with a POST request, e.g., using curl.\n- Validation & Errors: Ensure URLs are correctly formatted. Check the 'Executions' tab for any submission errors. Keep in mind that there is only basic error handling in this workflow."},"typeVersion":1},{"id":"aaaac5aa-f0a0-4452-99b4-d78d55a80564","name":"Sticky Note2","type":"n8n-nodes-base.stickyNote","position":[1070,266.9400418986032],"parameters":{"width":827.7173647545219,"height":936.2889303743061,"content":"![VirusTotal](https://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/Google_2015_logo.svg/320px-Google_2015_logo.svg.png)\n## Data Standardization & Google DNS Integration\n-   Purpose:\n\n    -   Standardize the diverse input sources---either from form submissions or API calls---by streamlining the input through a uniform processing pipeline. This ensures that whether the data entered is an IP address or a domain name, it can be consistently managed and transformed for threat intelligence tasks.\n    -   Extract IP from URL by passing it to Google DNS and attaching it to the URL.\n\n## Details and Best Practices\nTo guarantee the efficacy of the workflow, adhere to the prescribed input formats. For IP addresses, ensure they conform to IPv4 or IPv6 standards; for domains, verify that they are properly structured URLs. The system assumes clean inputs, as there are no built-in sanitation mechanisms---erroneous inputs may result in processing errors.\n\nIn case of an unsuccessful DNS lookup or other discrepancies, consult the 'Executions' tab for comprehensive error logs and apply the necessary corrections. To mitigate workflow disruptions, establish a set of error-handling protocols to manage and rectify such incidents.\n\nBe mindful that while the workflow is designed to automatically discern between IP addresses and domain names, it is imperative that the data entered is accurate to prevent any fallbacks or unnecessary processing overhead."},"typeVersion":1},{"id":"32e80421-b608-4d89-b6fe-a95ab5b9e3bd","name":"Sticky Note3","type":"n8n-nodes-base.stickyNote","position":[1950,68.30371042491026],"parameters":{"width":1485.5734904392764,"height":987.7653566551932,"content":"![VirusTotal](https://i.imgur.com/fd6Ng5R.png)\n## Greynoise Integration\n\n-   Purpose:\n    -   The aim is to tap into Greynoise's robust API to enrich and contextualize IP-related information within the workflow. By querying Greynoise's specialized noise and RIOT databases, the workflow can quickly ascertain the nature of the IP activity and determine its relevance and potential threat level to an organization.\n    -   Classify and assess IP addresses by consulting with Greynoise databases, providing an additional layer of security intelligence.\n\n## Details and Best Practices\nTo ensure reliable results from the Greynoise integration, it's important to use well-formatted IP addresses. Confirm that IPs meet standard internet protocols for either IPv4 or IPv6. The workflow assumes that inputs are pre-sanitized, so any deviation may lead to errors or inaccurate assessments.\n\nIf the Greynoise lookup does not yield results or encounters errors, investigate the issue using the 'Executions' tab to view detailed error logs. Proactively develop error-handling strategies to effectively manage and recover from these errors.\n\nThe workflow is pre-configured to discern and process IP information accurately; however, it relies heavily on the integrity of the input. Incorrectly entered IPs can cause incorrect lookups and potentially miss significant threat data, thereby undermining the security posture.\n\n**Please note that this workflow segment is designed for the enterprise edition of Greynoise's API. Users must have a valid API key with enterprise access which should be configured in the HTTP request nodes that perform the API calls.**"},"typeVersion":1},{"id":"adda919b-f65f-4fe0-9f66-11a5a9b65674","name":"Sticky Note4","type":"n8n-nodes-base.stickyNote","position":[1952,1088],"parameters":{"width":1483.145187368557,"height":774.1502041707245,"content":"![VirusTotal](https://upload.wikimedia.org/wikipedia/commons/thumb/b/b7/VirusTotal_logo.svg/320px-VirusTotal_logo.svg.png)\n## VirusTotal Integration\n-   Purpose:\n    -   This workflow component is specifically crafted to harness the VirusTotal API's capabilities, allowing URLs to be submitted for thorough scanning. The goal is to seamlessly integrate the scanning process into the workflow, handling the asynchronous nature of VirusTotal scans by effectively managing state checks and result compilation.\n    -   Implement URL scanning by submitting requests to the VirusTotal API and accurately aggregating the scan results for analysis.\n\n## Details and Best Practices\nFor successful VirusTotal integration, it's crucial to submit URLs following standard web formats. The workflow is configured to expect correct URL inputs; deviations can disrupt the scanning process.\n\nUpon submission, if the VirusTotal scan results are pending or if errors are encountered, these can be tracked and examined under the 'Executions' tab. Develop a proactive strategy for handling such cases, including error logging, maximum retry limitations, or implementing a timeout mechanism.\n\nThe configuration of the workflow takes into account the need to prevent rapid, repetitive status checks, which can strain the VirusTotal API. As a result, it employs a looping system for status re-evaluation, which should be managed with precision to avoid unnecessary delays or excessive polling.\n\nNote that this integration is tailored for workflows that involve the VirusTotal API. While it works with the free VirusTotal license, too many requests may cause errors due to rate limiting. The Public API is limited to 500 requests per day and a rate of 4 requests per minute. It requires valid credentials set up in the HTTP request nodes to authenticate API calls successfully. Users should configure their API keys for access, and handle any API error responses, like HTTP 4xx or 5xx codes, with a robust error-logging and retry mechanism to ensure reliability and effectiveness of the scan process."},"typeVersion":1},{"id":"cdaec18e-e9f1-4567-89e6-f5474bff42c4","name":"Sticky Note5","type":"n8n-nodes-base.stickyNote","position":[3470,247],"parameters":{"width":898.9279259630971,"height":1146.6423884335761,"content":"![VirusTotal](https://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/Slack_Logo_Pre_2019.svg/320px-Slack_Logo_Pre_2019.svg.png)\n![VirusTotal](https://i.imgur.com/f6f6my0.png)\n## Reporting Integration\n-   Purpose:\n    -   This component of the workflow is designed to amalgamate and communicate the insights from threat intelligence analysis to the team effectively. By integrating data from VirusTotal and Greynoise, it generates comprehensive reports that are automatically shared via Slack and email, fostering situational awareness and facilitating prompt action.\n    -   Compile and disseminate threat intelligence reports that highlight the significance and implications of the analyzed IP or domain data, ensuring that the team remains informed and ready to act.\n\n## Details and Best Practices\nThe heart of this workflow lies in the synthesis of threat intelligence gathered from both Greynoise and VirusTotal. By merging these data points, the logic creates a thorough examination of the URLs/IPs under scrutiny.\n\nHere's an expanded view of best practices to adhere to in this critical stage of the workflow:\n\nThe merging process must be precise, using the 'ip' fields as the common key to unify data from the two distinct sources. This unified view is crucial for accurate analysis and reporting.\n\nIt’s advisable to extend the merging capabilities to include additional data fields that may enhance the intelligence report. This could mean incorporating timestamps, geolocation data, or even threat levels.\n\nWhen integrating new messaging or reporting nodes, leverage the provided JSON structure to maintain consistency. To replicate the logic in another node, simply copy the JSON snippet from the expression editor and paste it into the configuration of the new node."},"typeVersion":1},{"id":"577d4b74-9155-440f-a752-6654f8e54669","name":"GreyNoise RIOT lookup","type":"n8n-nodes-base.httpRequest","position":[2070,708],"parameters":{"url":"=https://api.greynoise.io/v2/riot/{{ $json.ip }}","options":{},"authentication":"genericCredentialType","genericAuthType":"httpHeaderAuth"},"typeVersion":4.1},{"id":"9a41df79-3d81-4b2e-b21c-7f31985d8d1e","name":"GreyNoise IP Check","type":"n8n-nodes-base.httpRequest","position":[2070,888],"parameters":{"url":"=https://api.greynoise.io/v2/noise/context/{{ $json.ip }}","options":{},"authentication":"genericCredentialType","genericAuthType":"httpHeaderAuth"},"typeVersion":4.1},{"id":"006eb997-5851-41bd-9d5c-9f44d3b7ec08","name":"Form Trigger","type":"n8n-nodes-base.formTrigger","position":[640,800],"webhookId":"087145f7-3c00-4a1a-8e04-181b536606e7","parameters":{"path":"url-scan-form","options":{},"formTitle":"Scan URL or IP and get a report","formFields":{"values":[{"fieldLabel":"url","requiredField":true},{"fieldLabel":"Email","requiredField":true}]},"formDescription":"Get a report from Virus Total and Greynoise on an IP address of URL"},"typeVersion":2}],"pinData":{},"connections":{"Merge":{"main":[[{"node":"Start VirusTotal Scan","type":"main","index":0},{"node":"GreyNoise IP Check","type":"main","index":0},{"node":"GreyNoise RIOT lookup","type":"main","index":0}]]},"Filter":{"main":[[{"node":"VirusTotal ready?","type":"main","index":0}]]},"Is IP?":{"main":[[{"node":"Set IP","type":"main","index":0}],[{"node":"DNS Lookup","type":"main","index":0}]]},"Set IP":{"main":[[{"node":"Merge","type":"main","index":0}]]},"Wait 5s":{"main":[[{"node":"VirusTotal result","type":"main","index":0}]]},"Webhook":{"main":[[{"node":"Get List of URLs","type":"main","index":0}]]},"Set Email":{"main":[[{"node":"Is IP?","type":"main","index":0}]]},"DNS Lookup":{"main":[[{"node":"Set IP From Lookup","type":"main","index":0}]]},"Form Trigger":{"main":[[{"node":"Is IP?","type":"main","index":0}]]},"Get List of URLs":{"main":[[{"node":"Set Email","type":"main","index":0}]]},"Greynoise Summary":{"main":[[{"node":"Merge VirusTotal & Greynoise results","type":"main","index":0}]]},"VirusTotal ready?":{"main":[[{"node":"Combine looped items","type":"main","index":0}],[{"node":"Wait 5s","type":"main","index":0}]]},"VirusTotal result":{"main":[[{"node":"Filter","type":"main","index":0}]]},"GreyNoise IP Check":{"main":[[{"node":"Merge Greynoise results","type":"main","index":1}]]},"Set IP From Lookup":{"main":[[{"node":"Merge","type":"main","index":1}]]},"VirusTotal Summary":{"main":[[{"node":"Merge VirusTotal & Greynoise results","type":"main","index":1}]]},"Combine looped items":{"main":[[{"node":"VirusTotal Summary","type":"main","index":0}]]},"GreyNoise RIOT lookup":{"main":[[{"node":"Merge Greynoise results","type":"main","index":0}]]},"Start VirusTotal Scan":{"main":[[{"node":"Wait 5s","type":"main","index":0}]]},"Merge Greynoise results":{"main":[[{"node":"Greynoise Summary","type":"main","index":0}]]},"Merge VirusTotal & Greynoise results":{"main":[[{"node":"Send Report Slack","type":"main","index":0},{"node":"Send Report Email","type":"main","index":0}]]}}}