update lookup_iocs pipeline

lookup_iocs/README.md

@@ -2,7 +2,82 @@
 
 Looks up submitted IOCs (Indicators of Compromise) in different applications and services, including AbuseIPDB, Hunter.io, VirusTotal, etc., and then returns selected results.
 
+## Getting Started
+
+### Credentials
+
+By default, the following environment variables will be used for authentication:
+
+- `ABUSEIPDB_API_KEY`
+- `VTCLI_APIKEY`
+- `URLSCAN_API_KEY`
+- `IP2LOCATION_API_KEY`
+
+You can also create `credential` resources in configuration files:
+
+```sh
+vi ~/.flowpipe/config/abuseipdb.fpc
+```
+
+```hcl
+credential "abuseipdb" "abuseipdb_api_key" {
+  api_key = "bfc6f1c42dsfsdfdxxxx26977977b2xxxsfsdda98f313c3d389126de0d"
+}
+```
+
+```sh
+vi ~/.flowpipe/config/virustotal.fpc
+```
+
+```hcl
+credential "virustotal" "my_virustotal" {
+  api_key = "AG.U7..."
+}
+```
+
+```sh
+vi ~/.flowpipe/config/urlscan.fpc
+```
+
+```hcl
+credential "urlscan" "my_urlscan" {
+  api_key = "AKIA...2"
+}
+```
+
+```sh
+vi ~/.flowpipe/config/ip2location.fpc
+```
+
+```hcl
+credential "ip2location" "my_ip2location" {
+  token = "00B630jSCGU4jV4o5Yh4KQMAdqizwE2OgVcS7N9UHb"
+}
+```
+
+For more information on credentials in Flowpipe, please see [Managing Credentials](https://flowpipe.io/docs/run/credentials).
+
 ## Usage
 
-- Add your APIVoid, IP2Location, Urlscan, Virustotal, AbuseIPDB, Hunter, Kickbox and HybridAnalysis API key to `flowpipe.pvars`
-- Run the pipeline and specify `domain`, e.g., `flowpipe pipeline run lookup_iocs --pipeline-arg 'iocs=[{"id" : "1","type" : "ip","value" : "192.168.1.10"},{"id" : "2","type" : "domain","value" : "malicious-domain.com"}]'`
+Run the pipeline to lookup IOCs in different tools
+
+```sh
+flowpipe pipeline run lookup_iocs --pipeline-arg 'iocs=[{"id" : "1","type" : "ip","value" : "192.168.1.10"},{"id" : "2","type" : "domain","value" : "malicious-domain.com"}]'`
+```
+
+### Configuration
+
+To avoid entering variable values when running the pipeline or starting the server, you can set variable values:
+
+```sh
+cp flowpipe.fpvars.example flowpipe.fpvars
+vi flowpipe.fpvars
+```
+
+```hcl
+# Required
+apivoid_api_key="Your_APIVoid_API_Key"
+hunter_api_key="Your_Hunter_API_Key"
+kickbox_api_key="Your_Kickbox_API_Key"
+hybrid_analysis_api_key="Your_Hybrid_Analysis_API_Key"
+```

lookup_iocs/flowpipe.fpvars.example

@@ -1,3 +1,4 @@
+# Required
 apivoid_api_key="Your_APIVoid_API_Key"
 hunter_api_key="Your_Hunter_API_Key"
 kickbox_api_key="Your_Kickbox_API_Key"

lookup_iocs/lookup_domain.fp

@@ -2,12 +2,23 @@ pipeline "lookup_domain" {
   title       = "Lookup Domain in Different Tools"
   description = "A composite Flowpipe mod that lookup a domain in VirusTotal, Urlscan and other tools."
 
+  param "virustotal_cred" {
+    type        = string
+    description = "Name for VirusTotal credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
+  param "urlscanio_cred" {
+    type        = string
+    description = "Name for  URLScan.io credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
   param "domain" {
     type        = string
     description = "The domain to be scanned."
   }
 
-  # URLhaus
   step "http" "urlhaus_domain_lookup" {
     method = "post"
     url    = "https://urlhaus-api.abuse.ch/v1/host"
@@ -19,23 +30,22 @@ pipeline "lookup_domain" {
     request_body = "host=${param.domain}"
   }
 
-  # VirusTotal
   step "pipeline" "virustotal_domain_lookup" {
     pipeline = virustotal.pipeline.get_domain_report
     args = {
+      cred   = param.virustotal_cred
       domain = param.domain
     }
   }
 
-  # Urlscan.io
   step "pipeline" "urlscan_domain_lookup" {
     pipeline = urlscanio.pipeline.search_scan
     args = {
+      cred   = param.urlscanio_cred
       query = "domain:${param.domain}"
     }
   }
 
-  # Pulsedive
   step "http" "pulsedive_domain_lookup" {
     method = "post"
     url    = "https://pulsedive.com/api/explore.php?q=${param.domain}"

lookup_iocs/lookup_email.fp

@@ -19,7 +19,6 @@ pipeline "lookup_email" {
     description = "The email ID to be scanned."
   }
 
-  # Hunter
   step "http" "hunter_email_verify_status" {
     method = "get"
     url    = "https://api.hunter.io/v2/email-verifier?email=${param.email}&api_key=${param.hunter_api_key}"
@@ -29,7 +28,6 @@ pipeline "lookup_email" {
     }
   }
 
-  # Kickbox
   step "http" "kickbox_email_verify_status" {
     method = "get"
     url    = "https://api.kickbox.com/v2/verify?email=${param.email}&apikey=${param.kickbox_api_key}"

lookup_iocs/lookup_file_hash.fp

@@ -2,6 +2,18 @@ pipeline "lookup_file_hash" {
   title       = "Lookup File hash in Different Tools"
   description = "A composite Flowpipe mod that lookup a file hash in VirusTotal, Urlscan and other tools."
 
+  param "virustotal_cred" {
+    type        = string
+    description = "Name for VirusTotal credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
+  param "urlscanio_cred" {
+    type        = string
+    description = "Name for  URLScan.io credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
   param "hybrid_analysis_api_key" {
     type        = string
     default     = var.hybrid_analysis_api_key
@@ -13,23 +25,22 @@ pipeline "lookup_file_hash" {
     description = "The file hash to be scanned."
   }
 
-  # VirusTotal
   step "pipeline" "virustotal_file_hash_lookup" {
     pipeline = virustotal.pipeline.get_file_analysis
     args = {
+      cred      = param.virustotal_cred
       file_hash = param.file_hash
     }
   }
 
-  # Urlscan.io
   step "pipeline" "urlscan_file_hash_lookup" {
     pipeline = urlscanio.pipeline.search_scan
     args = {
+      cred  = param.urlscanio_cred
       query = "hash:${param.file_hash}"
     }
   }
 
-  # Hybrid Analysis
   step "http" "hybrid_analysis_file_hash_lookup" {
     method = "post"
     url    = "https://www.hybrid-analysis.com/api/v2/search/hash"

lookup_iocs/lookup_iocs.fp

@@ -2,6 +2,10 @@ pipeline "lookup_iocs" {
   title       = "Lookup IOCs in Different Tools"
   description = "A composite Flowpipe mod that lookup IOCs in VirusTotal, Urlscan and other tools."
 
+  tags = {
+    type = "featured"
+  }
+
   param "hunter_api_key" {
     type        = string
     default     = var.hunter_api_key
@@ -31,7 +35,6 @@ pipeline "lookup_iocs" {
     description = "The iocs to be scanned."
   }
 
-  # Domain
   step "pipeline" "domain_lookup" {
     for_each = { for ioc in param.iocs : ioc.id => ioc if ioc.type == "domain" }
     pipeline = pipeline.lookup_domain
@@ -40,7 +43,6 @@ pipeline "lookup_iocs" {
     }
   }
 
-  # Email
   step "pipeline" "email_lookup" {
     for_each = { for ioc in param.iocs : ioc.id => ioc if ioc.type == "email" }
     pipeline = pipeline.lookup_email
@@ -51,7 +53,6 @@ pipeline "lookup_iocs" {
     }
   }
 
-  # File hash
   step "pipeline" "file_hash_lookup" {
     for_each = { for ioc in param.iocs : ioc.id => ioc if ioc.type == "file_hash" }
     pipeline = pipeline.lookup_file_hash
@@ -61,7 +62,6 @@ pipeline "lookup_iocs" {
     }
   }
 
-  # IP
   step "pipeline" "ip_lookup" {
     for_each = { for ioc in param.iocs : ioc.id => ioc if ioc.type == "ip" }
     pipeline = pipeline.lookup_ip
@@ -70,7 +70,6 @@ pipeline "lookup_iocs" {
     }
   }
 
-  # URL
   step "pipeline" "url_lookup" {
     for_each = { for ioc in param.iocs : ioc.id => ioc if ioc.type == "url" }
     pipeline = pipeline.lookup_url

lookup_iocs/lookup_ip.fp

@@ -2,6 +2,24 @@ pipeline "lookup_ip" {
   title       = "Lookup IP in Different Tools"
   description = "A composite Flowpipe mod that lookup an IP in VirusTotal, Urlscan and other tools."
 
+  param "abuseipdb_cred" {
+    type        = string
+    description = "Name for AbuseIPDB credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
+  param "ip2locationio_cred" {
+    type        = string
+    description = "Name for IP2Location credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
+  param "urlscanio_cred" {
+    type        = string
+    description = "Name for  URLScan.io credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
   param "ip_address" {
     type        = string
     description = "The IP address to be scanned."
@@ -14,10 +32,10 @@ pipeline "lookup_ip" {
     description = "Maximum age in days for the AbuseIPDB reports to retrieve. Defaults to 30 days."
   }
 
-  # AbuseIPDB
   step "pipeline" "abuseipdb_ip_info" {
     pipeline = abuseipdb.pipeline.check_ip_address
     args = {
+      cred            = param.abuseipdb_cred
       ip_address      = param.ip_address
       max_age_in_days = param.max_age_in_days
     }
@@ -26,23 +44,24 @@ pipeline "lookup_ip" {
   step "pipeline" "abuseipdb_reports" {
     pipeline = abuseipdb.pipeline.list_ip_address_reports
     args = {
+      cred            = param.abuseipdb_cred
       ip_address      = param.ip_address
       max_age_in_days = param.max_age_in_days
     }
   }
 
-  # IP2Location.io
   step "pipeline" "ip2location_ip_lookup" {
     pipeline = ip2locationio.pipeline.get_ip_info
     args = {
+      cred       = param.ip2locationio_cred
       ip_address = param.ip_address
     }
   }
 
-  # Urlscan.io
   step "pipeline" "urlscan_ip_lookup" {
     pipeline = urlscanio.pipeline.search_scan
     args = {
+      cred  = param.urlscanio_cred
       query = "domain:${param.ip_address}"
     }
   }

lookup_iocs/lookup_url.fp

@@ -2,6 +2,12 @@ pipeline "lookup_url" {
   title       = "Lookup URL in Different Tools"
   description = "A composite Flowpipe mod that lookup an url in VirusTotal, Urlscan and other tools."
 
+  param "virustotal_cred" {
+    type        = string
+    description = "Name for VirusTotal credentials to use. If not provided, the default credentials will be used."
+    default     = "default"
+  }
+
   param "apivoid_api_key" {
     type        = string
     default     = var.apivoid_api_key
@@ -13,15 +19,14 @@ pipeline "lookup_url" {
     description = "The url to be scanned."
   }
 
-  # VirusTotal
   step "pipeline" "virustotal_url_lookup" {
     pipeline = virustotal.pipeline.get_url_analysis
     args = {
-      url = param.url
+      cred = param.virustotal_cred
+      url  = param.url
     }
   }
 
-  # APIVoid
   step "http" "apivoid_url_reputation" {
     method = "get"
     url    = "https://endpoint.apivoid.com/urlrep/v1/pay-as-you-go/?key=${param.apivoid_api_key}&url=${param.url}"

lookup_iocs/mod.fp

@@ -3,18 +3,23 @@ mod "lookup_iocs" {
   description = "A composite Flowpipe mod that lookup IOCS in VirusTotal, Urlscan and other tools."
   categories  = ["security"]
 
+  opengraph {
+    title       = "Lookup IOCs in different tools"
+    description = "A composite Flowpipe mod that lookup IOCS in VirusTotal, Urlscan and other tools."
+  }
+
   require {
     mod "github.com/turbot/flowpipe-mod-abuseipdb" {
-      version = "v0.0.1-rc.5"
+      version = "v0.0.1-rc.8"
     }
     mod "github.com/turbot/flowpipe-mod-virustotal" {
-      version = "v0.0.1-rc.7"
+      version = "v0.0.1-rc.9"
     }
     mod "github.com/turbot/flowpipe-mod-urlscanio" {
-      version = "v0.0.1-rc.2"
+      version = "v0.0.1-rc.7"
     }
     mod "github.com/turbot/flowpipe-mod-ip2locationio" {
-      version = "v0.0.1-rc.1"
+      version = "v0.0.1-rc.4"
     }
   }
 }