Search Open Websites/Domains

let KnownReconUserAgents = dynamic(["python-requests", "python-urllib", "go-http-client", "curl/", "wget/", "nuclei", "nikto", "dirbuster", "gobuster", "feroxbuster", "ffuf", "sqlmap", "scrapy", "zgrab", "masscan", "shodan", "censys", "binaryedge", "nmap", "burpsuite", "zap", "httpx", "katana", "subfinder", "amass", "theHarvester", "mechanize", "httplib2", "libwww-perl"]);
let SensitivePaths = dynamic(["/.git", "/.env", "/wp-admin", "/phpmyadmin", "/admin", "/robots.txt", "/sitemap.xml", "/.htaccess", "/web.config", "/backup", "/config", "/.well-known", "/xmlrpc.php", "/wp-login"]);
W3CIISLog
| where TimeGenerated > ago(1h)
| where isnotempty(cIP)
| extend UserAgentLower = tolower(csUserAgent)
| extend IsReconUA = iff(
    csUserAgent has_any (KnownReconUserAgents) or isempty(csUserAgent),
    true, false)
| extend IsSensitivePath = iff(
    csUriStem has_any (SensitivePaths),
    true, false)
| summarize
    TotalRequests = count(),
    UniqueURIs = dcount(csUriStem),
    UniquePaths = make_set(csUriStem, 30),
    ReconUARequests = countif(IsReconUA == true),
    SensitivePathHits = countif(IsSensitivePath == true),
    StatusCodes = make_set(scStatus),
    UserAgents = make_set(csUserAgent, 10),
    FirstRequest = min(TimeGenerated),
    LastRequest = max(TimeGenerated)
    by cIP, bin(TimeGenerated, 1h)
| where TotalRequests > 30 or ReconUARequests > 5 or SensitivePathHits > 3 or UniqueURIs > 25
| extend RiskScore = case(
    ReconUARequests > 20 and SensitivePathHits > 5, "High",
    ReconUARequests > 5 or SensitivePathHits > 3 or UniqueURIs > 50, "Medium",
    "Low")
| project
    TimeGenerated,
    SourceIP = cIP,
    TotalRequests,
    UniqueURIs,
    ReconUARequests,
    SensitivePathHits,
    SampledPaths = UniquePaths,
    UserAgents,
    StatusCodes,
    RiskScore,
    FirstRequest,
    LastRequest
| order by RiskScore asc, TotalRequests desc

index=web (sourcetype="iis" OR sourcetype="apache:access" OR sourcetype="nginx:plus:access" OR sourcetype="access_combined" OR sourcetype="access_combined_wcookie")
| eval ua_lower=lower(http_user_agent)
| eval is_recon_ua=if(
    match(ua_lower, "(python-requests|python-urllib|go-http-client|nuclei|nikto|dirbuster|gobuster|feroxbuster|ffuf|sqlmap|scrapy|zgrab|masscan|shodan|censys|binaryedge|nmap|burpsuite|zaproxy|httpx|katana|subfinder|amass|theharvester|mechanize|libwww-perl|curl\/|wget\/)")
    OR isnull(http_user_agent) OR http_user_agent="-",
    1, 0)
| eval is_sensitive_path=if(
    match(uri_path, "(\/\.git|\/\.env|\/wp-admin|\/phpmyadmin|\/admin|\/robots\.txt|\/sitemap\.xml|\/\.htaccess|\/web\.config|\/backup|\/config|\/xmlrpc\.php|\/wp-login|\/\.well-known)"),
    1, 0)
| bin _time span=1h
| stats
    count as total_requests,
    dc(uri_path) as unique_uris,
    sum(is_recon_ua) as recon_ua_requests,
    sum(is_sensitive_path) as sensitive_path_hits,
    values(http_user_agent) as user_agents,
    values(status) as status_codes,
    values(uri_path) as sampled_paths,
    min(_time) as first_request,
    max(_time) as last_request
    by _time, src_ip
| where total_requests > 30 OR recon_ua_requests > 5 OR sensitive_path_hits > 3 OR unique_uris > 25
| eval risk_score=case(
    recon_ua_requests > 20 AND sensitive_path_hits > 5, "High",
    recon_ua_requests > 5 OR sensitive_path_hits > 3 OR unique_uris > 50, "Medium",
    1=1, "Low")
| table _time, src_ip, total_requests, unique_uris, recon_ua_requests, sensitive_path_hits, user_agents, status_codes, risk_score, first_request, last_request
| sort - risk_score total_requests

// T1593 — Search Open Websites
any where event.dataset : ("iis.access", "apache_http_server.access")
  and (user_agent.original : ("python-requests*", "go-http-client*", "curl/*", "nuclei*",
    "nikto*", "dirbuster*", "gobuster*", "scrapy*", "masscan*")
  or url.path : ("/.git/*", "/.env", "/wp-admin/*", "/robots.txt", "/sitemap.xml",
    "/.htaccess", "/web.config", "/backup/*", "/config/*"))

SELECT
    DATEFORMAT(devicetime, 'yyyy-MM-dd HH:mm:ss') AS "EventTime",
    LOGSOURCENAME(logsourceid) AS "LogSource",
    LOGSOURCETYPENAME(devicetype) AS "LogSourceType",
    "username", "sourceip", "destinationip",
    "eventid", "deviceaction", "message",
    CASE
        WHEN LOWER("useragent") ILIKE '%python-requests%' OR LOWER("useragent") ILIKE '%nuclei%' OR LOWER("requesturl") ILIKE '%.git%' OR LOWER("requesturl") ILIKE '%.env%' THEN 8
        ELSE 4
      END AS "RiskScore"
  FROM events
  WHERE (LOWER("useragent") ILIKE '%python-requests%' OR LOWER("useragent") ILIKE '%nuclei%' OR LOWER("requesturl") ILIKE '%.git%' OR LOWER("requesturl") ILIKE '%.env%')
    AND LOGSOURCETYPENAME(devicetype) NOT IN ('SIM Audit', 'Custom Rule Engine')
  ORDER BY "RiskScore" DESC, "EventTime" DESC
  LAST 24 HOURS

_sourceCategory=*web* OR _sourceCategory=*iis* OR _sourceCategory=*apache*
| parse regex "(?<client_ip>\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}) - (?<user>[^ ]+) .* \"(?<method>[A-Z]+) (?<uri>[^ ]+).*\" (?<status>\\d+)"
| count by client_ip, uri
| sort by _count desc

rule t1593_search_open_websites_domains {
  meta:
    author = "df00tech"
    description = "Detects Search Open Websites/Domains (T1593)"
    mitre_attack_tactic = "TA0043"
    mitre_attack_technique = "T1593"
    confidence = "low"
    severity = "medium"
  events:
    $e.metadata.event_type = "NETWORK_HTTP"
    $e.principal.ip != ""
  condition:
    $e
}

#event_simpleName = "ProcessRollup2"
| ImageFileName = /python|curl|wget|nmap|masscan/i
| TechniqueLabel := "T1593 - Reconnaissance"
| table([@timestamp, ComputerName, UserName, ImageFileName, CommandLine, TechniqueLabel])