diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 573b986ea8..40567ba136 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,12 +12,13 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner +from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context logger = logging.getLogger(__name__) TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") -DNSSEC_NAMESERVER_HOSTNAME= os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") @dataclass @@ -27,6 +28,8 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None + ns_delegations: dict = None + registrar_context: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -50,9 +53,9 @@ def find_zone_apex(domain, resolver=None): while name != dns.name.root: logger.debug(f"Checking for SOA at {name}") try: - answers = resolver.resolve(name, 'SOA') + answers = resolver.resolve(name, "SOA") logger.debug(f"Found SOA for {domain} at {name}: {answers[0]}") - zone_apex = str(name).rstrip('.') + zone_apex = str(name).rstrip(".") return zone_apex except NoAnswer: # Go up one level @@ -105,7 +108,13 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker """ try: q = dns.message.make_query(domain, dns.rdatatype.DNSKEY, want_dnssec=True) - resp = dns.query.tls(q, where=nameserver_ip, timeout=TIMEOUT, server_hostname=nameserver_hostname, verify=True) + resp = dns.query.tls( + q, + where=nameserver_ip, + timeout=TIMEOUT, + server_hostname=nameserver_hostname, + verify=True, + ) if resp.rcode() != dns.rcode.NOERROR: return None except Timeout: @@ -114,7 +123,9 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker ) return None except Exception as e: - logger.error(f"Error while running minimal DNSSEC check for domain '{domain}': {e}") + logger.error( + f"Error while running minimal DNSSEC check for domain '{domain}': {e}" + ) return None # Check if AD (Authenticated Data) flag is set, showing that the data is DNSSEC validated return bool(resp.flags & dns.flags.AD) @@ -136,7 +147,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): result = {"wildcard_entry": False, "wildcard_sibling": False} try: wildcard_sibling_domain = re.sub(r"^[^.]+", "*", domain) - wildcard_record = dns.resolver.resolve( + wildcard_record = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.A, raise_on_no_answer=False, @@ -148,7 +159,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): try: # check for mail-only subdomain (e.g. mail.example.com) mx_records = resolver.resolve(qname=domain, rdtype=dns.rdatatype.MX) - wildcard_mx = dns.resolver.resolve( + wildcard_mx = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.MX, raise_on_no_answer=False, @@ -284,10 +295,16 @@ def scan_domain(domain, dkim_selectors=None): logger.debug(f"Skipping DNSSEC check for {domain} - No zone apex found") zone_dnssec_enabled = None elif not DNSSEC_NAMESERVER_IP or not DNSSEC_NAMESERVER_HOSTNAME: - logger.debug(f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set") + logger.debug( + f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set" + ) zone_dnssec_enabled = None else: - zone_dnssec_enabled = dnssec_check_with_ttl(domain=zone_apex, nameserver_ip=DNSSEC_NAMESERVER_IP, nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME) + zone_dnssec_enabled = dnssec_check_with_ttl( + domain=zone_apex, + nameserver_ip=DNSSEC_NAMESERVER_IP, + nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME, + ) scan_result.zone_dnssec_enabled = zone_dnssec_enabled @@ -297,12 +314,24 @@ def scan_domain(domain, dkim_selectors=None): dmarc_scanner = DMARCScanner(domain) dmarc_scan_result = dmarc_scanner.run() scan_result.base_domain = dmarc_scan_result.get("base_domain", "") - scan_result.ns_records = dmarc_scan_result.get("ns", {}) scan_result.mx_records = dmarc_scan_result.get("mx", {}) scan_result.spf = dmarc_scan_result.get("spf", {}) scan_result.dmarc = dmarc_scan_result.get("dmarc", {}) logger.debug(f"DMARC scan elapsed time: {time.monotonic() - dmarc_start_time}") + ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) + scan_result.ns_records = ns_records + # check nameserver delegations + scan_result.ns_delegations = check_ns_delegations( + domain=domain, zone_apex=zone_apex, ns_records=ns_records + ) + + registrar_domain = scan_result.base_domain or zone_apex or domain + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=ns_delegations.get("ns_hosts", []), + ) + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py new file mode 100644 index 0000000000..4e60d31dc4 --- /dev/null +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -0,0 +1,196 @@ +import os + +import dns +import dns.resolver +import requests +from dns.exception import Timeout +from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers + +TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) + + +def probe_nameserver( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query( + qname, + dns.rdatatype.from_text(qtype), + use_edns=True, + ) + if not recursion_desired: + query.flags &= ~dns.flags.RD + return dns.query.udp(query, where=where, timeout=timeout) + + +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + + ns_hosts = ns_records.get("hostnames", []) + if len(ns_hosts) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", + }, + } + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": qname, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = get_ns_ip(host, resolver) + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = probe_nameserver(ns_ip, qname, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + +def get_registrar_context(base_domain, ns_hosts=None): + context = { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": None, + } + + if not base_domain: + context["error"] = "missing_base_domain" + return context + + rdap_url = f"https://rdap.org/domain/{base_domain}" + context["rdap_url"] = rdap_url + + try: + response = requests.get(rdap_url, timeout=TIMEOUT) + response.raise_for_status() + payload = response.json() + except Exception as e: + context["error"] = str(e) + return context + + context["lookup_success"] = True + + nameservers = payload.get("nameservers", []) + context["rdap_nameservers"] = [ + (ns.get("ldhName") or "").rstrip(".").lower() + for ns in nameservers + if ns.get("ldhName") + ] + + entities = payload.get("entities", []) + for entity in entities: + roles = [r.lower() for r in entity.get("roles", [])] + if "registrar" not in roles: + continue + + context["registrar_id"] = entity.get("handle") + + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) == 2 and isinstance(vcard[1], list): + for entry in vcard[1]: + if not isinstance(entry, list) or len(entry) < 4: + continue + key = entry[0] + value = entry[3] + if key in {"fn", "org"} and value: + context["registrar_name"] = value + break + + if context["registrar_name"] is None: + public_ids = entity.get("publicIds", []) + if public_ids: + context["registrar_name"] = public_ids[0].get("identifier") + + break + + if ns_hosts is not None: + normalized_hosts = {h.rstrip(".").lower() for h in ns_hosts if h} + rdap_hosts = set(context["rdap_nameservers"]) + if rdap_hosts: + context["delegation_matches_rdap"] = normalized_hosts == rdap_hosts + + return context diff --git a/scanners/subdomain-takeover/.env.example b/scanners/subdomain-takeover/.env.example new file mode 100644 index 0000000000..31d2f06813 --- /dev/null +++ b/scanners/subdomain-takeover/.env.example @@ -0,0 +1,7 @@ +NATS_URL= +NATS_STREAM= +SUBJECT_IN= +SUBJECT_OUT= +DURABLE_NAME= +WORKER_COUNT= +LOG_LEVEL= diff --git a/scanners/subdomain-takeover/Dockerfile b/scanners/subdomain-takeover/Dockerfile new file mode 100644 index 0000000000..6c7f3eafaa --- /dev/null +++ b/scanners/subdomain-takeover/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache git ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/subdomain-takeover ./cmd/service + +# Runtime stage +FROM alpine:3.22 + +RUN addgroup -S scanner && adduser -S scanner -G scanner +WORKDIR /app +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /out/subdomain-takeover /app/subdomain-takeover + +USER scanner +ENTRYPOINT ["/app/subdomain-takeover"] diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md new file mode 100644 index 0000000000..7e28388320 --- /dev/null +++ b/scanners/subdomain-takeover/README.md @@ -0,0 +1,140 @@ +# subdomain-takeover-service + +Go microservice that consumes DNS scan results and emits normalized subdomain takeover findings. + +## What it does + +1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) +2. Evaluates passive CNAME and NS takeover risk using provider fingerprints +3. Assigns confidence (`suspected`, `probable`, `confirmed`) +4. Publishes findings to `scans.findings.upsert` + +## Current detection scope + +Implemented today: + +- CNAME passive detection +- NS passive detection (vulnerable-only emission policy) +- Deterministic NS candidate selection when multiple providers/hosts match +- Nil-safe evidence extraction for CNAME and NS +- Classifier refactor to support injected fingerprint sources (`FingerprintSource`) + +Current NS behavior: + +- NS provider/host matches are evaluated and ranked +- Findings are emitted only for exploitable NS outcomes: + - `NS_FULL_LAME_PROVIDER_VULNERABLE` + - `NS_PARTIAL_LAME_PROVIDER_VULNERABLE` +- Non-exploitable NS matches (`NS_LAME_PROVIDER_UNKNOWN`, `NS_PROVIDER_MATCH_ONLY`) are classified but not emitted + +## Passive-only detection policy + +This service uses passive checks only. +It does not attempt account takeover, resource claiming, or any active validation. + +As a result, confidence is conservative and evidence-driven: +- `suspected`: weak or incomplete takeover indicators +- `probable`: strong passive indicators with known exploitable conditions +- `confirmed`: reserved for deterministic passive signatures only + +## Confidence model (current) + +### CNAME + +- `suspected` + - provider target matched, but required passive signal is missing +- `probable` + - dangling target with NXDOMAIN where required, or + - provider fingerprint body match for providers that require body verification +- `confirmed` + - reserved; not currently emitted by CNAME rules + +### NS + +- `confirmed` + - full lame delegation + exploitable provider status +- `probable` + - partial lame delegation + exploitable provider status +- `suspected` + - reason codes exist for weaker NS states, but these are currently not emitted + +## Internal package layout + +Detection logic was split into focused files to reduce coupling: + +- `internal/detect/cname_rules.go` +- `internal/detect/ns_rules.go` +- `internal/detect/ns_reasoning.go` +- `internal/detect/cname_evidence.go` +- `internal/detect/ns_evidence.go` +- `internal/detect/fingerprint_source.go` + +`Classifier` now supports dependency injection via `NewClassifierWithSource(...)`, while `NewClassifier(...)` defaults to global loaded fingerprints. + +## Fingerprint data + +Provider fingerprint data is stored in: + +- `internal/detect/data/cname_fingerprints.json` +- `internal/detect/data/ns_fingerprints.json` + +The service embeds and validates these files at startup (`detect.LoadFingerprints`). + +For CNAME body matching: + +- `mode: literal` performs plain substring search +- `mode: regex` performs regular expression matching +- if `mode` is omitted, the loader applies backward-compatible inference + +## Prerequisites + +- Go 1.25+ +- NATS with JetStream enabled +- `nats` CLI (optional, for local testing) + +## Quick start + +```bash +go mod tidy +go run ./cmd/service +``` + +Environment variables: +- `NATS_URL` (default: `nats://localhost:4222`) +- `NATS_STREAM` (default: `SCANS`) +- `SUBJECT_IN` (default: `scans.dns_scanner_results`) +- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `DURABLE_NAME` (default: `subdomain_takeover`) +- `WORKER_COUNT` (default: `5`) +- `LOG_LEVEL` (default: `info`) + +## Message ack behavior + +The worker uses explicit JetStream ack semantics: + +- decode failures -> `Term()` (drop poison message) +- classify/publish failures -> `Nak()` (retryable) +- successful processing -> `Ack()` + +## Local test flow + +Publish a test request: + +```bash +nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"example.gc.ca","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_delegations":{"ns_hosts":["ns1.example-dns-provider.net"],"ns_checks":[],"ns_delegation":{"total_ns":1,"authoritative_ok":0,"lame_count":1,"lame_type":"full"},"error":""}}}' +``` + +Watch findings: + +```bash +nats sub "scans.findings.upsert" +``` + +## Next steps / nice-to-haves + +1. Add table-driven tests for NS reason mapping, ranking, and emission gating. +2. Add classifier integration tests using injected `FingerprintSource`. +3. Document and implement explicit policy for `edge_case` and `registration_closed` NS provider statuses. +4. Add structured debug logs for NS matching decisions (host, provider, status, reason). +5. Add registrar-context mismatch handling (`RegistrarMismatch`) and confidence policy. +6. Add metrics (counts by reason code, emitted vs suppressed) for production observability. diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..78a2500bff --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/rs/zerolog" +) + +func main() { + logger := bootstrap.NewLogger(zerolog.InfoLevel) + + cfg, err := config.Load() + if err != nil { + logger.Fatal().Err(err).Msg("failed to load config") + } + + logger = bootstrap.NewLogger(cfg.LogLevel) + logger.Info(). + Str("nats_url", cfg.NATSURL). + Str("stream", cfg.NATSStream). + Str("subject_in", cfg.SubjectIn). + Str("subject_out", cfg.SubjectOut). + Str("durable", cfg.DurableName). + Int("worker_count", cfg.WorkerCount). + Str("log_level", cfg.LogLevel.String()). + Msg("service configuration loaded") + + if err := detect.LoadFingerprints(logger); err != nil { + logger.Fatal().Err(err).Msg("failed to load fingerprints") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) + + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg, logger) + if err != nil { + logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") + } + + logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) + + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) + classifier := detect.NewClassifier(matcher).WithLogger(logger) + worker := app.NewWorker(logger, pub, classifier) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + runtimeDeps.Iter.Stop() + }() + + deps := app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, + } + + app.Run(ctx, deps) + + logger.Info().Msgf("Disconnecting from NATS at %s", runtimeDeps.NC.ConnectedUrl()) + if err := runtimeDeps.NC.Flush(); err != nil { + logger.Error().Err(err).Msg("failed to flush nats connection") + } + runtimeDeps.NC.Close() + + if err := ctx.Err(); err != nil && err != context.Canceled { + logger.Error().Err(err).Msg("service exited with context error") + } +} diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod new file mode 100644 index 0000000000..ca7b20f4ac --- /dev/null +++ b/scanners/subdomain-takeover/go.mod @@ -0,0 +1,19 @@ +module github.com/canada-ca/tracker/scanners/subdomain-takeover + +go 1.25.0 + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats.go v1.52.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/klauspost/compress v1.18.5 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum new file mode 100644 index 0000000000..470146821e --- /dev/null +++ b/scanners/subdomain-takeover/go.sum @@ -0,0 +1,21 @@ +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go new file mode 100644 index 0000000000..b8c0121f87 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -0,0 +1,95 @@ +package app + +import ( + "context" + "sync" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RunnerDeps struct { + Logger zerolog.Logger + WorkerCount int + Iter jetstream.MessagesContext + Worker MessageHandler + NC *nats.Conn +} + +type MessageHandler interface { + Handle(ctx context.Context, msg jetstream.Msg) error +} + +func Run(ctx context.Context, deps RunnerDeps) { + logger := deps.Logger + iter := deps.Iter + nextErrCount := 0 + + if deps.WorkerCount < 1 { + deps.WorkerCount = 1 + } + + logger.Info().Int("worker_count", deps.WorkerCount).Msg("runner started") + + var wg sync.WaitGroup + sem := make(chan struct{}, deps.WorkerCount) + +Loop: + for { + if err := messaging.CheckConnection(deps.NC); err != nil { + logger.Error().Err(err).Msg("NATS connection unhealthy") + break Loop + } + + select { + case <-ctx.Done(): + break Loop + default: + } + + msg, err := iter.Next() + if err != nil { + if ctx.Err() != nil { + logger.Info().Msg("runner stopping: context canceled") + break Loop + } + nextErrCount++ + if nextErrCount%10 == 0 { + logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") + } else { + logger.Debug().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("next returned, continuing") + } + continue + } + nextErrCount = 0 + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + deps.Worker.Handle(ctx, m) + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Warn().Msg("drain timeout") + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go new file mode 100644 index 0000000000..eca59f91bc --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -0,0 +1,95 @@ +package app + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type FindingPublisher interface { + Publish(ctx context.Context, finding model.Finding) error +} + +type ScanClassifier interface { + Classify(input model.Input) ([]model.Finding, error) +} + +type Worker struct { + logger zerolog.Logger + publisher FindingPublisher + classifier ScanClassifier +} + +func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { + return &Worker{logger: logger, publisher: publisher, classifier: classifier} +} + +func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { + log := w.logger.With(). + Str("component", "worker"). + Str("subject", msg.Subject()). + Int("msg_size", len(msg.Data())). + Logger() + + scan, err := decodeScan(msg.Data()) + if err != nil { + log.Err(err).Msg("decode error") + w.term(msg, log, err) + return err + } + + log = log.With().Str("domain_key", scan.DomainKey).Logger() + + findings, err := w.classifier.Classify(scan) + if err != nil { + log.Err(err).Msg("classify error") + w.nak(msg, log, err) + return err + } + + if len(findings) == 0 { + log.Debug().Msg("classification produced no findings") + } + + for _, finding := range findings { + err = w.publisher.Publish(ctx, finding) + if err != nil { + log.Err(err).Msg("publish error") + w.nak(msg, log, err) + return err + } + } + + if err := msg.Ack(); err != nil { + log.Err(err).Msg("ack error") + return err + } + + log.Debug().Int("findings_count", len(findings)).Msg("message acknowledged") + + return nil +} + +func decodeScan(data []byte) (model.Input, error) { + var scan model.Input + trimmed := bytes.Trim(data, "\n") + err := json.Unmarshal(trimmed, &scan) + return scan, err +} + +func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Nak(); err != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") + } +} + +func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Term(); err != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") + } +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/logger.go b/scanners/subdomain-takeover/internal/bootstrap/logger.go new file mode 100644 index 0000000000..3721d23e40 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/logger.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "os" + + "github.com/rs/zerolog" +) + +func NewLogger(level zerolog.Level) zerolog.Logger { + zerolog.SetGlobalLevel(level) + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + return zerolog.New(multi).With().Timestamp().Logger() +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go new file mode 100644 index 0000000000..ea2fc49e21 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -0,0 +1,65 @@ +package bootstrap + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RuntimeDeps struct { + NC *nats.Conn + Iter jetstream.MessagesContext + JS jetstream.JetStream +} + +func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logger) (*RuntimeDeps, error) { + logger.Debug().Str("nats_url", cfg.NATSURL).Msg("connecting to nats") + nc, err := nats.Connect(cfg.NATSURL) + if err != nil { + return nil, err + } + + logger.Debug().Msg("creating jetstream client") + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Msg("creating pull message iterator") + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + nc.Close() + return nil, err + } + + return &RuntimeDeps{ + NC: nc, + Iter: iter, + JS: js, + }, nil +} diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go new file mode 100644 index 0000000000..1b18e68117 --- /dev/null +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -0,0 +1,25 @@ +package config + +import ( + "github.com/kelseyhightower/envconfig" + "github.com/rs/zerolog" +) + +type Config struct { + NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` + SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` + WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` + LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` +} + +func Load() (*Config, error) { + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + return nil, err + } + + return &cfg, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go new file mode 100644 index 0000000000..622b2f7c51 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -0,0 +1,65 @@ +package detect + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +type BodyFingerprintMatcher interface { + Contains(domain string, fingerprint string, mode FingerprintMode) bool +} + +type HTTPBodyFingerprintMatcher struct { + client *http.Client +} + +func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { + return &HTTPBodyFingerprintMatcher{ + client: &http.Client{Timeout: timeout}, + } +} + +type NoopBodyFingerprintMatcher struct{} + +func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { + return &NoopBodyFingerprintMatcher{} +} + +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + return false +} + +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if m == nil || m.client == nil { + return false + } + + url := fmt.Sprintf("http://%s", domain) + res, err := m.client.Get(url) + if err != nil { + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false + } + + bodyText := string(body) + resolvedMode := normalizeFingerprintMode(mode, fingerprint) + + if resolvedMode == FingerprintModeRegex { + re, err := regexp.Compile(fingerprint) + if err != nil { + return false + } + return re.MatchString(bodyText) + } + + return strings.Contains(bodyText, fingerprint) +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go new file mode 100644 index 0000000000..ffd1d527c6 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -0,0 +1,129 @@ +package detect + +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +type Classifier struct { + Matcher BodyFingerprintMatcher + Source FingerprintSource + logger zerolog.Logger +} + +func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} +} + +func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { + if source == nil { + source = GlobalFingerprintSource{} + } + + return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} +} + +func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { + c.logger = logger.With().Str("component", "classifier").Logger() + SetLogger(logger) + return c +} + +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { + matcher := c.Matcher + if matcher == nil { + matcher = NewNoopBodyFingerprintMatcher() + } + + source := c.Source + if source == nil { + source = GlobalFingerprintSource{} + } + + logger := c.logger + if logger.GetLevel() == zerolog.NoLevel { + logger = zerolog.Nop() + } + + return Classify(input, matcher, source, logger) +} + +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { + findings := []model.Finding{} + + cnameFingerprints := source.CNAME() + nsFingerprints := source.NS() + + cnameEvidence := ExtractCNAMEEvidence(input.Results) + if cnameEvidence != nil { + logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) + if ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("emitting cname finding") + findings = append(findings, model.Finding{ + Domain: cnameEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeCNAME, + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: string(cnameHit.ReasonCode), + Confidence: ConfidenceForReason(cnameHit.ReasonCode), + }) + } + if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("cname finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") + } + + nsEvidence := ExtractNSEvidence(input.Results) + if nsEvidence != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Int("ns_hosts", len(nsEvidence.NSHosts)). + Msg("ns evidence extracted") + nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) + if ShouldEmitNSHijack(nsHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("emitting ns finding") + findings = append(findings, model.Finding{ + Domain: nsEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeNS, + Target: nsHit.Host, + Provider: nsHit.Provider, + ReasonCode: string(nsHit.ReasonCode), + Confidence: ConfidenceForReason(nsHit.ReasonCode), + }) + } else if nsHit != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("ns finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") + } + + return findings, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_evidence.go b/scanners/subdomain-takeover/internal/detect/cname_evidence.go new file mode 100644 index 0000000000..4466dcd9fc --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type CNAMEEvidence struct { + Domain string + Target string + NoResolve bool +} + +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil || results.Domain == nil { + return nil + } + + return &CNAMEEvidence{ + Domain: *results.Domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, + } +} + +func parseCname(record string) string { + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") + return recordTokens[len(recordTokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go new file mode 100644 index 0000000000..b693a4cd70 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -0,0 +1,77 @@ +package detect + +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode ReasonCode + NeedsNX bool +} + +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("target", evidence.Target). + Str("provider", fp.Name). + Bool("needs_nx", fp.Nxdomain). + Msg("cname provider target matched") + + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + + if hit.NeedsNX { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN + } + } else { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("fingerprint_mode", string(mode)). + Msg("checking cname body fingerprint") + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname body fingerprint matched") + } else { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: body fingerprint missing") + } + } + + if hit.NeedsNX && !hit.Matched { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: required nxdomain signal missing") + } + + return &hit + } + } + return nil +} + +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go new file mode 100644 index 0000000000..9865a561db --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -0,0 +1,33 @@ +package detect + +const ( + ConfidenceSuspected = "suspected" + ConfidenceProbable = "probable" + ConfidenceConfirmed = "confirmed" +) + +// ConfidenceForReason maps stable reason codes to confidence levels. +// Unknown reason codes default to suspected. +func ConfidenceForReason(reasonCode ReasonCode) string { + switch reasonCode { + case ReasonCNAMEDanglingNXDOMAIN: + return ConfidenceProbable + case ReasonCNAMEProviderFingerprintBodyMatch: + return ConfidenceProbable + case ReasonCNAMETargetMatchMissingNXDOMAIN: + return ConfidenceSuspected + case ReasonCNAMETargetMatchMissingBodyFP: + return ConfidenceSuspected + + case ReasonNSFullLameProviderVulnerable: + return ConfidenceConfirmed + case ReasonNSPartialLameProviderVulnerable: + return ConfidenceProbable + case ReasonNSLameProviderUnknown: + return ConfidenceSuspected + case ReasonNSProviderMatchOnly: + return ConfidenceSuspected + default: + return ConfidenceSuspected + } +} diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json new file mode 100644 index 0000000000..cbf3aa24be --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json @@ -0,0 +1,227 @@ +[ + { + "cname": [ + "elasticbeanstalk.com" + ], + "name": "AWS/Elastic Beanstalk", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "s3.amazonaws.com" + ], + "name": "AWS/S3", + "nxdomain": false, + "fingerprint": "The specified bucket does not exist" + }, + { + "cname": [ + "agilecrm.com" + ], + "name": "Agile CRM", + "nxdomain": false, + "fingerprint": "Sorry, this page is no longer available." + }, + { + "cname": [ + "airee.ru" + ], + "name": "Airee.ru", + "nxdomain": false, + "fingerprint": "Ошибка 402. Сервис Айри.рф не оплачен" + }, + { + "cname": [ + "animaapp.io" + ], + "name": "Anima", + "nxdomain": false, + "fingerprint": "The page you were looking for does not exist." + }, + { + "cname": [ + "bitbucket.io" + ], + "name": "Bitbucket", + "nxdomain": false, + "fingerprint": "Repository not found" + }, + { + "cname": [ + "trydiscourse.com" + ], + "name": "Discourse", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "furyns.com" + ], + "name": "Gemfury", + "nxdomain": false, + "fingerprint": "404: This page could not be found." + }, + { + "cname": [ + "ghost.io" + ], + "name": "Ghost", + "nxdomain": false, + "fingerprint": "Site unavailable\\.\u0026#124;Failed to resolve DNS path for this host" + }, + { + "cname": [ + "hatenablog.com" + ], + "name": "HatenaBlog", + "nxdomain": false, + "fingerprint": "404 Blog is not found" + }, + { + "cname": [ + "helpjuice.com" + ], + "name": "Help Juice", + "nxdomain": false, + "fingerprint": "We could not find what you're looking for." + }, + { + "cname": [ + "helpscoutdocs.com" + ], + "name": "Help Scout", + "nxdomain": false, + "fingerprint": "No settings were found for this company:" + }, + { + "cname": [ + "helprace.com" + ], + "name": "Helprace", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=301" + }, + { + "cname": [ + "youtrack.cloud" + ], + "name": "JetBrains", + "nxdomain": false, + "fingerprint": "is not a registered InCloud YouTrack" + }, + { + "cname": [ + "launchrock.com" + ], + "name": "LaunchRock", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=500" + }, + { + "cname": [ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com" + ], + "name": "Microsoft Azure", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "ngrok.io" + ], + "name": "Ngrok", + "nxdomain": false, + "fingerprint": "Tunnel .*.ngrok.io not found" + }, + { + "cname": [ + "readme.io" + ], + "name": "Readme.io", + "nxdomain": false, + "fingerprint": "The creators of this project are still working on making everything perfect!" + }, + { + "cname": [ + "52.16.160.97" + ], + "name": "SmartJobBoard", + "nxdomain": false, + "fingerprint": "This job board website is either expired or its domain name is invalid." + }, + { + "cname": [ + "s.strikinglydns.com" + ], + "name": "Strikingly", + "nxdomain": false, + "fingerprint": "PAGE NOT FOUND." + }, + { + "cname": [ + "na-west1.surge.sh" + ], + "name": "Surge.sh", + "nxdomain": false, + "fingerprint": "project not found" + }, + { + "cname": [ + "surveysparrow.com" + ], + "name": "SurveySparrow", + "nxdomain": false, + "fingerprint": "Account not found." + }, + { + "cname": [ + "read.uberflip.com" + ], + "name": "Uberflip", + "nxdomain": false, + "fingerprint": "The URL you've accessed does not provide a hub." + }, + { + "cname": [ + "stats.uptimerobot.com" + ], + "name": "Uptimerobot", + "nxdomain": false, + "fingerprint": "page not found" + }, + { + "cname": [ + "wordpress.com" + ], + "name": "Wordpress", + "nxdomain": false, + "fingerprint": "Do you want to register .*.wordpress.com?" + }, + { + "cname": [ + "worksites.net", + "69.164.223.206" + ], + "name": "Worksites", + "nxdomain": false, + "fingerprint": "Hello! Sorry, but the website you\u0026rsquo;re looking for doesn\u0026rsquo;t exist." + } +] diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json new file mode 100644 index 0000000000..85fd5262bf --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json @@ -0,0 +1,416 @@ +[ + { + "name": "000Domains", + "provider_url": "https://000domains.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.000domains.com", + "ns2.000domains.com", + "fwns1.000domains.com", + "fwns2.000domains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/19" + }, + { + "name": "AWS Route 53", + "provider_url": "https://aws.amazon.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns-*.awsdns-*.org", + "ns-*.awsdns-*.co.uk", + "ns-*.awsdns-*.com", + "ns-*.awsdns-*.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/1" + }, + { + "name": "Azure (Microsoft)", + "provider_url": "https://azure.microsoft.com/", + "status": "edge_case", + "host_patterns": [ + "ns1-*.azure-dns.com", + "ns2-*.azure-dns.net", + "ns3-*.azure-dns.org", + "ns4-*.azure-dns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/5" + }, + { + "name": "BigCommerce", + "provider_url": "https://bigcommerce.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bigcommerce.com", + "ns2.bigcommerce.com", + "ns3.bigcommerce.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/35" + }, + { + "name": "Bizland", + "provider_url": "https://bizland.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bizland.com", + "ns2.bizland.com", + "clickme.click2site.com", + "clickme2.click2site.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/3" + }, + { + "name": "ClouDNS", + "provider_url": "https://cloudns.net/", + "status": "not_vulnerable", + "host_patterns": [ + "*.cloudns.net" + ] + }, + { + "name": "Cloudflare", + "provider_url": "https://cloudflare.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.ns.cloudflare.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/10" + }, + { + "name": "Digital Ocean", + "provider_url": "https://digitalocean.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/22" + }, + { + "name": "DNSMadeEasy", + "provider_url": "https://dnsmadeeasy.com/", + "status": "vulnerable", + "host_patterns": [ + "ns*.dnsmadeeasy.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/6" + }, + { + "name": "DNSimple", + "provider_url": "https://dnsimple.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.dnsimple.com", + "ns2.dnsimple.com", + "ns3.dnsimple.com", + "ns4.dnsimple.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/16" + }, + { + "name": "Domain.com", + "provider_url": "https://domain.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1.domain.com", + "ns2.domain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/17" + }, + { + "name": "DomainPeople", + "provider_url": "https://domainpeople.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.domainpeople.com", + "ns2.domainpeople.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/14" + }, + { + "name": "Dotster", + "provider_url": "https://dotster.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.dotster.com", + "ns2.dotster.com", + "ns1.nameresolve.com", + "ns2.nameresolve.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/18" + }, + { + "name": "Dreamhost", + "provider_url": "https://dreamhost.com/", + "status": "edge_case", + "host_patterns": [ + "ns1.dreamhost.com", + "ns2.dreamhost.com", + "ns3.dreamhost.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/40" + }, + { + "name": "EasyDNS", + "provider_url": "https://easydns.com/", + "status": "not_vulnerable", + "host_patterns": [ + "dns1.easydns.com", + "dns2.easydns.net", + "dns3.easydns.org", + "dns4.easydns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/9" + }, + { + "name": "Gandi.net", + "provider_url": "https://gandi.net/", + "status": "not_vulnerable", + "host_patterns": [ + "a.dns.gandi.net", + "b.dns.gandi.net", + "c.dns.gandi.net" + ] + }, + { + "name": "Google Cloud", + "provider_url": "https://cloud.google.com/", + "status": "edge_case", + "host_patterns": [ + "ns-cloud-*.googledomains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/2" + }, + { + "name": "Hostinger (old NS)", + "provider_url": "https://hostinger.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hostinger.com", + "ns2.hostinger.com" + ] + }, + { + "name": "Hover", + "provider_url": "https://hover.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hover.com", + "ns2.hover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/21" + }, + { + "name": "Hurricane Electric", + "provider_url": "https://dns.he.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.he.net", + "ns2.he.net", + "ns3.he.net", + "ns4.he.net", + "ns5.he.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/25" + }, + { + "name": "Linode", + "provider_url": "https://linode.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.linode.com", + "ns2.linode.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/26" + }, + { + "name": "MediaTemple (mt)", + "provider_url": "https://mediatemple.net/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mediatemple.net", + "ns2.mediatemple.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/23" + }, + { + "name": "MyDomain", + "provider_url": "https://mydomain.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mydomain.com", + "ns2.mydomain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/4" + }, + { + "name": "Name.com", + "provider_url": "https://name.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1*.name.com", + "ns2*.name.com", + "ns3*.name.com", + "ns4*.name.com" + ], + "instructions_url": "https://github.com/libertalialtd/can-i-take-over-dns/issues/8" + }, + { + "name": "Namecheap", + "provider_url": "https://namecheap.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.namecheaphosting.com", + "*.registrar-servers.com" + ] + }, + { + "name": "Network Solutions", + "provider_url": "https://networksolutions.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.worldnic.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/15" + }, + { + "name": "NS1", + "provider_url": "https://nsone.net/", + "status": "registration_closed", + "host_patterns": [ + "dns1.p*.nsone.net", + "dns2.p*.nsone.net", + "dns3.p*.nsone.net", + "dns4.p*.nsone.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/7" + }, + { + "name": "TierraNet", + "provider_url": "https://tierra.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.domaindiscover.com", + "ns2.domaindiscover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/24" + }, + { + "name": "Reg.ru", + "provider_url": "https://reg.ru/", + "status": "vulnerable", + "host_patterns": [ + "ns1.reg.ru", + "ns2.reg.ru" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/28" + }, + { + "name": "UltraDNS", + "provider_url": "https://www.home.neustar/dns-services/ultra-dns", + "status": "not_vulnerable", + "host_patterns": [ + "pdns*.ultradns.com", + "udns*.ultradns.com", + "sdns*.ultradns.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/29" + }, + { + "name": "Yahoo Small Business", + "provider_url": "https://yahoosmallbusiness.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "yns1.yahoo.com", + "yns2.yahoo.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/20" + }, + { + "name": "Activision", + "provider_url": "https://activision.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.activision.com" + ], + "private_dns": true + }, + { + "name": "Adobe", + "provider_url": "https://adobe.com/", + "status": "not_vulnerable", + "host_patterns": [ + "adobe-dns-0*.adobe.com" + ], + "private_dns": true + }, + { + "name": "Apple", + "provider_url": "https://apple.com/", + "status": "not_vulnerable", + "host_patterns": [ + "a.ns.apple.com", + "b.ns.apple.com", + "c.ns.apple.com", + "d.ns.apple.com" + ], + "private_dns": true + }, + { + "name": "Automattic", + "provider_url": "https://automattic.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.automattic.com" + ], + "private_dns": true + }, + { + "name": "Capital One", + "provider_url": "https://capitalone.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.capitalone.com" + ], + "private_dns": true + }, + { + "name": "Disney", + "provider_url": "https://disney.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.twdcns.com", + "ns*.twdcns.info", + "ns*.twdcns.co.uk" + ], + "private_dns": true + }, + { + "name": "Google", + "provider_url": "https://google.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.google.com" + ], + "private_dns": true + }, + { + "name": "Lowe's", + "provider_url": "https://lowes.com/", + "status": "not_vulnerable", + "host_patterns": [ + "authns*.lowes.com" + ], + "private_dns": true + }, + { + "name": "T-Mobile", + "provider_url": "https://tmobileus.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns10.tmobileus.com", + "ns10.tmobileus.net" + ], + "private_dns": true + } +] diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go new file mode 100644 index 0000000000..da2101e51a --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -0,0 +1,117 @@ +package detect + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/rs/zerolog" +) + +//go:embed data/*.json +var fingerprintFS embed.FS + +var ( + loadFingerprintsOnce sync.Once + loadFingerprintsErr error +) + +func LoadFingerprints(logger zerolog.Logger) error { + log := logger.With().Str("component", "fingerprint_loader").Logger() + + loadFingerprintsOnce.Do(func() { + var cname []CNAMEProviderFingerprint + var ns []NSProviderFingerprint + + if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") + loadFingerprintsErr = err + return + } + + if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") + loadFingerprintsErr = err + return + } + + if err := validateCNAMEFingerprints(cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") + loadFingerprintsErr = err + return + } + + if err := validateNSFingerprints(ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") + loadFingerprintsErr = err + return + } + + CNAMEProviderFingerprints = cname + NSProviderFingerprints = ns + + log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") + }) + + return loadFingerprintsErr +} + +func loadJSON(path string, out any) error { + b, err := fingerprintFS.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + + return nil +} + +func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("cname fingerprint[%d] missing name", i) + } + if len(fp.Cname) == 0 { + return fmt.Errorf("cname fingerprint[%d] has no cname patterns", i) + } + + for j, cname := range fp.Cname { + if strings.TrimSpace(cname) == "" { + return fmt.Errorf("cname fingerprint[%d] has empty cname pattern at index %d", i, j) + } + } + + if strings.TrimSpace(fp.Fingerprint) == "" { + return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) + } + + fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + } + + return nil +} + +func validateNSFingerprints(fingerprints []NSProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("ns fingerprint[%d] missing name", i) + } + + if len(fp.HostPatterns) == 0 { + return fmt.Errorf("ns fingerprint[%d] has no host patterns", i) + } + + for j, pattern := range fp.HostPatterns { + if strings.TrimSpace(pattern) == "" { + return fmt.Errorf("ns fingerprint[%d] has empty host pattern at index %d", i, j) + } + } + } + + return nil +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go new file mode 100644 index 0000000000..2ecc9981f8 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -0,0 +1,16 @@ +package detect + +type FingerprintSource interface { + CNAME() []CNAMEProviderFingerprint + NS() []NSProviderFingerprint +} + +type GlobalFingerprintSource struct{} + +func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { + return CNAMEProviderFingerprints +} + +func (GlobalFingerprintSource) NS() []NSProviderFingerprint { + return NSProviderFingerprints +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go new file mode 100644 index 0000000000..d1491995cd --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -0,0 +1,108 @@ +package detect + +import "strings" + +type CNAMEProviderFingerprint struct { + Cname []string `json:"cname"` + Name string `json:"name"` + Nxdomain bool `json:"nxdomain"` + Fingerprint string `json:"fingerprint"` + Mode FingerprintMode `json:"mode,omitempty"` +} + +type FingerprintMode string + +const ( + FingerprintModeLiteral FingerprintMode = "literal" + FingerprintModeRegex FingerprintMode = "regex" +) + +var CNAMEProviderFingerprints []CNAMEProviderFingerprint +var NSProviderFingerprints []NSProviderFingerprint + +type NSProviderStatus string + +const ( + NSStatusVulnerable NSProviderStatus = "vulnerable" + NSStatusNotVulnerable NSProviderStatus = "not_vulnerable" + NSStatusEdgeCase NSProviderStatus = "edge_case" + NSStatusVulnerableWithPurchase NSProviderStatus = "vulnerable_with_purchase" + NSStatusRegistrationClosed NSProviderStatus = "registration_closed" +) + +type NSProviderFingerprint struct { + Name string `json:"name"` + ProviderURL string `json:"provider_url"` + Status NSProviderStatus `json:"status"` + HostPatterns []string `json:"host_patterns"` + InstructionsURL string `json:"instructions_url,omitempty"` + PrivateDNS bool `json:"private_dns,omitempty"` +} + +func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, pattern := range f.HostPatterns { + if wildcardHostMatch(pattern, host) { + return true + } + } + return false +} + +func wildcardHostMatch(pattern, host string) bool { + pattern = strings.ToLower(strings.TrimSuffix(pattern, ".")) + if !strings.Contains(pattern, "*") { + return host == pattern + } + + parts := strings.Split(pattern, "*") + if len(parts) == 2 { + return strings.HasPrefix(host, parts[0]) && strings.HasSuffix(host, parts[1]) + } + + idx := 0 + for i, part := range parts { + if part == "" { + continue + } + + pos := strings.Index(host[idx:], part) + if pos < 0 { + return false + } + + if i == 0 && !strings.HasPrefix(host, part) { + return false + } + + idx += pos + len(part) + } + + last := parts[len(parts)-1] + if last != "" && !strings.HasSuffix(host, last) { + return false + } + + return true +} + +func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { + for _, cname := range f.Cname { + if strings.HasSuffix(target, cname) { + return true + } + } + return false +} + +func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { + if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { + return mode + } + + if strings.Contains(fingerprint, ".*") || strings.Contains(fingerprint, "\\") { + return FingerprintModeRegex + } + + return FingerprintModeLiteral +} diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go new file mode 100644 index 0000000000..c3d4a30293 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger.go @@ -0,0 +1,9 @@ +package detect + +import "github.com/rs/zerolog" + +var detectLogger = zerolog.Nop() + +func SetLogger(logger zerolog.Logger) { + detectLogger = logger.With().Str("component", "detect").Logger() +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go new file mode 100644 index 0000000000..78ddb2f428 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -0,0 +1,42 @@ +package detect + +import ( + "slices" + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type NSEvidence struct { + Domain string + NSHosts []string + NSDelegations model.NsDelegations + // Registrar model.RegistrarContext // if/when added +} + +func ExtractNSEvidence(input model.ScanResults) *NSEvidence { + nsDelegations := input.NsDelegations + if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { + return nil + } + + return &NSEvidence{ + Domain: *input.Domain, + NSHosts: parseHostnames(input.NsDelegations.Hosts), + NSDelegations: *input.NsDelegations, + } +} + +func parseHostnames(hosts []string) []string { + normalizedHosts := make([]string, 0, len(hosts)) + for _, host := range hosts { + normalizedHosts = append(normalizedHosts, normalizeHost(host)) + } + + slices.Sort(normalizedHosts) + return slices.Compact(normalizedHosts) +} + +func normalizeHost(host string) string { + return strings.Trim(strings.ToLower(host), ".") +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go new file mode 100644 index 0000000000..953cee5580 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -0,0 +1,47 @@ +package detect + +import "strings" + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly + } +} + +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go new file mode 100644 index 0000000000..5485e9cc95 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -0,0 +1,71 @@ +package detect + +type NSHit struct { + Matched bool + Host string + Provider string + ReasonCode ReasonCode +} + +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + rank := nsReasonRank(reasonCode) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", host). + Str("provider", fp.Name). + Str("provider_status", string(fp.Status)). + Str("lame_type", lameType). + Str("reason_code", string(reasonCode)). + Int("rank", rank). + Bool("emittable", hit.Matched). + Msg("ns candidate evaluated") + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + if best == nil { + detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + return nil + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", best.Host). + Str("provider", best.Provider). + Str("reason_code", string(best.ReasonCode)). + Int("rank", nsReasonRank(best.ReasonCode)). + Bool("emittable", best.Matched). + Msg("ns best candidate selected") + + return best +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go new file mode 100644 index 0000000000..f080b18329 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -0,0 +1,15 @@ +package detect + +type ReasonCode string + +const ( + ReasonCNAMEDanglingNXDOMAIN ReasonCode = "CNAME_DANGLING_NXDOMAIN" + ReasonCNAMEProviderFingerprintBodyMatch ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + ReasonCNAMETargetMatchMissingNXDOMAIN ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" + ReasonCNAMETargetMatchMissingBodyFP ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + + ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" + ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" + ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" +) diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go new file mode 100644 index 0000000000..593ca1faff --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -0,0 +1,21 @@ +package messaging + +import ( + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +func CheckConnection(nc *nats.Conn) error { + if nc == nil { + return errors.New("nats connection is nil") + } + if nc.IsClosed() { + return errors.New("nats connection is closed") + } + if !nc.IsConnected() { + return fmt.Errorf("nats not connected (status=%s)", nc.Status().String()) + } + return nil +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go new file mode 100644 index 0000000000..5626f45983 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -0,0 +1,58 @@ +package messaging + +import ( + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Publisher struct { + logger zerolog.Logger + js jetstream.JetStream + subject string +} + +func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} +} + +func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { + payload, err := json.Marshal(finding) + if err != nil { + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Msg("marshal finding failed") + return err + } + + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("publish failed") + return err + } + + p.logger.Debug(). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("finding published") + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go new file mode 100644 index 0000000000..66e98a2f35 --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -0,0 +1,19 @@ +package model + +type RecordType string + +const ( + RecordTypeCNAME RecordType = "CNAME" + RecordTypeNS RecordType = "NS" +) + +type Finding struct { + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType RecordType `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go new file mode 100644 index 0000000000..263e74928d --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -0,0 +1,55 @@ +package model + +type Input struct { + DomainKey string `json:"domain_key"` + Results ScanResults `json:"results"` +} + +type ScanResults struct { + Domain *string `json:"domain"` + NsDelegations *NsDelegations `json:"ns_delegations"` + RegistrarContext *RegistrarContext `json:"registrar_context"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords *NsRecords `json:"ns_records"` +} + +type NsRecords struct { + Hostnames []string `json:"hostnames"` + Warnings []string `json:"warnings"` + Error string `json:"error"` +} + +type NsDelegations struct { + Hosts []string `json:"ns_hosts"` + Checks []NsCheck `json:"ns_checks"` + Delegation Delegation `json:"ns_delegation"` + Error string `json:"error"` +} + +type NsCheck struct { + Host string `json:"ns_host"` + Qname string `json:"qname"` + Qtype string `json:"qtype"` + Rcode string `json:"rcode"` + AnsweredAuthoritatively bool `json:"answered_authoritatively"` + Error *string `json:"error"` + Timeout bool `json:"timeout"` +} +type Delegation struct { + TotalHosts int `json:"total_ns"` + OkCount int `json:"authoritative_ok"` + LameCount int `json:"lame_count"` + LameType string `json:"lame_type"` +} + +type RegistrarContext struct { + BaseDomain string `json:"base_domain"` + LookupSuccess bool `json:"lookup_success"` + RDAPURL string `json:"rdap_url"` + RegistrarName string `json:"registrar_name"` + RegistrarID string `json:"registrar_id"` + RDAPNameservers []string `json:"rdap_nameservers"` + DelegationMatchesRDAP *bool `json:"delegation_matches_rdap"` + Error string `json:"error"` +}