From 85310ada9dfd999ee085aa2b827d9545cbd38deb Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 08:48:24 -0300 Subject: [PATCH 01/45] init subdomain-takeover + NATS skeleton --- scanners/subdomain-takeover/.env.example | 7 + scanners/subdomain-takeover/Dockerfile | 22 ++ scanners/subdomain-takeover/README.md | 137 ++++++++++++ .../subdomain-takeover/cmd/service/main.go | 92 ++++++++ scanners/subdomain-takeover/config.go | 37 ++++ .../fixtures/cyber-gc-ca_dns_scan.json | 202 ++++++++++++++++++ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 ++ scanners/subdomain-takeover/go.mod | 19 ++ scanners/subdomain-takeover/go.sum | 21 ++ scanners/subdomain-takeover/model/finding.go | 0 scanners/subdomain-takeover/model/types.go | 0 scanners/subdomain-takeover/nats/consumer.go | 0 scanners/subdomain-takeover/nats/publisher.go | 0 .../subdomain-takeover/scanner/confidence.go | 0 .../subdomain-takeover/scanner/dns_checks.go | 0 .../subdomain-takeover/scanner/scanner.go | 0 16 files changed, 560 insertions(+) create mode 100644 scanners/subdomain-takeover/.env.example create mode 100644 scanners/subdomain-takeover/Dockerfile create mode 100644 scanners/subdomain-takeover/README.md create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/config.go create mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/go.mod create mode 100644 scanners/subdomain-takeover/go.sum create mode 100644 scanners/subdomain-takeover/model/finding.go create mode 100644 scanners/subdomain-takeover/model/types.go create mode 100644 scanners/subdomain-takeover/nats/consumer.go create mode 100644 scanners/subdomain-takeover/nats/publisher.go create mode 100644 scanners/subdomain-takeover/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/scanner/scanner.go diff --git a/scanners/subdomain-takeover/.env.example b/scanners/subdomain-takeover/.env.example new file mode 100644 index 0000000000..31d2f06813 --- /dev/null +++ b/scanners/subdomain-takeover/.env.example @@ -0,0 +1,7 @@ +NATS_URL= +NATS_STREAM= +SUBJECT_IN= +SUBJECT_OUT= +DURABLE_NAME= +WORKER_COUNT= +LOG_LEVEL= diff --git a/scanners/subdomain-takeover/Dockerfile b/scanners/subdomain-takeover/Dockerfile new file mode 100644 index 0000000000..6c7f3eafaa --- /dev/null +++ b/scanners/subdomain-takeover/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache git ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/subdomain-takeover ./cmd/service + +# Runtime stage +FROM alpine:3.22 + +RUN addgroup -S scanner && adduser -S scanner -G scanner +WORKDIR /app +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /out/subdomain-takeover /app/subdomain-takeover + +USER scanner +ENTRYPOINT ["/app/subdomain-takeover"] diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md new file mode 100644 index 0000000000..0bfcdaf9ba --- /dev/null +++ b/scanners/subdomain-takeover/README.md @@ -0,0 +1,137 @@ +# subdomain-takeover-service + +Go microservice that consumes DNS scan results and emits normalized subdomain takeover findings. + +## What it does + +1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) +2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +3. Assigns confidence (`suspected`, `probable`, `confirmed`) +4. Publishes findings to `scans.findings.upsert` + +## Passive-only detection policy + +This service uses passive checks only. +It does not attempt account takeover, resource claiming, or any active validation. + +As a result, confidence is conservative and evidence-driven: +- `suspected`: weak or incomplete takeover indicators +- `probable`: strong passive indicators with known exploitable conditions +- `confirmed`: reserved for deterministic passive signatures only + +## Sitting Duck DNS hijack coverage + +The scanner includes passive detection for Sitting Duck style DNS hijack risk. + +A Sitting Duck takeover is considered possible when all of the following are true: + +1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. +2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). +3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. + +### Variations covered + +- Partially lame delegation: some NS records are lame, others still answer. +- Full lame delegation: all authoritative NS are lame. +- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. + +If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. + +## Confidence model + +This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. + +- `suspected` + - dangling/suspicious DNS pattern only, or + - lame delegation evidence without known exploitable-provider confirmation +- `probable` + - DNS pattern + known provider unclaimed signature, or + - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match +- `confirmed` + - explicit passive takeover indicator with deterministic evidence + - if deterministic passive proof is missing, cap at `probable` + +## Required evidence for findings + +Each finding should include: +- `domain` +- `domain_key` +- `record_type` (`CNAME` or `NS`) +- `target` or `ns_host` +- `provider` +- `lame_type` (`partial` or `full`) for Sitting Duck findings +- per-NS resolver outcomes where applicable +- `confidence` +- `reason_code` +- remediation guidance + +## Remediation guidance + +Typical remediation: +1. Fix delegation at registrar and authoritative DNS provider. +2. Remove stale or dead NS entries. +3. Reclaim or securely rebind abandoned DNS zones or accounts. +4. Validate ownership controls for DNS provider account workflows. + +## Prerequisites + +- Go 1.24+ +- NATS with JetStream enabled +- `nats` CLI (optional, for local testing) + +## Quick start + +```bash +go mod tidy +go run ./cmd/service +``` + +Environment variables: +- `NATS_URL` (default: `nats://localhost:4222`) +- `NATS_STREAM` (default: `SCANS`) +- `SUBJECT_IN` (default: `scans.dns_scanner_results`) +- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `DURABLE_NAME` (default: `subdomain_takeover`) +- `WORKER_COUNT` (default: `5`) +- `LOG_LEVEL` (default: `info`) + +Local test flow: + +Publish a test request: +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' +``` + +Watch findings: +```bash +nats sub "scans.findings.upsert" +``` + +## Development milestones + +1. Build config + NATS consume/publish loop. +2. Add DNS evidence extraction. +3. Add provider fingerprint rule engine. +4. Add confidence scoring and remediation text. +5. Add tests for classification rules and message handling. + +--- + +## External Go modules to install + +Start minimal: + +```bash +go get github.com/nats-io/nats.go +go get github.com/miekg/dns +go get github.com/rs/zerolog +go get golang.org/x/sync/errgroup +go get github.com/stretchr/testify +``` + +What each is for: +- `nats.go`: JetStream consume/publish +- `miekg/dns`: robust DNS lookups and record parsing +- `zerolog`: structured logs without heavy setup +- `errgroup`: clean goroutine lifecycle management +- `testify`: easier assertions in unit tests diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..175a930507 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + }() + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break + } + + select { + case <-ctx.Done(): + return + default: + continue + } + + // sem <- struct{}{} + // go func() { + // defer func() { + // <-sem + // }() + // msg, err := iter.Next() + // if err != nil { + // logger.Error().Msgf("next err:", err.Error()) + // continue + // } + // fmt.Printf("Processing msg: %s\n", string(msg.Data())) + // // doWork() + // msg.Ack() + // }() + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/config.go new file mode 100644 index 0000000000..b14c80710a --- /dev/null +++ b/scanners/subdomain-takeover/config.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + + "github.com/kelseyhightower/envconfig" + "github.com/rs/zerolog" +) + +type Config struct { + NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` + WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` + LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` + Logger zerolog.Logger +} + +func initConfig() *Config { + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + panic(err) + } + + zerolog.SetGlobalLevel(cfg.LogLevel) + + // Create multiple output steams for zerolog + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + + logger := zerolog.New(multi).With().Timestamp().Logger() + + cfg.Logger = logger + + return &cfg +} diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..cad720cc93 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json @@ -0,0 +1,202 @@ +{ + "domain": "cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + [ + "cyber.gc.ca. 7161 IN A 20.151.96.73" + ] + ], + "resolve_ips": [ + "20.151.96.73" + ], + "cname_record": null, + "mx_records": { + "hosts": [ + { + "preference": 10, + "hostname": "cyber-gc-ca.mail.protection.outlook.com", + "addresses": [ + "52.101.190.1", + "52.101.192.0", + "52.101.192.1", + "52.101.190.0" + ] + } + ], + "warnings": [] + }, + "ns_records": { + "hostnames": [ + "ns1.d-zone.ca", + "ns2.d-zone.ca", + "ns40.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns10.ent.global.gc.ca" + ], + "warnings": [] + }, + "dkim": {}, + "spf": { + "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", + "valid": true, + "dns_void_lookups": 0, + "warnings": [], + "parsed": { + "pass": [ + { + "value": "40.80.253.57", + "mechanism": "ip4" + }, + { + "value": "52.139.3.76", + "mechanism": "a" + }, + { + "value": "205.193.218.114", + "mechanism": "a" + }, + { + "value": "205.193.218.115", + "mechanism": "a" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [ + { + "domain": "spf.protection.outlook.com", + "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", + "dns_void_lookups": 0, + "parsed": { + "pass": [ + { + "value": "40.92.0.0/15", + "mechanism": "ip4" + }, + { + "value": "40.107.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.100.0.0/15", + "mechanism": "ip4" + }, + { + "value": "52.102.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.103.0.0/17", + "mechanism": "ip4" + }, + { + "value": "104.47.0.0/17", + "mechanism": "ip4" + }, + { + "value": "2a01:111:f400::/48", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403::/49", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:8000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:c000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:f000::/52", + "mechanism": "ip6" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "warnings": [], + "spf_default": "fail", + "lookups": 0 + } + ], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "spf_default": "fail", + "lookups": 3 + }, + "dmarc": { + "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", + "valid": true, + "location": "cyber.gc.ca", + "warnings": [], + "tags": { + "v": { + "value": "DMARC1", + "explicit": true + }, + "p": { + "value": "reject", + "explicit": true + }, + "pct": { + "value": 100, + "explicit": true + }, + "rua": { + "value": [ + { + "scheme": "mailto", + "address": "dmarc@cyber.gc.ca", + "size_limit": null, + "accepting": true + } + ], + "explicit": true + }, + "adkim": { + "value": "r", + "explicit": false + }, + "aspf": { + "value": "r", + "explicit": false + }, + "fo": { + "value": "0", + "explicit": false + }, + "rf": { + "value": "afrf", + "explicit": false + }, + "ri": { + "value": 86400, + "explicit": false + }, + "sp": { + "value": "reject", + "explicit": false + } + }, + "effective_policy_source": "p", + "effective_policy": "reject" + }, + "wildcard_sibling": false, + "wildcard_entry": false, + "zone_dnssec_enabled": null +} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..8f546a74a5 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json @@ -0,0 +1,23 @@ +{ + "domain": "www.cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "www.cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], + ["cyber.gc.ca. 5550 IN A 20.151.96.73"] + ], + "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", + "ns_records": { + "hostnames": [ + "ns40.ent.global.gc.ca", + "ns10.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns2.d-zone.ca", + "ns1.d-zone.ca" + ], + "warnings": [] + } +} diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod new file mode 100644 index 0000000000..f4bfb6b567 --- /dev/null +++ b/scanners/subdomain-takeover/go.mod @@ -0,0 +1,19 @@ +module github.com/canada-ca/tracker/scanners/subdomain-takeover + +go 1.25.0 + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/klauspost/compress v1.18.5 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum new file mode 100644 index 0000000000..470146821e --- /dev/null +++ b/scanners/subdomain-takeover/go.sum @@ -0,0 +1,21 @@ +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/nats/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/nats/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 From 794cc25ac7db66544e6638fc2e2b00ff058312ae Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:22:07 -0300 Subject: [PATCH 02/45] finish NATS consumer logic --- scanners/subdomain-takeover/main.go | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go new file mode 100644 index 0000000000..102e03b573 --- /dev/null +++ b/scanners/subdomain-takeover/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err) + } + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + +Loop: + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break Loop + } + + select { + case <-ctx.Done(): + break Loop + default: + } + + msg, err := iter.Next() + if err != nil { + if ctx.Err() != nil { + break Loop + } + logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + fmt.Println(string(m.Data())) + m.Ack() + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} From c64e5e4c69eee06506c3e02a46e8f5fc4ede82fc Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:24:14 -0300 Subject: [PATCH 03/45] move main.go to top dir --- .../subdomain-takeover/cmd/service/main.go | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 scanners/subdomain-takeover/cmd/service/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go deleted file mode 100644 index 175a930507..0000000000 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - }() - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break - } - - select { - case <-ctx.Done(): - return - default: - continue - } - - // sem <- struct{}{} - // go func() { - // defer func() { - // <-sem - // }() - // msg, err := iter.Next() - // if err != nil { - // logger.Error().Msgf("next err:", err.Error()) - // continue - // } - // fmt.Printf("Processing msg: %s\n", string(msg.Data())) - // // doWork() - // msg.Ack() - // }() - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From bb6da4966a0792dafcb40673908d72bf8660a586 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 16:24:41 -0300 Subject: [PATCH 04/45] refactor main loop into app/runner.go --- .../subdomain-takeover/cmd/service/main.go | 50 ++++++++ .../subdomain-takeover/internal/app/runner.go | 89 +++++++++++++++ .../subdomain-takeover/internal/app/worker.go | 1 + .../{ => internal/config}/config.go | 4 +- .../detect/classifier.go} | 0 .../detect}/confidence.go | 0 .../types.go => internal/detect/evidence.go} | 0 .../detect/remediation.go} | 0 .../publisher.go => internal/detect/rules.go} | 0 .../logging/logger.go} | 0 .../messaging/client.go} | 0 .../internal/messaging/consumer.go | 0 .../internal/messaging/publisher.go | 0 .../internal/model/finding.go | 0 .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 scanners/subdomain-takeover/main.go | 107 ------------------ 19 files changed, 142 insertions(+), 109 deletions(-) create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/internal/app/runner.go create mode 100644 scanners/subdomain-takeover/internal/app/worker.go rename scanners/subdomain-takeover/{ => internal/config}/config.go (95%) rename scanners/subdomain-takeover/{model/finding.go => internal/detect/classifier.go} (100%) rename scanners/subdomain-takeover/{scanner => internal/detect}/confidence.go (100%) rename scanners/subdomain-takeover/{model/types.go => internal/detect/evidence.go} (100%) rename scanners/subdomain-takeover/{nats/consumer.go => internal/detect/remediation.go} (100%) rename scanners/subdomain-takeover/{nats/publisher.go => internal/detect/rules.go} (100%) rename scanners/subdomain-takeover/{scanner/dns_checks.go => internal/logging/logger.go} (100%) rename scanners/subdomain-takeover/{scanner/scanner.go => internal/messaging/client.go} (100%) create mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher.go create mode 100644 scanners/subdomain-takeover/internal/model/finding.go create mode 100644 scanners/subdomain-takeover/internal/model/types.go create mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go delete mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..4fafed7523 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + cfg := config.InitConfig() + logger := cfg.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + nc, _ := nats.Connect(cfg.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NatsStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err).Msg("Error creating message iterator") + } + + app.Run( + app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Cancel: cancel, + Context: ctx, + }) + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go new file mode 100644 index 0000000000..f02d48800d --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -0,0 +1,89 @@ +package app + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RunnerDeps struct { + Logger zerolog.Logger + WorkerCount int + Iter jetstream.MessagesContext + Cancel context.CancelFunc + Context context.Context +} + +func Run(deps RunnerDeps) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, deps.WorkerCount) + + go func() { + <-sig + deps.Logger.Info().Msg("Shutdown requested...") + deps.Cancel() + deps.Iter.Stop() + }() + +Loop: + for { + // if nc.IsClosed() { + // deps.Logger.Error().Msg("Connection to NATS is closed.") + // break Loop + // } + + select { + case <-deps.Context.Done(): + break Loop + default: + } + + msg, err := deps.Iter.Next() + if err != nil { + if deps.Context.Err() != nil { + break Loop + } + deps.Logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-deps.Context.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + deps.Logger.Info().Msg(string(m.Data())) + err := m.Ack() + if err != nil { + deps.Logger.Err(err).Msg("Ack error") + } + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + deps.Logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + deps.Logger.Info().Msg("drain timeout") + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go new file mode 100644 index 0000000000..4879f7a483 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -0,0 +1 @@ +package app diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/internal/config/config.go similarity index 95% rename from scanners/subdomain-takeover/config.go rename to scanners/subdomain-takeover/internal/config/config.go index b14c80710a..28d2ddf824 100644 --- a/scanners/subdomain-takeover/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,4 +1,4 @@ -package main +package config import ( "os" @@ -18,7 +18,7 @@ type Config struct { Logger zerolog.Logger } -func initConfig() *Config { +func InitConfig() *Config { var cfg Config if err := envconfig.Process("", &cfg); err != nil { panic(err) diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/internal/detect/classifier.go similarity index 100% rename from scanners/subdomain-takeover/model/finding.go rename to scanners/subdomain-takeover/internal/detect/classifier.go diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go similarity index 100% rename from scanners/subdomain-takeover/scanner/confidence.go rename to scanners/subdomain-takeover/internal/detect/confidence.go diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/internal/detect/evidence.go similarity index 100% rename from scanners/subdomain-takeover/model/types.go rename to scanners/subdomain-takeover/internal/detect/evidence.go diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/internal/detect/remediation.go similarity index 100% rename from scanners/subdomain-takeover/nats/consumer.go rename to scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/internal/detect/rules.go similarity index 100% rename from scanners/subdomain-takeover/nats/publisher.go rename to scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/logging/logger.go similarity index 100% rename from scanners/subdomain-takeover/scanner/dns_checks.go rename to scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/internal/messaging/client.go similarity index 100% rename from scanners/subdomain-takeover/scanner/scanner.go rename to scanners/subdomain-takeover/internal/messaging/client.go diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go deleted file mode 100644 index 102e03b573..0000000000 --- a/scanners/subdomain-takeover/main.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err) - } - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - iter.Stop() - }() - -Loop: - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break Loop - } - - select { - case <-ctx.Done(): - break Loop - default: - } - - msg, err := iter.Next() - if err != nil { - if ctx.Err() != nil { - break Loop - } - logger.Debug().Err(err).Msg("next returned, continuing") - continue - } - - select { - case sem <- struct{}{}: - case <-ctx.Done(): - break Loop - } - - wg.Add(1) - go func(m jetstream.Msg) { - defer wg.Done() - defer func() { <-sem }() - fmt.Println(string(m.Data())) - m.Ack() - }(msg) - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From ba669c435966f36a0cf933a2ea020703e4f6b291 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:00:18 -0300 Subject: [PATCH 05/45] init worker and publisher objects --- .../subdomain-takeover/cmd/service/main.go | 60 ++++++++++++++----- .../subdomain-takeover/internal/app/runner.go | 45 ++++++-------- .../subdomain-takeover/internal/app/worker.go | 50 ++++++++++++++++ .../internal/messaging/client.go | 0 .../internal/messaging/consumer.go | 1 + .../internal/messaging/publisher.go | 36 +++++++++++ .../internal/model/finding.go | 13 ++++ .../internal/model/input.go | 17 ++++++ .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 12 files changed, 180 insertions(+), 42 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/client.go create mode 100644 scanners/subdomain-takeover/internal/model/input.go delete mode 100644 scanners/subdomain-takeover/internal/model/types.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 4fafed7523..5275a170f2 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -2,47 +2,79 @@ package main import ( "context" + "os" + "os/signal" + "syscall" "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) +func checkErr(err error, log zerolog.Logger, msg string, exit bool) { + if err != nil { + log.Err(err).Msg(msg) + if exit { + os.Exit(1) + } + } + +} + func main() { cfg := config.InitConfig() logger := cfg.Logger ctx, cancel := context.WithCancel(context.Background()) - nc, _ := nats.Connect(cfg.NatsUrl) - js, _ := jetstream.New(nc) + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + nc, err := nats.Connect(cfg.NatsUrl) + checkErr(err, logger, "", true) + + js, err := jetstream.New(nc) + checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NatsStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, }) + checkErr(err, logger, "", true) - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, }) + checkErr(err, logger, "", true) iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err).Msg("Error creating message iterator") + checkErr(err, logger, "", true) + + pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) + worker := app.NewWorker(logger, *pub) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + + deps := app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Worker: *worker, } - app.Run( - app.RunnerDeps{ - Logger: logger, - WorkerCount: cfg.WorkerCount, - Iter: iter, - Cancel: cancel, - Context: ctx, - }) + app.Run(ctx, deps) logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) nc.Flush() diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index f02d48800d..a41650355d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -2,10 +2,7 @@ package app import ( "context" - "os" - "os/signal" "sync" - "syscall" "time" "github.com/nats-io/nats.go/jetstream" @@ -16,49 +13,45 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Cancel context.CancelFunc - Context context.Context + Worker Worker } -func Run(deps RunnerDeps) { - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) +func Run(ctx context.Context, deps RunnerDeps) { + logger := deps.Logger + iter := deps.Iter + + if deps.WorkerCount < 1 { + deps.WorkerCount = 1 + } var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) - go func() { - <-sig - deps.Logger.Info().Msg("Shutdown requested...") - deps.Cancel() - deps.Iter.Stop() - }() - Loop: for { // if nc.IsClosed() { - // deps.Logger.Error().Msg("Connection to NATS is closed.") + // logger.Error().Msg("Connection to NATS is closed.") // break Loop // } select { - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop default: } - msg, err := deps.Iter.Next() + msg, err := iter.Next() if err != nil { - if deps.Context.Err() != nil { + if ctx.Err() != nil { break Loop } - deps.Logger.Debug().Err(err).Msg("next returned, continuing") + logger.Debug().Err(err).Msg("next returned, continuing") continue } select { case sem <- struct{}{}: - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop } @@ -66,11 +59,7 @@ Loop: go func(m jetstream.Msg) { defer wg.Done() defer func() { <-sem }() - deps.Logger.Info().Msg(string(m.Data())) - err := m.Ack() - if err != nil { - deps.Logger.Err(err).Msg("Ack error") - } + deps.Worker.Handle(ctx, m) }(msg) } @@ -82,8 +71,8 @@ Loop: select { case <-waitDone: - deps.Logger.Info().Msg("all workers drained") + logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - deps.Logger.Info().Msg("drain timeout") + logger.Info().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 4879f7a483..44e5452724 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -1 +1,51 @@ package app + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Worker struct { + logger zerolog.Logger + pub messaging.Publisher +} + +func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { + return &Worker{logger: logger, pub: pub} +} + +func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { + log := w.logger.With().Str("component", "worker").Logger() + // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) + if err != nil { + log.Err(err).Msg("Decoding error") + } + log.Info().Msg(scan.Domain) + + // var finding model.Finding + // err := w.pub.Publish(ctx, finding) + // if err != nil { + // return err + // } + + if err := msg.Ack(); err != nil { + log.Err(err).Msg("Ack error") + return err + } + + return nil +} + +func decodeScan(data []byte) (model.Input, error) { + var scan model.Input + trimmed := bytes.Trim(data, "\n") + err := json.Unmarshal(trimmed, &scan) + return scan, err +} diff --git a/scanners/subdomain-takeover/internal/messaging/client.go b/scanners/subdomain-takeover/internal/messaging/client.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go index e69de29bb2..cbb7a43f8b 100644 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ b/scanners/subdomain-takeover/internal/messaging/consumer.go @@ -0,0 +1 @@ +package messaging diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index e69de29bb2..5b5585a75f 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -0,0 +1,36 @@ +package messaging + +import ( + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Publisher struct { + logger zerolog.Logger + js jetstream.JetStream + subject string +} + +func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} +} + +func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { + payload, err := json.Marshal(finding) + if err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + return err + } + + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + return err + } + + p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index e69de29bb2..5d387c0f1d 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -0,0 +1,13 @@ +package model + +type Finding struct { + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType string `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` + Remediation string `json:"remediation"` +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go new file mode 100644 index 0000000000..8d52ec9b96 --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -0,0 +1,17 @@ +package model + +type Input struct { + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + RCode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord string `json:"cname_record"` + NSRecords NSRecords `json:"ns_records"` +} + +type NSRecords struct { + Hostnames []string `json:"hostnames"` + Warnings []string `json:"warnings"` +} diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go deleted file mode 100644 index e69de29bb2..0000000000 From b4e9b8cbbaed8c62e95d1dd6d0f6858fdb932ca9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:14:17 -0300 Subject: [PATCH 06/45] add health check at start of run loop --- .../subdomain-takeover/cmd/service/main.go | 1 + .../subdomain-takeover/internal/app/runner.go | 11 ++++++---- .../internal/messaging/health.go | 21 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/messaging/health.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 5275a170f2..66085ffffa 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -72,6 +72,7 @@ func main() { WorkerCount: cfg.WorkerCount, Iter: iter, Worker: *worker, + NC: nc, } app.Run(ctx, deps) diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index a41650355d..42539405f0 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) @@ -14,6 +16,7 @@ type RunnerDeps struct { WorkerCount int Iter jetstream.MessagesContext Worker Worker + NC *nats.Conn } func Run(ctx context.Context, deps RunnerDeps) { @@ -29,10 +32,10 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - // if nc.IsClosed() { - // logger.Error().Msg("Connection to NATS is closed.") - // break Loop - // } + if err := messaging.CheckConnection(deps.NC); err != nil { + logger.Error().Err(err).Msg("NATS connection unhealthy") + break Loop + } select { case <-ctx.Done(): diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go new file mode 100644 index 0000000000..593ca1faff --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -0,0 +1,21 @@ +package messaging + +import ( + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +func CheckConnection(nc *nats.Conn) error { + if nc == nil { + return errors.New("nats connection is nil") + } + if nc.IsClosed() { + return errors.New("nats connection is closed") + } + if !nc.IsConnected() { + return fmt.Errorf("nats not connected (status=%s)", nc.Status().String()) + } + return nil +} From 4afcd60c5b261f23506f60c6de112ca7b6261e10 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 3 Jul 2026 11:58:39 -0300 Subject: [PATCH 07/45] add init logic for cname takeover scanning --- .../subdomain-takeover/internal/app/worker.go | 3 + .../internal/detect/classifier.go | 1 + .../internal/detect/confidence.go | 1 + .../internal/detect/evidence.go | 31 +++ .../internal/detect/fingerprints.go | 247 ++++++++++++++++++ .../internal/detect/remediation.go | 1 + .../internal/detect/rules.go | 29 ++ .../internal/model/input.go | 8 +- 8 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprints.go diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 44e5452724..7d000378e7 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" @@ -29,6 +30,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { } log.Info().Msg(scan.Domain) + detect.GetEvidence(scan) + // var finding model.Finding // err := w.pub.Publish(ctx, finding) // if err != nil { diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index e69de29bb2..c835b7444b 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetEvidence(input model.Input) []model.Finding { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cname := parseCname(*input.CnameRecord) + cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) + evidence = append(evidence, *nsEvidence) + } + + return evidence +} + +func parseCname(record string) string { + record_tokens := strings.Split(strings.Trim(record, "."), " ") + return record_tokens[len(record_tokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go new file mode 100644 index 0000000000..5d13a5489f --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -0,0 +1,247 @@ +package detect + +import "strings" + +type Fingerprint struct { + Cname []string + Name string + Nxdomain bool + Fingerprint string +} + +func (f *Fingerprint) ContainsCname(target string) bool { + for _, cname := range f.Cname { + if strings.HasSuffix(target, cname) { + return true + } + } + return false +} + +var Fingerprints = []Fingerprint{ + { + Cname: []string{ + "elasticbeanstalk.com", + }, + Nxdomain: true, + Name: "AWS/Elastic Beanstalk", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "s3.amazonaws.com", + }, + Nxdomain: false, + Name: "AWS/S3", + Fingerprint: "The specified bucket does not exist", + }, + { + Cname: []string{ + "agilecrm.com", + }, + Nxdomain: false, + Name: "Agile CRM", + Fingerprint: "Sorry, this page is no longer available.", + }, + { + Cname: []string{ + "airee.ru", + }, + Nxdomain: false, + Name: "Airee.ru", + Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", + }, + { + Cname: []string{ + "animaapp.io", + }, + Nxdomain: false, + Name: "Anima", + Fingerprint: "The page you were looking for does not exist.", + }, + { + Cname: []string{ + "bitbucket.io", + }, + Nxdomain: false, + Name: "Bitbucket", + Fingerprint: "Repository not found", + }, + { + Cname: []string{ + "trydiscourse.com", + }, + Nxdomain: true, + Name: "Discourse", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "furyns.com", + }, + Nxdomain: false, + Name: "Gemfury", + Fingerprint: "404: This page could not be found.", + }, + { + Cname: []string{ + "ghost.io", + }, + Nxdomain: false, + Name: "Ghost", + Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", + }, + { + Cname: []string{ + "hatenablog.com", + }, + Nxdomain: false, + Name: "HatenaBlog", + Fingerprint: "404 Blog is not found", + }, + { + Cname: []string{ + "helpjuice.com", + }, + Nxdomain: false, + Name: "Help Juice", + Fingerprint: "We could not find what you're looking for.", + }, + { + Cname: []string{ + "helpscoutdocs.com", + }, + Nxdomain: false, + Name: "Help Scout", + Fingerprint: "No settings were found for this company:", + }, + { + Cname: []string{ + "helprace.com", + }, + Nxdomain: false, + Name: "Helprace", + Fingerprint: "HTTP_STATUS=301", + }, + { + Cname: []string{ + "youtrack.cloud", + }, + Nxdomain: false, + Name: "JetBrains", + Fingerprint: "is not a registered InCloud YouTrack", + }, + { + Cname: []string{ + "launchrock.com", + }, + Nxdomain: false, + Name: "LaunchRock", + Fingerprint: "HTTP_STATUS=500", + }, + { + Cname: []string{ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com", + }, + Nxdomain: true, + Name: "Microsoft Azure", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "ngrok.io", + }, + Nxdomain: false, + Name: "Ngrok", + Fingerprint: "Tunnel .*.ngrok.io not found", + }, + { + Cname: []string{ + "readme.io", + }, + Nxdomain: false, + Name: "Readme.io", + Fingerprint: "The creators of this project are still working on making everything perfect!", + }, + { + Cname: []string{ + "52.16.160.97", + }, + Nxdomain: false, + Name: "SmartJobBoard", + Fingerprint: "This job board website is either expired or its domain name is invalid.", + }, + { + Cname: []string{ + "s.strikinglydns.com", + }, + Nxdomain: false, + Name: "Strikingly", + Fingerprint: "PAGE NOT FOUND.", + }, + { + Cname: []string{ + "na-west1.surge.sh", + }, + Nxdomain: false, + Name: "Surge.sh", + Fingerprint: "project not found", + }, + { + Cname: []string{ + "surveysparrow.com", + }, + Nxdomain: false, + Name: "SurveySparrow", + Fingerprint: "Account not found.", + }, + { + Cname: []string{ + "read.uberflip.com", + }, + Nxdomain: false, + Name: "Uberflip", + Fingerprint: "The URL you've accessed does not provide a hub.", + }, + { + Cname: []string{ + "stats.uptimerobot.com", + }, + Nxdomain: false, + Name: "Uptimerobot", + Fingerprint: "page not found", + }, + { + Cname: []string{ + "wordpress.com", + }, + Nxdomain: false, + Name: "Wordpress", + Fingerprint: "Do you want to register .*.wordpress.com?", + }, + { + Cname: []string{ + "worksites.net", + "69.164.223.206", + }, + Nxdomain: false, + Name: "Worksites", + Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", + }, +} diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index e69de29bb2..ced758ce62 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -0,0 +1,29 @@ +package detect + +import ( + "fmt" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if rcode == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + } + } else { + fmt.Println("check for other fingerprint") + } + } + } + + return nil +} + +func getNsTakeoverEvidence(hostnames []string) *model.Finding { + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 8d52ec9b96..57a4da346a 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -5,13 +5,13 @@ type Input struct { BaseDomain string `json:"base_domain"` ZoneApex string `json:"zone_apex"` RecordExists bool `json:"record_exists"` - RCode string `json:"rcode"` + Rcode string `json:"rcode"` ResolveChain [][]string `json:"resolve_chain"` - CnameRecord string `json:"cname_record"` - NSRecords NSRecords `json:"ns_records"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` } -type NSRecords struct { +type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } From 26f6811ef2c6ac4532de022cfc041d54632ab72e Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 6 Jul 2026 15:51:16 -0300 Subject: [PATCH 08/45] return dict of rcode answers for each dns query type --- .../dns-scanner/dns_scanner/dns_scanner.py | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 573b986ea8..e1a720542f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -17,7 +17,7 @@ TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") -DNSSEC_NAMESERVER_HOSTNAME= os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") @dataclass @@ -27,6 +27,7 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None + query_res: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -50,9 +51,9 @@ def find_zone_apex(domain, resolver=None): while name != dns.name.root: logger.debug(f"Checking for SOA at {name}") try: - answers = resolver.resolve(name, 'SOA') + answers = resolver.resolve(name, "SOA") logger.debug(f"Found SOA for {domain} at {name}: {answers[0]}") - zone_apex = str(name).rstrip('.') + zone_apex = str(name).rstrip(".") return zone_apex except NoAnswer: # Go up one level @@ -105,7 +106,13 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker """ try: q = dns.message.make_query(domain, dns.rdatatype.DNSKEY, want_dnssec=True) - resp = dns.query.tls(q, where=nameserver_ip, timeout=TIMEOUT, server_hostname=nameserver_hostname, verify=True) + resp = dns.query.tls( + q, + where=nameserver_ip, + timeout=TIMEOUT, + server_hostname=nameserver_hostname, + verify=True, + ) if resp.rcode() != dns.rcode.NOERROR: return None except Timeout: @@ -114,7 +121,9 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker ) return None except Exception as e: - logger.error(f"Error while running minimal DNSSEC check for domain '{domain}': {e}") + logger.error( + f"Error while running minimal DNSSEC check for domain '{domain}': {e}" + ) return None # Check if AD (Authenticated Data) flag is set, showing that the data is DNSSEC validated return bool(resp.flags & dns.flags.AD) @@ -201,8 +210,15 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: + query_res = {} + for query_type in [ + dns.rdatatype.A, + dns.rdatatype.SOA, + dns.rdatatype.NS, + dns.rdatatype.CNAME, + ]: rtype = get_dns_return_type(domain, query_type) + query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) break @@ -221,6 +237,7 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) + scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -284,10 +301,16 @@ def scan_domain(domain, dkim_selectors=None): logger.debug(f"Skipping DNSSEC check for {domain} - No zone apex found") zone_dnssec_enabled = None elif not DNSSEC_NAMESERVER_IP or not DNSSEC_NAMESERVER_HOSTNAME: - logger.debug(f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set") + logger.debug( + f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set" + ) zone_dnssec_enabled = None else: - zone_dnssec_enabled = dnssec_check_with_ttl(domain=zone_apex, nameserver_ip=DNSSEC_NAMESERVER_IP, nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME) + zone_dnssec_enabled = dnssec_check_with_ttl( + domain=zone_apex, + nameserver_ip=DNSSEC_NAMESERVER_IP, + nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME, + ) scan_result.zone_dnssec_enabled = zone_dnssec_enabled From 314d207332ce402e452fbcca4b71ed004d5d6ff0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:01:38 -0300 Subject: [PATCH 09/45] add function to query and process ns delegations --- .../dns-scanner/dns_scanner/dns_scanner.py | 112 +++++++++++++++++- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index e1a720542f..7441cf816f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,6 +28,7 @@ class DNSScanResult: record_exists: bool = None rcode: str = None query_res: dict = None + ns_delegations: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -145,7 +146,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): result = {"wildcard_entry": False, "wildcard_sibling": False} try: wildcard_sibling_domain = re.sub(r"^[^.]+", "*", domain) - wildcard_record = dns.resolver.resolve( + wildcard_record = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.A, raise_on_no_answer=False, @@ -157,7 +158,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): try: # check for mail-only subdomain (e.g. mail.example.com) mx_records = resolver.resolve(qname=domain, rdtype=dns.rdatatype.MX) - wildcard_mx = dns.resolver.resolve( + wildcard_mx = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.MX, raise_on_no_answer=False, @@ -194,6 +195,104 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result +def dns_query_direct( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) + if not recursion_desired: + query.flags &= ~dns.flags.RD + return dns.query.udp(query, where=where, timeout=timeout) + + +def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): + ns_hosts = ns_records.get("hostnames", []) + # Always return structured output, even when empty + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], # per-NS evidence rows + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", # none | partial | full + }, + } + + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": domain, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -221,10 +320,8 @@ def scan_domain(domain, dkim_selectors=None): query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) - break elif rtype is None: dns_answer_return_types.append(None) - continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -320,12 +417,17 @@ def scan_domain(domain, dkim_selectors=None): dmarc_scanner = DMARCScanner(domain) dmarc_scan_result = dmarc_scanner.run() scan_result.base_domain = dmarc_scan_result.get("base_domain", "") - scan_result.ns_records = dmarc_scan_result.get("ns", {}) scan_result.mx_records = dmarc_scan_result.get("mx", {}) scan_result.spf = dmarc_scan_result.get("spf", {}) scan_result.dmarc = dmarc_scan_result.get("dmarc", {}) logger.debug(f"DMARC scan elapsed time: {time.monotonic() - dmarc_start_time}") + ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) + scan_result.ns_records = ns_records + # check nameserver delegations + ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + scan_result.ns_delegations = ns_delegations + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 From ddefa536c64b32ef42d3661af333029583440786 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:30:06 -0300 Subject: [PATCH 10/45] use zone apex for probe qname and recheck for ns hostnames --- .../dns-scanner/dns_scanner/dns_scanner.py | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 7441cf816f..f82113bdeb 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -204,8 +204,42 @@ def dns_query_direct( return dns.query.udp(query, where=where, timeout=timeout) -def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + # Always return structured output, even when empty output = { "ns_hosts": ns_hosts, @@ -217,18 +251,14 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): "lame_type": "none", # none | partial | full }, } - if len(ns_hosts) == 0: output["ns_delegation"]["lame_type"] = "unknown" return output - if resolver is None: - resolver = dns.resolver.get_default_resolver() - for host in ns_hosts: row = { "ns_host": host, - "qname": domain, + "qname": qname, "qtype": "SOA", "rcode": None, "answered_authoritatively": False, @@ -237,22 +267,7 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): } try: - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - + ns_ip = get_ns_ip(host, resolver) if ns_ip is None: row["error"] = "ns_ip_resolution_failed" output["ns_delegation"]["lame_count"] += 1 @@ -425,7 +440,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + ns_delegations = check_ns_delegations( + domain=domain, zone_apex=zone_apex, ns_records=ns_records + ) scan_result.ns_delegations = ns_delegations # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records From 0155dd634646c35117267014aded10340f6510a9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:35:01 -0300 Subject: [PATCH 11/45] better func names --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index f82113bdeb..da2e3aca1b 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -195,7 +195,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def dns_query_direct( +def probe_nameserver( where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int ): query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) @@ -274,7 +274,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s output["ns_checks"].append(row) continue - res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) row["rcode"] = dns.rcode.to_text(res.rcode()) row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) From 9101a702d5c23ace565088f7234d2c43e95da848 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:23:43 -0300 Subject: [PATCH 12/45] add new input fields --- .../internal/detect/classifier.go | 20 + .../internal/detect/evidence.go | 48 ++- .../internal/detect/fingerprints.go | 345 ++++++++++++++++++ .../internal/detect/remediation.go | 4 + .../internal/detect/rules.go | 30 +- .../internal/model/input.go | 47 ++- 6 files changed, 452 insertions(+), 42 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index ad6c03d18d..5e4a157c0d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1 +1,21 @@ package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + +func Classify(input model.Input) ([]model.Finding, error) { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cnameEvidence := ExtractCNAMEEvidence(input) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := ExtractNSEvidence(input) + evidence = append(evidence, *nsEvidence) + } + + return evidence, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index c835b7444b..f0ce3424e0 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,31 +1,49 @@ package detect import ( + "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetEvidence(input model.Input) []model.Finding { - evidence := []model.Finding{} - - // parse CNAME chain - if input.CnameRecord != nil { - cname := parseCname(*input.CnameRecord) - cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) - evidence = append(evidence, *cnameEvidence) +func ExtractCNAMEEvidence(input model.Input) *model.Finding { + cname := parseCname(*input.CnameRecord) + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if input.QueryAnswers.A == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + return &model.Finding{ + Domain: input.Domain, + RecordType: "CNAME", + Provider: fp.Name, + Confidence: "confirmed", + Target: cname, + } + } + } else { + fmt.Println("check for other fingerprint") + } + } } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) - evidence = append(evidence, *nsEvidence) - } + return nil +} + +func ExtractNSEvidence(input model.Input) *model.Finding { + return nil +} - return evidence +func ClassifyLameType(nsChecks []model.NsCheck) { + return } func parseCname(record string) string { - record_tokens := strings.Split(strings.Trim(record, "."), " ") + trimmed := strings.Trim(record, ".") + lower := strings.ToLower(trimmed) + record_tokens := strings.Split(lower, " ") return record_tokens[len(record_tokens)-1] } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 5d13a5489f..9c4e642e1b 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -9,6 +9,351 @@ type Fingerprint struct { Fingerprint string } +type NSProviderStatus string + +const ( + NSStatusVulnerable NSProviderStatus = "vulnerable" + NSStatusNotVulnerable NSProviderStatus = "not_vulnerable" + NSStatusEdgeCase NSProviderStatus = "edge_case" + NSStatusVulnerableWithPurchase NSProviderStatus = "vulnerable_with_purchase" + NSStatusRegistrationClosed NSProviderStatus = "registration_closed" +) + +type NSProviderFingerprint struct { + Name string + ProviderURL string + Status NSProviderStatus + HostPatterns []string + InstructionsURL string + PrivateDNS bool +} + +func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, pattern := range f.HostPatterns { + if wildcardHostMatch(pattern, host) { + return true + } + } + return false +} + +func wildcardHostMatch(pattern, host string) bool { + pattern = strings.ToLower(strings.TrimSuffix(pattern, ".")) + if !strings.Contains(pattern, "*") { + return host == pattern + } + + parts := strings.Split(pattern, "*") + if len(parts) == 2 { + return strings.HasPrefix(host, parts[0]) && strings.HasSuffix(host, parts[1]) + } + + idx := 0 + for i, part := range parts { + if part == "" { + continue + } + + pos := strings.Index(host[idx:], part) + if pos < 0 { + return false + } + + if i == 0 && !strings.HasPrefix(host, part) { + return false + } + + idx += pos + len(part) + } + + last := parts[len(parts)-1] + if last != "" && !strings.HasSuffix(host, last) { + return false + } + + return true +} + +var NSProviderFingerprints = []NSProviderFingerprint{ + { + Name: "000Domains", + ProviderURL: "https://000domains.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", + }, + { + Name: "AWS Route 53", + ProviderURL: "https://aws.amazon.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", + }, + { + Name: "Azure (Microsoft)", + ProviderURL: "https://azure.microsoft.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", + }, + { + Name: "BigCommerce", + ProviderURL: "https://bigcommerce.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", + }, + { + Name: "Bizland", + ProviderURL: "https://bizland.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", + }, + { + Name: "ClouDNS", + ProviderURL: "https://cloudns.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.cloudns.net"}, + }, + { + Name: "Cloudflare", + ProviderURL: "https://cloudflare.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.ns.cloudflare.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", + }, + { + Name: "Digital Ocean", + ProviderURL: "https://digitalocean.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", + }, + { + Name: "DNSMadeEasy", + ProviderURL: "https://dnsmadeeasy.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns*.dnsmadeeasy.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", + }, + { + Name: "DNSimple", + ProviderURL: "https://dnsimple.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", + }, + { + Name: "Domain.com", + ProviderURL: "https://domain.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", + }, + { + Name: "DomainPeople", + ProviderURL: "https://domainpeople.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", + }, + { + Name: "Dotster", + ProviderURL: "https://dotster.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", + }, + { + Name: "Dreamhost", + ProviderURL: "https://dreamhost.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", + }, + { + Name: "EasyDNS", + ProviderURL: "https://easydns.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", + }, + { + Name: "Gandi.net", + ProviderURL: "https://gandi.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, + }, + { + Name: "Google Cloud", + ProviderURL: "https://cloud.google.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns-cloud-*.googledomains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", + }, + { + Name: "Hostinger (old NS)", + ProviderURL: "https://hostinger.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, + }, + { + Name: "Hover", + ProviderURL: "https://hover.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", + }, + { + Name: "Hurricane Electric", + ProviderURL: "https://dns.he.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", + }, + { + Name: "Linode", + ProviderURL: "https://linode.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", + }, + { + Name: "MediaTemple (mt)", + ProviderURL: "https://mediatemple.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", + }, + { + Name: "MyDomain", + ProviderURL: "https://mydomain.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", + }, + { + Name: "Name.com", + ProviderURL: "https://name.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, + InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", + }, + { + Name: "Namecheap", + ProviderURL: "https://namecheap.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, + }, + { + Name: "Network Solutions", + ProviderURL: "https://networksolutions.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.worldnic.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", + }, + { + Name: "NS1", + ProviderURL: "https://nsone.net/", + Status: NSStatusRegistrationClosed, + HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", + }, + { + Name: "TierraNet", + ProviderURL: "https://tierra.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", + }, + { + Name: "Reg.ru", + ProviderURL: "https://reg.ru/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", + }, + { + Name: "UltraDNS", + ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", + }, + { + Name: "Yahoo Small Business", + ProviderURL: "https://yahoosmallbusiness.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", + }, + { + Name: "Activision", + ProviderURL: "https://activision.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.activision.com"}, + PrivateDNS: true, + }, + { + Name: "Adobe", + ProviderURL: "https://adobe.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"adobe-dns-0*.adobe.com"}, + PrivateDNS: true, + }, + { + Name: "Apple", + ProviderURL: "https://apple.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, + PrivateDNS: true, + }, + { + Name: "Automattic", + ProviderURL: "https://automattic.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.automattic.com"}, + PrivateDNS: true, + }, + { + Name: "Capital One", + ProviderURL: "https://capitalone.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.capitalone.com"}, + PrivateDNS: true, + }, + { + Name: "Disney", + ProviderURL: "https://disney.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, + PrivateDNS: true, + }, + { + Name: "Google", + ProviderURL: "https://google.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.google.com"}, + PrivateDNS: true, + }, + { + Name: "Lowe's", + ProviderURL: "https://lowes.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"authns*.lowes.com"}, + PrivateDNS: true, + }, + { + Name: "T-Mobile", + ProviderURL: "https://tmobileus.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, + PrivateDNS: true, + }, +} + func (f *Fingerprint) ContainsCname(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index ad6c03d18d..9d577baa7f 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -1 +1,5 @@ package detect + +func RemediationForReason(reasonCode string, provider string, lameType string) { + return +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index ced758ce62..1d2bb6c10a 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,29 +1,21 @@ package detect import ( - "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if rcode == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - } - } else { - fmt.Println("check for other fingerprint") - } - } - } +func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { + return +} + +func MatchNSProviderRules(evidence model.Finding, providerRules []string) { + return +} - return nil +func ShouldEmitCNAME() { + return } -func getNsTakeoverEvidence(hostnames []string) *model.Finding { - return nil +func ShouldEmitNSHijack() { + return } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 57a4da346a..7bdd179f73 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,17 +1,48 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + Rcode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` + QueryAnswers QueryAnswers `json:"query_res"` + NsDelegations NsDelegations `json:"ns_delegations"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } + +type QueryAnswers struct { + A string `json:"A"` + SOA string `json:"SOA"` + NS string `json:"NS"` + CNAME string `json:"CNAME"` +} + +type NsDelegations struct { + Hosts []string `json:"ns_hosts"` + Checks []NsCheck `json:"ns_checks"` + Delegation Delegation `json:"ns_delegation"` +} + +type NsCheck struct { + Host string `json:"ns_host"` + Qname string `json:"qname"` + Qtype string `json:"qtype"` + Rcode string `json:"rcode"` + AnsweredAuthoritatively bool `json:"answered_authoritatively"` + Error *string `json:"error"` + Timeout bool `json:"timeout"` +} +type Delegation struct { + TotalHosts int `json:"total_ns"` + OkCount int `json:"authoritative_ok"` + LameCount int `json:"lame_count"` + LameType string `json:"lame_type"` +} From c213301480bc39bc8d5706a1d9571519dcbbb8e5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:24:50 -0300 Subject: [PATCH 13/45] add registrar checks and move new logic to ns_registrar.py --- .../dns-scanner/dns_scanner/dns_scanner.py | 121 +---------- .../dns-scanner/dns_scanner/ns_registrar.py | 195 ++++++++++++++++++ 2 files changed, 203 insertions(+), 113 deletions(-) create mode 100644 scanners/dns-scanner/dns_scanner/ns_registrar.py diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index da2e3aca1b..06b72869d5 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,6 +12,7 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner +from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context logger = logging.getLogger(__name__) @@ -29,6 +30,7 @@ class DNSScanResult: rcode: str = None query_res: dict = None ns_delegations: dict = None + registrar_context: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -195,119 +197,6 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def probe_nameserver( - where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int -): - query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) - if not recursion_desired: - query.flags &= ~dns.flags.RD - return dns.query.udp(query, where=where, timeout=timeout) - - -def get_ns_ip(host: str, resolver): - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - return ns_ip - - -def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): - if resolver is None: - resolver = dns.resolver.get_default_resolver() - - qname = zone_apex - if not zone_apex: - qname = domain - - ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: - try: - ns_res = resolver.resolve(domain, dns.rdatatype.NS) - ns_hosts = [host.to_text() for host in ns_res] - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_hosts = [] - - # Always return structured output, even when empty - output = { - "ns_hosts": ns_hosts, - "ns_checks": [], # per-NS evidence rows - "ns_delegation": { - "total_ns": len(ns_hosts), - "authoritative_ok": 0, - "lame_count": 0, - "lame_type": "none", # none | partial | full - }, - } - if len(ns_hosts) == 0: - output["ns_delegation"]["lame_type"] = "unknown" - return output - - for host in ns_hosts: - row = { - "ns_host": host, - "qname": qname, - "qtype": "SOA", - "rcode": None, - "answered_authoritatively": False, - "error": None, - "timeout": False, - } - - try: - ns_ip = get_ns_ip(host, resolver) - if ns_ip is None: - row["error"] = "ns_ip_resolution_failed" - output["ns_delegation"]["lame_count"] += 1 - output["ns_checks"].append(row) - continue - - res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) - row["rcode"] = dns.rcode.to_text(res.rcode()) - row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) - - if row["answered_authoritatively"] and row["rcode"] in [ - "NOERROR", - "NXDOMAIN", - ]: - output["ns_delegation"]["authoritative_ok"] += 1 - else: - output["ns_delegation"]["lame_count"] += 1 - except Timeout: - row["timeout"] = True - row["error"] = "timeout" - output["ns_delegation"]["lame_count"] += 1 - except Exception as e: - row["error"] = str(e) - output["ns_delegation"]["lame_count"] += 1 - - output["ns_checks"].append(row) - - ok = output["ns_delegation"]["authoritative_ok"] - total = output["ns_delegation"]["total_ns"] - - if ok == total: - output["ns_delegation"]["lame_type"] = "none" - elif ok == 0: - output["ns_delegation"]["lame_type"] = "full" - else: - output["ns_delegation"]["lame_type"] = "partial" - - return output - - def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -445,6 +334,12 @@ def scan_domain(domain, dkim_selectors=None): ) scan_result.ns_delegations = ns_delegations + registrar_domain = scan_result.base_domain or zone_apex or domain + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=ns_delegations.get("ns_hosts", []), + ) + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py new file mode 100644 index 0000000000..9b48e7754f --- /dev/null +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -0,0 +1,195 @@ +import os + +import dns +import dns.resolver +import requests +from dns.exception import Timeout +from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers + +TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) + + +def probe_nameserver( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query( + qname, + dns.rdatatype.from_text(qtype), + use_edns=True, + recursion_desired=recursion_desired, + ) + return dns.query.udp(query, where=where, timeout=timeout) + + +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", + }, + } + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": qname, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = get_ns_ip(host, resolver) + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = probe_nameserver(ns_ip, qname, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + +def get_registrar_context(base_domain, ns_hosts=None): + context = { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": None, + } + + if not base_domain: + context["error"] = "missing_base_domain" + return context + + rdap_url = f"https://rdap.org/domain/{base_domain}" + context["rdap_url"] = rdap_url + + try: + response = requests.get(rdap_url, timeout=TIMEOUT) + response.raise_for_status() + payload = response.json() + except Exception as e: + context["error"] = str(e) + return context + + context["lookup_success"] = True + + nameservers = payload.get("nameservers", []) + context["rdap_nameservers"] = [ + (ns.get("ldhName") or "").rstrip(".").lower() + for ns in nameservers + if ns.get("ldhName") + ] + + entities = payload.get("entities", []) + for entity in entities: + roles = [r.lower() for r in entity.get("roles", [])] + if "registrar" not in roles: + continue + + context["registrar_id"] = entity.get("handle") + + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) == 2 and isinstance(vcard[1], list): + for entry in vcard[1]: + if not isinstance(entry, list) or len(entry) < 4: + continue + key = entry[0] + value = entry[3] + if key in {"fn", "org"} and value: + context["registrar_name"] = value + break + + if context["registrar_name"] is None: + public_ids = entity.get("publicIds", []) + if public_ids: + context["registrar_name"] = public_ids[0].get("identifier") + + break + + if ns_hosts is not None: + normalized_hosts = {h.rstrip(".").lower() for h in ns_hosts if h} + rdap_hosts = set(context["rdap_nameservers"]) + if rdap_hosts: + context["delegation_matches_rdap"] = normalized_hosts == rdap_hosts + + return context From 8442f3adda185fa1c842fb5eb51b952d5cc9eeaa Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 18:23:44 -0300 Subject: [PATCH 14/45] establish Classify() workflow --- .../subdomain-takeover/internal/app/worker.go | 20 ++++--- .../internal/detect/classifier.go | 25 +++++---- .../internal/detect/evidence.go | 52 ++++++++++--------- .../internal/detect/fingerprints.go | 6 +-- .../internal/detect/rules.go | 44 +++++++++++++--- 5 files changed, 93 insertions(+), 54 deletions(-) diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 7d000378e7..40176d6c88 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -28,15 +28,21 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { if err != nil { log.Err(err).Msg("Decoding error") } - log.Info().Msg(scan.Domain) + // log.Info().Msg(scan.Domain) - detect.GetEvidence(scan) + findings, err := detect.Classify(scan) + if err != nil { + log.Err(err).Msg("classify error") + return err + } - // var finding model.Finding - // err := w.pub.Publish(ctx, finding) - // if err != nil { - // return err - // } + for _, finding := range findings { + err = w.pub.Publish(ctx, finding) + if err != nil { + log.Err(err).Msg("publish error") + return err + } + } if err := msg.Ack(); err != nil { log.Err(err).Msg("Ack error") diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5e4a157c0d..f14a5b7f35 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -3,19 +3,22 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" func Classify(input model.Input) ([]model.Finding, error) { - evidence := []model.Finding{} + findings := []model.Finding{} - // parse CNAME chain - if input.CnameRecord != nil { - cnameEvidence := ExtractCNAMEEvidence(input) - evidence = append(evidence, *cnameEvidence) + cnameEvidence := ExtractCNAMEEvidence(input) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) + if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := ExtractNSEvidence(input) - evidence = append(evidence, *nsEvidence) + nsEvidence := ExtractNSEvidence(input) + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(*nsEvidence, nsHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - - return evidence, nil + return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f0ce3424e0..fb40318cfc 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,40 +1,42 @@ package detect import ( - "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func ExtractCNAMEEvidence(input model.Input) *model.Finding { - cname := parseCname(*input.CnameRecord) - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if input.QueryAnswers.A == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - return &model.Finding{ - Domain: input.Domain, - RecordType: "CNAME", - Provider: fp.Name, - Confidence: "confirmed", - Target: cname, - } - } - } else { - fmt.Println("check for other fingerprint") - } - } +// inputs from evidence.go +type CNAMEEvidence struct { + Domain string + Target string + QueryAnswer model.QueryAnswers // or just needed fields +} + +type NSEvidence struct { + Domain string + NSHosts []string + NSDelegations model.NsDelegations + // Registrar model.RegistrarContext // if/when added +} + +func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { + if input.CnameRecord == nil { + return nil } - return nil + return &CNAMEEvidence{ + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + QueryAnswer: input.QueryAnswers, + } } -func ExtractNSEvidence(input model.Input) *model.Finding { - return nil +func ExtractNSEvidence(input model.Input) *NSEvidence { + if len(input.NsDelegations.Hosts) == 0 { + return nil + } + return &NSEvidence{} } func ClassifyLameType(nsChecks []model.NsCheck) { diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 9c4e642e1b..b7098341a2 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -2,7 +2,7 @@ package detect import "strings" -type Fingerprint struct { +type CNAMEProviderFingerprint struct { Cname []string Name string Nxdomain bool @@ -354,7 +354,7 @@ var NSProviderFingerprints = []NSProviderFingerprint{ }, } -func (f *Fingerprint) ContainsCname(target string) bool { +func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { return true @@ -363,7 +363,7 @@ func (f *Fingerprint) ContainsCname(target string) bool { return false } -var Fingerprints = []Fingerprint{ +var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ { Cname: []string{ "elasticbeanstalk.com", diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 1d2bb6c10a..15edb479d6 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,21 +1,49 @@ package detect import ( - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "fmt" ) -func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { - return +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + NeedsNX bool } -func MatchNSProviderRules(evidence model.Finding, providerRules []string) { - return +type NSHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + ProviderVulnerable bool + RegistrarMismatch bool } -func ShouldEmitCNAME() { - return +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if evidence.QueryAnswer.A == "NXDOMAIN" { + fmt.Println("Bingo") + } + } // else { + // fmt.Println("check for other fingerprint") + // } + } + } } -func ShouldEmitNSHijack() { +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { return } + +func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { + return false +} + +func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { + return false +} From c4c1582a16b045ade48036f8a4d53a57df9c400a Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 10:08:51 -0300 Subject: [PATCH 15/45] match cname fingerprints and emit finding --- .../internal/detect/classifier.go | 14 ++++- .../internal/detect/rules.go | 62 ++++++++++++++++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index f14a5b7f35..0caea61196 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -7,15 +7,25 @@ func Classify(input model.Input) ([]model.Finding, error) { cnameEvidence := ExtractCNAMEEvidence(input) cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + if ShouldEmitCNAME(cnameHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ - Domain: input.Domain, + Domain: cnameEvidence.Domain, + RecordType: "CNAME", + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: cnameHit.ReasonCode, + Confidence: "", + Remediation: "", }) } nsEvidence := ExtractNSEvidence(input) nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ Domain: input.Domain, }) diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 15edb479d6..9cb5ac5738 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -2,13 +2,15 @@ package detect import ( "fmt" + "io" + "net/http" + "strings" ) type CNAMEHit struct { Matched bool Provider string ReasonCode string - RuleID string NeedsNX bool } @@ -16,7 +18,6 @@ type NSHit struct { Matched bool Provider string ReasonCode string - RuleID string ProviderVulnerable bool RegistrarMismatch bool } @@ -24,26 +25,67 @@ type NSHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + if hit.NeedsNX { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" if evidence.QueryAnswer.A == "NXDOMAIN" { - fmt.Println("Bingo") + hit.Matched = true + hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" } - } // else { - // fmt.Println("check for other fingerprint") - // } + } else { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.Matched = true + hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + } + } + return &hit } } + return nil } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return + return &NSHit{} } -func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + if hit.Matched { + return true + } return false } func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } + +func urlBodyContainsFingerprint(domain string, fp string) bool { + // curl domain and look for matching fp in body + url := fmt.Sprintf("http://%s", domain) + res, err := http.Get(url) + if err != nil { + // log error + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + // log err + return false + } + + if strings.Contains(string(body), fp) { + return true + } + return false +} From 61b90a5c44884f12ae4be2566ebcf91035d820f4 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 12:39:40 -0300 Subject: [PATCH 16/45] revert changes to record gate --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 16 ++++------------ scanners/dns-scanner/dns_scanner/ns_registrar.py | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 06b72869d5..40567ba136 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,7 +28,6 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None - query_res: dict = None ns_delegations: dict = None registrar_context: dict = None resolve_chain: list[list[str]] = None @@ -213,19 +212,14 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - query_res = {} - for query_type in [ - dns.rdatatype.A, - dns.rdatatype.SOA, - dns.rdatatype.NS, - dns.rdatatype.CNAME, - ]: + for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: rtype = get_dns_return_type(domain, query_type) - query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) + break elif rtype is None: dns_answer_return_types.append(None) + continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -238,7 +232,6 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) - scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -329,10 +322,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations( + scan_result.ns_delegations = check_ns_delegations( domain=domain, zone_apex=zone_apex, ns_records=ns_records ) - scan_result.ns_delegations = ns_delegations registrar_domain = scan_result.base_domain or zone_apex or domain scan_result.registrar_context = get_registrar_context( diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py index 9b48e7754f..4e60d31dc4 100644 --- a/scanners/dns-scanner/dns_scanner/ns_registrar.py +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -16,8 +16,9 @@ def probe_nameserver( qname, dns.rdatatype.from_text(qtype), use_edns=True, - recursion_desired=recursion_desired, ) + if not recursion_desired: + query.flags &= ~dns.flags.RD return dns.query.udp(query, where=where, timeout=timeout) @@ -50,7 +51,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s qname = domain ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: + if len(ns_hosts) == 0: try: ns_res = resolver.resolve(domain, dns.rdatatype.NS) ns_hosts = [host.to_text() for host in ns_res] From c09a79000e6540d8562eefeeea251ef79c318859 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:14:24 -0300 Subject: [PATCH 17/45] run go mod tidy --- scanners/subdomain-takeover/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod index f4bfb6b567..ca7b20f4ac 100644 --- a/scanners/subdomain-takeover/go.mod +++ b/scanners/subdomain-takeover/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats.go v1.52.0 github.com/rs/zerolog v1.35.1 ) @@ -11,7 +12,6 @@ require ( github.com/klauspost/compress v1.18.5 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/nats-io/nats.go v1.52.0 // indirect github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.49.0 // indirect From 7e55e8828483d9bd9e19af66d1ba4f4bb6b87f36 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:14 -0300 Subject: [PATCH 18/45] move logger out of config.go --- .../internal/bootstrap/logger.go | 13 +++++++++++ .../internal/config/config.go | 22 +++++-------------- .../internal/logging/logger.go | 0 3 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/logger.go delete mode 100644 scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/internal/bootstrap/logger.go b/scanners/subdomain-takeover/internal/bootstrap/logger.go new file mode 100644 index 0000000000..3721d23e40 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/logger.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "os" + + "github.com/rs/zerolog" +) + +func NewLogger(level zerolog.Level) zerolog.Logger { + zerolog.SetGlobalLevel(level) + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + return zerolog.New(multi).With().Timestamp().Logger() +} diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 28d2ddf824..1b18e68117 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,37 +1,25 @@ package config import ( - "os" - "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog" ) type Config struct { - NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` - NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` - Logger zerolog.Logger } -func InitConfig() *Config { +func Load() (*Config, error) { var cfg Config if err := envconfig.Process("", &cfg); err != nil { - panic(err) + return nil, err } - zerolog.SetGlobalLevel(cfg.LogLevel) - - // Create multiple output steams for zerolog - multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) - - logger := zerolog.New(multi).With().Timestamp().Logger() - - cfg.Logger = logger - - return &cfg + return &cfg, nil } diff --git a/scanners/subdomain-takeover/internal/logging/logger.go b/scanners/subdomain-takeover/internal/logging/logger.go deleted file mode 100644 index e69de29bb2..0000000000 From 7af7e0f892161ba16f1bc6c75495742b671cada8 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:57 -0300 Subject: [PATCH 19/45] bootstrap nats and add input interfaces --- .../subdomain-takeover/internal/app/runner.go | 6 +- .../subdomain-takeover/internal/app/worker.go | 47 +++++++++++---- .../internal/bootstrap/nats.go | 59 +++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/nats.go diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 42539405f0..6d7ee52e8d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -15,10 +15,14 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Worker Worker + Worker MessageHandler NC *nats.Conn } +type MessageHandler interface { + Handle(ctx context.Context, msg jetstream.Msg) error +} + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 40176d6c88..e282fab952 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -4,48 +4,59 @@ import ( "bytes" "context" "encoding/json" + "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) +type FindingPublisher interface { + Publish(ctx context.Context, finding model.Finding) error +} + +type ScanClassifier interface { + Classify(input model.Input) ([]model.Finding, error) +} + type Worker struct { - logger zerolog.Logger - pub messaging.Publisher + logger zerolog.Logger + publisher FindingPublisher + classifier ScanClassifier } -func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { - return &Worker{logger: logger, pub: pub} +func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { + return &Worker{logger: logger, publisher: publisher, classifier: classifier} } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { log := w.logger.With().Str("component", "worker").Logger() - // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) if err != nil { - log.Err(err).Msg("Decoding error") + log.Err(err).Msg("decode error") + w.term(msg, log, err) + return err } - // log.Info().Msg(scan.Domain) - findings, err := detect.Classify(scan) + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") + w.nak(msg, log, err) return err } for _, finding := range findings { - err = w.pub.Publish(ctx, finding) + err = w.publisher.Publish(ctx, finding) if err != nil { log.Err(err).Msg("publish error") + w.nak(msg, log, err) return err } } if err := msg.Ack(); err != nil { - log.Err(err).Msg("Ack error") + log.Err(err).Msg("ack error") return err } @@ -58,3 +69,15 @@ func decodeScan(data []byte) (model.Input, error) { err := json.Unmarshal(trimmed, &scan) return scan, err } + +func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Nak(); err != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") + } +} + +func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Term(); err != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") + } +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go new file mode 100644 index 0000000000..51edb6b060 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -0,0 +1,59 @@ +package bootstrap + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type RuntimeDeps struct { + NC *nats.Conn + Iter jetstream.MessagesContext + JS jetstream.JetStream +} + +func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { + nc, err := nats.Connect(cfg.NATSURL) + if err != nil { + return nil, err + } + + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + if err != nil { + nc.Close() + return nil, err + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + if err != nil { + nc.Close() + return nil, err + } + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + nc.Close() + return nil, err + } + + return &RuntimeDeps{ + NC: nc, + Iter: iter, + JS: js, + }, nil +} From 7e4cdd9ba87fa811e0332be912622ab8d8269f31 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:17:37 -0300 Subject: [PATCH 20/45] add better error handling to main.go during startup and shutdown --- .../subdomain-takeover/cmd/service/main.go | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 66085ffffa..0f588ec30d 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -8,76 +8,70 @@ import ( "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) -func checkErr(err error, log zerolog.Logger, msg string, exit bool) { +func main() { + logger := bootstrap.NewLogger(zerolog.InfoLevel) + + cfg, err := config.Load() if err != nil { - log.Err(err).Msg(msg) - if exit { - os.Exit(1) - } + logger.Fatal().Err(err).Msg("failed to load config") } -} + logger = bootstrap.NewLogger(cfg.LogLevel) -func main() { - cfg := config.InitConfig() - logger := cfg.Logger + if err := detect.LoadFingerprints(); err != nil { + logger.Fatal().Err(err).Msg("failed to load fingerprints") + } ctx, cancel := context.WithCancel(context.Background()) + defer cancel() sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) - nc, err := nats.Connect(cfg.NatsUrl) - checkErr(err, logger, "", true) - - js, err := jetstream.New(nc) - checkErr(err, logger, "", true) - - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: cfg.NatsStream, - Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, - }) - checkErr(err, logger, "", true) - - cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: cfg.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - checkErr(err, logger, "", true) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + if err != nil { + logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") + } - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) - pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) - worker := app.NewWorker(logger, *pub) + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) + classifier := detect.NewClassifier(matcher) + worker := app.NewWorker(logger, pub, classifier) go func() { <-sig logger.Info().Msg("Shutdown requested...") cancel() - iter.Stop() + runtimeDeps.Iter.Stop() }() deps := app.RunnerDeps{ Logger: logger, WorkerCount: cfg.WorkerCount, - Iter: iter, - Worker: *worker, - NC: nc, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, } app.Run(ctx, deps) - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() + logger.Info().Msgf("Disconnecting from NATS at %s", runtimeDeps.NC.ConnectedUrl()) + if err := runtimeDeps.NC.Flush(); err != nil { + logger.Error().Err(err).Msg("failed to flush nats connection") + } + runtimeDeps.NC.Close() + + if err := ctx.Err(); err != nil && err != context.Canceled { + logger.Error().Err(err).Msg("service exited with context error") + } } From 1d19257e48a5e24e6863cb27b8b9723f08f7a172 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:22 -0300 Subject: [PATCH 21/45] move fingerprint data fixtures to own json files --- .../detect/data/cname_fingerprints.json | 227 ++++++++ .../internal/detect/data/ns_fingerprints.json | 416 +++++++++++++ .../internal/detect/fingerprint_loader.go | 107 ++++ .../internal/detect/fingerprints.go | 546 +----------------- 4 files changed, 781 insertions(+), 515 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_loader.go diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json new file mode 100644 index 0000000000..cbf3aa24be --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json @@ -0,0 +1,227 @@ +[ + { + "cname": [ + "elasticbeanstalk.com" + ], + "name": "AWS/Elastic Beanstalk", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "s3.amazonaws.com" + ], + "name": "AWS/S3", + "nxdomain": false, + "fingerprint": "The specified bucket does not exist" + }, + { + "cname": [ + "agilecrm.com" + ], + "name": "Agile CRM", + "nxdomain": false, + "fingerprint": "Sorry, this page is no longer available." + }, + { + "cname": [ + "airee.ru" + ], + "name": "Airee.ru", + "nxdomain": false, + "fingerprint": "Ошибка 402. Сервис Айри.рф не оплачен" + }, + { + "cname": [ + "animaapp.io" + ], + "name": "Anima", + "nxdomain": false, + "fingerprint": "The page you were looking for does not exist." + }, + { + "cname": [ + "bitbucket.io" + ], + "name": "Bitbucket", + "nxdomain": false, + "fingerprint": "Repository not found" + }, + { + "cname": [ + "trydiscourse.com" + ], + "name": "Discourse", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "furyns.com" + ], + "name": "Gemfury", + "nxdomain": false, + "fingerprint": "404: This page could not be found." + }, + { + "cname": [ + "ghost.io" + ], + "name": "Ghost", + "nxdomain": false, + "fingerprint": "Site unavailable\\.\u0026#124;Failed to resolve DNS path for this host" + }, + { + "cname": [ + "hatenablog.com" + ], + "name": "HatenaBlog", + "nxdomain": false, + "fingerprint": "404 Blog is not found" + }, + { + "cname": [ + "helpjuice.com" + ], + "name": "Help Juice", + "nxdomain": false, + "fingerprint": "We could not find what you're looking for." + }, + { + "cname": [ + "helpscoutdocs.com" + ], + "name": "Help Scout", + "nxdomain": false, + "fingerprint": "No settings were found for this company:" + }, + { + "cname": [ + "helprace.com" + ], + "name": "Helprace", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=301" + }, + { + "cname": [ + "youtrack.cloud" + ], + "name": "JetBrains", + "nxdomain": false, + "fingerprint": "is not a registered InCloud YouTrack" + }, + { + "cname": [ + "launchrock.com" + ], + "name": "LaunchRock", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=500" + }, + { + "cname": [ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com" + ], + "name": "Microsoft Azure", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "ngrok.io" + ], + "name": "Ngrok", + "nxdomain": false, + "fingerprint": "Tunnel .*.ngrok.io not found" + }, + { + "cname": [ + "readme.io" + ], + "name": "Readme.io", + "nxdomain": false, + "fingerprint": "The creators of this project are still working on making everything perfect!" + }, + { + "cname": [ + "52.16.160.97" + ], + "name": "SmartJobBoard", + "nxdomain": false, + "fingerprint": "This job board website is either expired or its domain name is invalid." + }, + { + "cname": [ + "s.strikinglydns.com" + ], + "name": "Strikingly", + "nxdomain": false, + "fingerprint": "PAGE NOT FOUND." + }, + { + "cname": [ + "na-west1.surge.sh" + ], + "name": "Surge.sh", + "nxdomain": false, + "fingerprint": "project not found" + }, + { + "cname": [ + "surveysparrow.com" + ], + "name": "SurveySparrow", + "nxdomain": false, + "fingerprint": "Account not found." + }, + { + "cname": [ + "read.uberflip.com" + ], + "name": "Uberflip", + "nxdomain": false, + "fingerprint": "The URL you've accessed does not provide a hub." + }, + { + "cname": [ + "stats.uptimerobot.com" + ], + "name": "Uptimerobot", + "nxdomain": false, + "fingerprint": "page not found" + }, + { + "cname": [ + "wordpress.com" + ], + "name": "Wordpress", + "nxdomain": false, + "fingerprint": "Do you want to register .*.wordpress.com?" + }, + { + "cname": [ + "worksites.net", + "69.164.223.206" + ], + "name": "Worksites", + "nxdomain": false, + "fingerprint": "Hello! Sorry, but the website you\u0026rsquo;re looking for doesn\u0026rsquo;t exist." + } +] diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json new file mode 100644 index 0000000000..85fd5262bf --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json @@ -0,0 +1,416 @@ +[ + { + "name": "000Domains", + "provider_url": "https://000domains.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.000domains.com", + "ns2.000domains.com", + "fwns1.000domains.com", + "fwns2.000domains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/19" + }, + { + "name": "AWS Route 53", + "provider_url": "https://aws.amazon.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns-*.awsdns-*.org", + "ns-*.awsdns-*.co.uk", + "ns-*.awsdns-*.com", + "ns-*.awsdns-*.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/1" + }, + { + "name": "Azure (Microsoft)", + "provider_url": "https://azure.microsoft.com/", + "status": "edge_case", + "host_patterns": [ + "ns1-*.azure-dns.com", + "ns2-*.azure-dns.net", + "ns3-*.azure-dns.org", + "ns4-*.azure-dns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/5" + }, + { + "name": "BigCommerce", + "provider_url": "https://bigcommerce.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bigcommerce.com", + "ns2.bigcommerce.com", + "ns3.bigcommerce.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/35" + }, + { + "name": "Bizland", + "provider_url": "https://bizland.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bizland.com", + "ns2.bizland.com", + "clickme.click2site.com", + "clickme2.click2site.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/3" + }, + { + "name": "ClouDNS", + "provider_url": "https://cloudns.net/", + "status": "not_vulnerable", + "host_patterns": [ + "*.cloudns.net" + ] + }, + { + "name": "Cloudflare", + "provider_url": "https://cloudflare.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.ns.cloudflare.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/10" + }, + { + "name": "Digital Ocean", + "provider_url": "https://digitalocean.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/22" + }, + { + "name": "DNSMadeEasy", + "provider_url": "https://dnsmadeeasy.com/", + "status": "vulnerable", + "host_patterns": [ + "ns*.dnsmadeeasy.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/6" + }, + { + "name": "DNSimple", + "provider_url": "https://dnsimple.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.dnsimple.com", + "ns2.dnsimple.com", + "ns3.dnsimple.com", + "ns4.dnsimple.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/16" + }, + { + "name": "Domain.com", + "provider_url": "https://domain.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1.domain.com", + "ns2.domain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/17" + }, + { + "name": "DomainPeople", + "provider_url": "https://domainpeople.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.domainpeople.com", + "ns2.domainpeople.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/14" + }, + { + "name": "Dotster", + "provider_url": "https://dotster.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.dotster.com", + "ns2.dotster.com", + "ns1.nameresolve.com", + "ns2.nameresolve.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/18" + }, + { + "name": "Dreamhost", + "provider_url": "https://dreamhost.com/", + "status": "edge_case", + "host_patterns": [ + "ns1.dreamhost.com", + "ns2.dreamhost.com", + "ns3.dreamhost.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/40" + }, + { + "name": "EasyDNS", + "provider_url": "https://easydns.com/", + "status": "not_vulnerable", + "host_patterns": [ + "dns1.easydns.com", + "dns2.easydns.net", + "dns3.easydns.org", + "dns4.easydns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/9" + }, + { + "name": "Gandi.net", + "provider_url": "https://gandi.net/", + "status": "not_vulnerable", + "host_patterns": [ + "a.dns.gandi.net", + "b.dns.gandi.net", + "c.dns.gandi.net" + ] + }, + { + "name": "Google Cloud", + "provider_url": "https://cloud.google.com/", + "status": "edge_case", + "host_patterns": [ + "ns-cloud-*.googledomains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/2" + }, + { + "name": "Hostinger (old NS)", + "provider_url": "https://hostinger.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hostinger.com", + "ns2.hostinger.com" + ] + }, + { + "name": "Hover", + "provider_url": "https://hover.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hover.com", + "ns2.hover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/21" + }, + { + "name": "Hurricane Electric", + "provider_url": "https://dns.he.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.he.net", + "ns2.he.net", + "ns3.he.net", + "ns4.he.net", + "ns5.he.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/25" + }, + { + "name": "Linode", + "provider_url": "https://linode.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.linode.com", + "ns2.linode.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/26" + }, + { + "name": "MediaTemple (mt)", + "provider_url": "https://mediatemple.net/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mediatemple.net", + "ns2.mediatemple.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/23" + }, + { + "name": "MyDomain", + "provider_url": "https://mydomain.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mydomain.com", + "ns2.mydomain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/4" + }, + { + "name": "Name.com", + "provider_url": "https://name.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1*.name.com", + "ns2*.name.com", + "ns3*.name.com", + "ns4*.name.com" + ], + "instructions_url": "https://github.com/libertalialtd/can-i-take-over-dns/issues/8" + }, + { + "name": "Namecheap", + "provider_url": "https://namecheap.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.namecheaphosting.com", + "*.registrar-servers.com" + ] + }, + { + "name": "Network Solutions", + "provider_url": "https://networksolutions.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.worldnic.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/15" + }, + { + "name": "NS1", + "provider_url": "https://nsone.net/", + "status": "registration_closed", + "host_patterns": [ + "dns1.p*.nsone.net", + "dns2.p*.nsone.net", + "dns3.p*.nsone.net", + "dns4.p*.nsone.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/7" + }, + { + "name": "TierraNet", + "provider_url": "https://tierra.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.domaindiscover.com", + "ns2.domaindiscover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/24" + }, + { + "name": "Reg.ru", + "provider_url": "https://reg.ru/", + "status": "vulnerable", + "host_patterns": [ + "ns1.reg.ru", + "ns2.reg.ru" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/28" + }, + { + "name": "UltraDNS", + "provider_url": "https://www.home.neustar/dns-services/ultra-dns", + "status": "not_vulnerable", + "host_patterns": [ + "pdns*.ultradns.com", + "udns*.ultradns.com", + "sdns*.ultradns.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/29" + }, + { + "name": "Yahoo Small Business", + "provider_url": "https://yahoosmallbusiness.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "yns1.yahoo.com", + "yns2.yahoo.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/20" + }, + { + "name": "Activision", + "provider_url": "https://activision.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.activision.com" + ], + "private_dns": true + }, + { + "name": "Adobe", + "provider_url": "https://adobe.com/", + "status": "not_vulnerable", + "host_patterns": [ + "adobe-dns-0*.adobe.com" + ], + "private_dns": true + }, + { + "name": "Apple", + "provider_url": "https://apple.com/", + "status": "not_vulnerable", + "host_patterns": [ + "a.ns.apple.com", + "b.ns.apple.com", + "c.ns.apple.com", + "d.ns.apple.com" + ], + "private_dns": true + }, + { + "name": "Automattic", + "provider_url": "https://automattic.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.automattic.com" + ], + "private_dns": true + }, + { + "name": "Capital One", + "provider_url": "https://capitalone.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.capitalone.com" + ], + "private_dns": true + }, + { + "name": "Disney", + "provider_url": "https://disney.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.twdcns.com", + "ns*.twdcns.info", + "ns*.twdcns.co.uk" + ], + "private_dns": true + }, + { + "name": "Google", + "provider_url": "https://google.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.google.com" + ], + "private_dns": true + }, + { + "name": "Lowe's", + "provider_url": "https://lowes.com/", + "status": "not_vulnerable", + "host_patterns": [ + "authns*.lowes.com" + ], + "private_dns": true + }, + { + "name": "T-Mobile", + "provider_url": "https://tmobileus.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns10.tmobileus.com", + "ns10.tmobileus.net" + ], + "private_dns": true + } +] diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go new file mode 100644 index 0000000000..694bf833aa --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -0,0 +1,107 @@ +package detect + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" +) + +//go:embed data/*.json +var fingerprintFS embed.FS + +var ( + loadFingerprintsOnce sync.Once + loadFingerprintsErr error +) + +func LoadFingerprints() error { + loadFingerprintsOnce.Do(func() { + var cname []CNAMEProviderFingerprint + var ns []NSProviderFingerprint + + if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateCNAMEFingerprints(cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateNSFingerprints(ns); err != nil { + loadFingerprintsErr = err + return + } + + CNAMEProviderFingerprints = cname + NSProviderFingerprints = ns + }) + + return loadFingerprintsErr +} + +func loadJSON(path string, out any) error { + b, err := fingerprintFS.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + + return nil +} + +func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("cname fingerprint[%d] missing name", i) + } + if len(fp.Cname) == 0 { + return fmt.Errorf("cname fingerprint[%d] has no cname patterns", i) + } + + for j, cname := range fp.Cname { + if strings.TrimSpace(cname) == "" { + return fmt.Errorf("cname fingerprint[%d] has empty cname pattern at index %d", i, j) + } + } + + if strings.TrimSpace(fp.Fingerprint) == "" { + return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) + } + + fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + } + + return nil +} + +func validateNSFingerprints(fingerprints []NSProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("ns fingerprint[%d] missing name", i) + } + + if len(fp.HostPatterns) == 0 { + return fmt.Errorf("ns fingerprint[%d] has no host patterns", i) + } + + for j, pattern := range fp.HostPatterns { + if strings.TrimSpace(pattern) == "" { + return fmt.Errorf("ns fingerprint[%d] has empty host pattern at index %d", i, j) + } + } + } + + return nil +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index b7098341a2..d1491995cd 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -3,12 +3,23 @@ package detect import "strings" type CNAMEProviderFingerprint struct { - Cname []string - Name string - Nxdomain bool - Fingerprint string + Cname []string `json:"cname"` + Name string `json:"name"` + Nxdomain bool `json:"nxdomain"` + Fingerprint string `json:"fingerprint"` + Mode FingerprintMode `json:"mode,omitempty"` } +type FingerprintMode string + +const ( + FingerprintModeLiteral FingerprintMode = "literal" + FingerprintModeRegex FingerprintMode = "regex" +) + +var CNAMEProviderFingerprints []CNAMEProviderFingerprint +var NSProviderFingerprints []NSProviderFingerprint + type NSProviderStatus string const ( @@ -20,12 +31,12 @@ const ( ) type NSProviderFingerprint struct { - Name string - ProviderURL string - Status NSProviderStatus - HostPatterns []string - InstructionsURL string - PrivateDNS bool + Name string `json:"name"` + ProviderURL string `json:"provider_url"` + Status NSProviderStatus `json:"status"` + HostPatterns []string `json:"host_patterns"` + InstructionsURL string `json:"instructions_url,omitempty"` + PrivateDNS bool `json:"private_dns,omitempty"` } func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { @@ -75,285 +86,6 @@ func wildcardHostMatch(pattern, host string) bool { return true } -var NSProviderFingerprints = []NSProviderFingerprint{ - { - Name: "000Domains", - ProviderURL: "https://000domains.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", - }, - { - Name: "AWS Route 53", - ProviderURL: "https://aws.amazon.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", - }, - { - Name: "Azure (Microsoft)", - ProviderURL: "https://azure.microsoft.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", - }, - { - Name: "BigCommerce", - ProviderURL: "https://bigcommerce.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", - }, - { - Name: "Bizland", - ProviderURL: "https://bizland.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", - }, - { - Name: "ClouDNS", - ProviderURL: "https://cloudns.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.cloudns.net"}, - }, - { - Name: "Cloudflare", - ProviderURL: "https://cloudflare.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.ns.cloudflare.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", - }, - { - Name: "Digital Ocean", - ProviderURL: "https://digitalocean.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", - }, - { - Name: "DNSMadeEasy", - ProviderURL: "https://dnsmadeeasy.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns*.dnsmadeeasy.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", - }, - { - Name: "DNSimple", - ProviderURL: "https://dnsimple.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", - }, - { - Name: "Domain.com", - ProviderURL: "https://domain.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", - }, - { - Name: "DomainPeople", - ProviderURL: "https://domainpeople.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", - }, - { - Name: "Dotster", - ProviderURL: "https://dotster.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", - }, - { - Name: "Dreamhost", - ProviderURL: "https://dreamhost.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", - }, - { - Name: "EasyDNS", - ProviderURL: "https://easydns.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", - }, - { - Name: "Gandi.net", - ProviderURL: "https://gandi.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, - }, - { - Name: "Google Cloud", - ProviderURL: "https://cloud.google.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns-cloud-*.googledomains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", - }, - { - Name: "Hostinger (old NS)", - ProviderURL: "https://hostinger.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, - }, - { - Name: "Hover", - ProviderURL: "https://hover.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", - }, - { - Name: "Hurricane Electric", - ProviderURL: "https://dns.he.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", - }, - { - Name: "Linode", - ProviderURL: "https://linode.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", - }, - { - Name: "MediaTemple (mt)", - ProviderURL: "https://mediatemple.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", - }, - { - Name: "MyDomain", - ProviderURL: "https://mydomain.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", - }, - { - Name: "Name.com", - ProviderURL: "https://name.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, - InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", - }, - { - Name: "Namecheap", - ProviderURL: "https://namecheap.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, - }, - { - Name: "Network Solutions", - ProviderURL: "https://networksolutions.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.worldnic.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", - }, - { - Name: "NS1", - ProviderURL: "https://nsone.net/", - Status: NSStatusRegistrationClosed, - HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", - }, - { - Name: "TierraNet", - ProviderURL: "https://tierra.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", - }, - { - Name: "Reg.ru", - ProviderURL: "https://reg.ru/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", - }, - { - Name: "UltraDNS", - ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", - }, - { - Name: "Yahoo Small Business", - ProviderURL: "https://yahoosmallbusiness.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", - }, - { - Name: "Activision", - ProviderURL: "https://activision.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.activision.com"}, - PrivateDNS: true, - }, - { - Name: "Adobe", - ProviderURL: "https://adobe.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"adobe-dns-0*.adobe.com"}, - PrivateDNS: true, - }, - { - Name: "Apple", - ProviderURL: "https://apple.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, - PrivateDNS: true, - }, - { - Name: "Automattic", - ProviderURL: "https://automattic.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.automattic.com"}, - PrivateDNS: true, - }, - { - Name: "Capital One", - ProviderURL: "https://capitalone.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.capitalone.com"}, - PrivateDNS: true, - }, - { - Name: "Disney", - ProviderURL: "https://disney.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, - PrivateDNS: true, - }, - { - Name: "Google", - ProviderURL: "https://google.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.google.com"}, - PrivateDNS: true, - }, - { - Name: "Lowe's", - ProviderURL: "https://lowes.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"authns*.lowes.com"}, - PrivateDNS: true, - }, - { - Name: "T-Mobile", - ProviderURL: "https://tmobileus.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, - PrivateDNS: true, - }, -} - func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { @@ -363,230 +95,14 @@ func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { return false } -var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ - { - Cname: []string{ - "elasticbeanstalk.com", - }, - Nxdomain: true, - Name: "AWS/Elastic Beanstalk", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "s3.amazonaws.com", - }, - Nxdomain: false, - Name: "AWS/S3", - Fingerprint: "The specified bucket does not exist", - }, - { - Cname: []string{ - "agilecrm.com", - }, - Nxdomain: false, - Name: "Agile CRM", - Fingerprint: "Sorry, this page is no longer available.", - }, - { - Cname: []string{ - "airee.ru", - }, - Nxdomain: false, - Name: "Airee.ru", - Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", - }, - { - Cname: []string{ - "animaapp.io", - }, - Nxdomain: false, - Name: "Anima", - Fingerprint: "The page you were looking for does not exist.", - }, - { - Cname: []string{ - "bitbucket.io", - }, - Nxdomain: false, - Name: "Bitbucket", - Fingerprint: "Repository not found", - }, - { - Cname: []string{ - "trydiscourse.com", - }, - Nxdomain: true, - Name: "Discourse", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "furyns.com", - }, - Nxdomain: false, - Name: "Gemfury", - Fingerprint: "404: This page could not be found.", - }, - { - Cname: []string{ - "ghost.io", - }, - Nxdomain: false, - Name: "Ghost", - Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", - }, - { - Cname: []string{ - "hatenablog.com", - }, - Nxdomain: false, - Name: "HatenaBlog", - Fingerprint: "404 Blog is not found", - }, - { - Cname: []string{ - "helpjuice.com", - }, - Nxdomain: false, - Name: "Help Juice", - Fingerprint: "We could not find what you're looking for.", - }, - { - Cname: []string{ - "helpscoutdocs.com", - }, - Nxdomain: false, - Name: "Help Scout", - Fingerprint: "No settings were found for this company:", - }, - { - Cname: []string{ - "helprace.com", - }, - Nxdomain: false, - Name: "Helprace", - Fingerprint: "HTTP_STATUS=301", - }, - { - Cname: []string{ - "youtrack.cloud", - }, - Nxdomain: false, - Name: "JetBrains", - Fingerprint: "is not a registered InCloud YouTrack", - }, - { - Cname: []string{ - "launchrock.com", - }, - Nxdomain: false, - Name: "LaunchRock", - Fingerprint: "HTTP_STATUS=500", - }, - { - Cname: []string{ - "cloudapp.azure.com", - "azurewebsites.net", - "blob.core.windows.net", - "cloudapp.azure.com", - "azure-api.net", - "azurehdinsight.net", - "azureedge.net", - "azurecontainer.io", - "database.windows.net", - "azuredatalakestore.net", - "search.windows.net", - "azurecr.io", - "redis.cache.windows.net", - "azurehdinsight.net", - "servicebus.windows.net", - "trafficmanager.net", - "visualstudio.com", - }, - Nxdomain: true, - Name: "Microsoft Azure", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "ngrok.io", - }, - Nxdomain: false, - Name: "Ngrok", - Fingerprint: "Tunnel .*.ngrok.io not found", - }, - { - Cname: []string{ - "readme.io", - }, - Nxdomain: false, - Name: "Readme.io", - Fingerprint: "The creators of this project are still working on making everything perfect!", - }, - { - Cname: []string{ - "52.16.160.97", - }, - Nxdomain: false, - Name: "SmartJobBoard", - Fingerprint: "This job board website is either expired or its domain name is invalid.", - }, - { - Cname: []string{ - "s.strikinglydns.com", - }, - Nxdomain: false, - Name: "Strikingly", - Fingerprint: "PAGE NOT FOUND.", - }, - { - Cname: []string{ - "na-west1.surge.sh", - }, - Nxdomain: false, - Name: "Surge.sh", - Fingerprint: "project not found", - }, - { - Cname: []string{ - "surveysparrow.com", - }, - Nxdomain: false, - Name: "SurveySparrow", - Fingerprint: "Account not found.", - }, - { - Cname: []string{ - "read.uberflip.com", - }, - Nxdomain: false, - Name: "Uberflip", - Fingerprint: "The URL you've accessed does not provide a hub.", - }, - { - Cname: []string{ - "stats.uptimerobot.com", - }, - Nxdomain: false, - Name: "Uptimerobot", - Fingerprint: "page not found", - }, - { - Cname: []string{ - "wordpress.com", - }, - Nxdomain: false, - Name: "Wordpress", - Fingerprint: "Do you want to register .*.wordpress.com?", - }, - { - Cname: []string{ - "worksites.net", - "69.164.223.206", - }, - Nxdomain: false, - Name: "Worksites", - Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", - }, +func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { + if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { + return mode + } + + if strings.Contains(fingerprint, ".*") || strings.Contains(fingerprint, "\\") { + return FingerprintModeRegex + } + + return FingerprintModeLiteral } From 64e789c7376985bcdebebfcd4b0e21d5a8c5f0b6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:51 -0300 Subject: [PATCH 22/45] remove remediation.go file to serve translated text in api/frontend --- scanners/subdomain-takeover/internal/detect/remediation.go | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go deleted file mode 100644 index 9d577baa7f..0000000000 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ /dev/null @@ -1,5 +0,0 @@ -package detect - -func RemediationForReason(reasonCode string, provider string, lameType string) { - return -} From 2c873aba90bd4c635c5ef6fef467c97388bd27f6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:19:47 -0300 Subject: [PATCH 23/45] add RecordType enums --- scanners/subdomain-takeover/internal/model/finding.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 5d387c0f1d..57dbc83c67 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -1,9 +1,16 @@ package model +type RecordType string + +const ( + RecordTypeCNAME RecordType = "CNAME" + RecordTypeNS RecordType = "NS" +) + type Finding struct { Domain string `json:"domain"` DomainKey string `json:"domain_key"` - RecordType string `json:"record_type"` + RecordType RecordType `json:"record_type"` Target string `json:"target"` Provider string `json:"provider"` LameType string `json:"lame_type"` From f405a61604273509af31c3c86e0c547df5cd0695 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:11 -0300 Subject: [PATCH 24/45] add confidence and reason code enums --- .../internal/detect/confidence.go | 32 +++++++++++++++++++ .../internal/detect/reason_codes.go | 15 +++++++++ 2 files changed, 47 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/reason_codes.go diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index ad6c03d18d..9865a561db 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -1 +1,33 @@ package detect + +const ( + ConfidenceSuspected = "suspected" + ConfidenceProbable = "probable" + ConfidenceConfirmed = "confirmed" +) + +// ConfidenceForReason maps stable reason codes to confidence levels. +// Unknown reason codes default to suspected. +func ConfidenceForReason(reasonCode ReasonCode) string { + switch reasonCode { + case ReasonCNAMEDanglingNXDOMAIN: + return ConfidenceProbable + case ReasonCNAMEProviderFingerprintBodyMatch: + return ConfidenceProbable + case ReasonCNAMETargetMatchMissingNXDOMAIN: + return ConfidenceSuspected + case ReasonCNAMETargetMatchMissingBodyFP: + return ConfidenceSuspected + + case ReasonNSFullLameProviderVulnerable: + return ConfidenceConfirmed + case ReasonNSPartialLameProviderVulnerable: + return ConfidenceProbable + case ReasonNSLameProviderUnknown: + return ConfidenceSuspected + case ReasonNSProviderMatchOnly: + return ConfidenceSuspected + default: + return ConfidenceSuspected + } +} diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go new file mode 100644 index 0000000000..f080b18329 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -0,0 +1,15 @@ +package detect + +type ReasonCode string + +const ( + ReasonCNAMEDanglingNXDOMAIN ReasonCode = "CNAME_DANGLING_NXDOMAIN" + ReasonCNAMEProviderFingerprintBodyMatch ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + ReasonCNAMETargetMatchMissingNXDOMAIN ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" + ReasonCNAMETargetMatchMissingBodyFP ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + + ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" + ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" + ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" +) From d18030718304344f9b109a458eea3bab09c5f0a2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:45 -0300 Subject: [PATCH 25/45] move http interface for NS lookups to own file --- .../internal/detect/body_matcher.go | 65 +++++++++++++++++++ .../internal/detect/rules.go | 48 +++----------- 2 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go new file mode 100644 index 0000000000..622b2f7c51 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -0,0 +1,65 @@ +package detect + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +type BodyFingerprintMatcher interface { + Contains(domain string, fingerprint string, mode FingerprintMode) bool +} + +type HTTPBodyFingerprintMatcher struct { + client *http.Client +} + +func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { + return &HTTPBodyFingerprintMatcher{ + client: &http.Client{Timeout: timeout}, + } +} + +type NoopBodyFingerprintMatcher struct{} + +func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { + return &NoopBodyFingerprintMatcher{} +} + +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + return false +} + +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if m == nil || m.client == nil { + return false + } + + url := fmt.Sprintf("http://%s", domain) + res, err := m.client.Get(url) + if err != nil { + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false + } + + bodyText := string(body) + resolvedMode := normalizeFingerprintMode(mode, fingerprint) + + if resolvedMode == FingerprintModeRegex { + re, err := regexp.Compile(fingerprint) + if err != nil { + return false + } + return re.MatchString(bodyText) + } + + return strings.Contains(bodyText, fingerprint) +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 9cb5ac5738..44626a372b 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,28 +1,21 @@ package detect -import ( - "fmt" - "io" - "net/http" - "strings" -) - type CNAMEHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode NeedsNX bool } type NSHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode ProviderVulnerable bool RegistrarMismatch bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { hit := CNAMEHit{ @@ -32,16 +25,17 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider NeedsNX: fp.Nxdomain, } if hit.NeedsNX { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" - if evidence.QueryAnswer.A == "NXDOMAIN" { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { hit.Matched = true - hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN } } else { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" - if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true - hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } return &hit @@ -67,25 +61,3 @@ func ShouldEmitCNAME(hit *CNAMEHit) bool { func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } - -func urlBodyContainsFingerprint(domain string, fp string) bool { - // curl domain and look for matching fp in body - url := fmt.Sprintf("http://%s", domain) - res, err := http.Get(url) - if err != nil { - // log error - return false - } - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - // log err - return false - } - - if strings.Contains(string(body), fp) { - return true - } - return false -} From ad7c2729c124905f932f7eb211e733b7edfb0a59 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:21:55 -0300 Subject: [PATCH 26/45] switch from using QueryAnswers struct field to NoResolve bool for CNAME evidence --- .../internal/detect/evidence.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index fb40318cfc..ba54926bb8 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -8,9 +8,9 @@ import ( // inputs from evidence.go type CNAMEEvidence struct { - Domain string - Target string - QueryAnswer model.QueryAnswers // or just needed fields + Domain string + Target string + NoResolve bool } type NSEvidence struct { @@ -26,9 +26,9 @@ func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { } return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - QueryAnswer: input.QueryAnswers, + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + NoResolve: input.ResolveChain == nil, } } @@ -46,6 +46,6 @@ func ClassifyLameType(nsChecks []model.NsCheck) { func parseCname(record string) string { trimmed := strings.Trim(record, ".") lower := strings.ToLower(trimmed) - record_tokens := strings.Split(lower, " ") - return record_tokens[len(record_tokens)-1] + recordTokens := strings.Split(lower, " ") + return recordTokens[len(recordTokens)-1] } From 967a0882a87a96e7a8499551667342aa36e65010 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:05 -0300 Subject: [PATCH 27/45] add contructor for Classifier --- .../internal/detect/classifier.go | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 0caea61196..28fb59649b 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,33 +2,48 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" -func Classify(input model.Input) ([]model.Finding, error) { +type Classifier struct { + Matcher BodyFingerprintMatcher +} + +func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { + return &Classifier{Matcher: matcher} +} + +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { + matcher := c.Matcher + if matcher == nil { + matcher = NewNoopBodyFingerprintMatcher() + } + + return Classify(input, matcher) +} + +func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} cnameEvidence := ExtractCNAMEEvidence(input) - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(cnameHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: cnameEvidence.Domain, - RecordType: "CNAME", - Target: cnameEvidence.Target, - Provider: cnameHit.Provider, - ReasonCode: cnameHit.ReasonCode, - Confidence: "", - Remediation: "", - }) + if cnameEvidence != nil { + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + if ShouldEmitCNAME(cnameHit) { + findings = append(findings, model.Finding{ + Domain: cnameEvidence.Domain, + RecordType: model.RecordTypeCNAME, + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: string(cnameHit.ReasonCode), + Confidence: ConfidenceForReason(cnameHit.ReasonCode), + }) + } } - nsEvidence := ExtractNSEvidence(input) - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: input.Domain, - }) - } + // nsEvidence := ExtractNSEvidence(input) + // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + // if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // findings = append(findings, model.Finding{ + // Domain: input.Domain, + // }) + // } + return findings, nil } From 92a05793c311af658b10577d8c418277e8d1ff90 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:15 -0300 Subject: [PATCH 28/45] update README.md --- scanners/subdomain-takeover/README.md | 117 +++++++++----------------- 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0bfcdaf9ba..640b246776 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,10 +5,17 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +2. Evaluates passive CNAME takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` +## Current detection scope + +- Implemented: CNAME-based passive detection +- Not implemented yet: NS-based (Sitting Duck) detection + +The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. + ## Passive-only detection policy This service uses passive checks only. @@ -19,63 +26,36 @@ As a result, confidence is conservative and evidence-driven: - `probable`: strong passive indicators with known exploitable conditions - `confirmed`: reserved for deterministic passive signatures only -## Sitting Duck DNS hijack coverage +## Confidence model (current) -The scanner includes passive detection for Sitting Duck style DNS hijack risk. +Current production behavior for CNAME findings: -A Sitting Duck takeover is considered possible when all of the following are true: - -1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. -2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). -3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. +- `suspected` + - provider target matched, but required passive signal is missing +- `probable` + - dangling target with NXDOMAIN where required, or + - provider fingerprint body match for providers that require body verification +- `confirmed` + - reserved; not currently emitted by CNAME rules -### Variations covered +## Fingerprint data -- Partially lame delegation: some NS records are lame, others still answer. -- Full lame delegation: all authoritative NS are lame. -- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. +Provider fingerprint data is stored in: -If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. +- `internal/detect/data/cname_fingerprints.json` +- `internal/detect/data/ns_fingerprints.json` -## Confidence model +The service embeds and validates these files at startup (`detect.LoadFingerprints`). -This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. +For CNAME body matching: -- `suspected` - - dangling/suspicious DNS pattern only, or - - lame delegation evidence without known exploitable-provider confirmation -- `probable` - - DNS pattern + known provider unclaimed signature, or - - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match -- `confirmed` - - explicit passive takeover indicator with deterministic evidence - - if deterministic passive proof is missing, cap at `probable` - -## Required evidence for findings - -Each finding should include: -- `domain` -- `domain_key` -- `record_type` (`CNAME` or `NS`) -- `target` or `ns_host` -- `provider` -- `lame_type` (`partial` or `full`) for Sitting Duck findings -- per-NS resolver outcomes where applicable -- `confidence` -- `reason_code` -- remediation guidance - -## Remediation guidance - -Typical remediation: -1. Fix delegation at registrar and authoritative DNS provider. -2. Remove stale or dead NS entries. -3. Reclaim or securely rebind abandoned DNS zones or accounts. -4. Validate ownership controls for DNS provider account workflows. +- `mode: literal` performs plain substring search +- `mode: regex` performs regular expression matching +- if `mode` is omitted, the loader applies backward-compatible inference ## Prerequisites -- Go 1.24+ +- Go 1.25+ - NATS with JetStream enabled - `nats` CLI (optional, for local testing) @@ -95,43 +75,24 @@ Environment variables: - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) -Local test flow: - -Publish a test request: -```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' -``` +## Message ack behavior -Watch findings: -```bash -nats sub "scans.findings.upsert" -``` +The worker uses explicit JetStream ack semantics: -## Development milestones +- decode failures -> `Term()` (drop poison message) +- classify/publish failures -> `Nak()` (retryable) +- successful processing -> `Ack()` -1. Build config + NATS consume/publish loop. -2. Add DNS evidence extraction. -3. Add provider fingerprint rule engine. -4. Add confidence scoring and remediation text. -5. Add tests for classification rules and message handling. +## Local test flow ---- +Publish a test request: -## External Go modules to install +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +``` -Start minimal: +Watch findings: ```bash -go get github.com/nats-io/nats.go -go get github.com/miekg/dns -go get github.com/rs/zerolog -go get golang.org/x/sync/errgroup -go get github.com/stretchr/testify +nats sub "scans.findings.upsert" ``` - -What each is for: -- `nats.go`: JetStream consume/publish -- `miekg/dns`: robust DNS lookups and record parsing -- `zerolog`: structured logs without heavy setup -- `errgroup`: clean goroutine lifecycle management -- `testify`: easier assertions in unit tests From 9ba9e1fd9bebf2e4aa7390f46a51849e7ad686e2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:04:05 -0300 Subject: [PATCH 29/45] remove unneeded scan result fixtures --- .../fixtures/cyber-gc-ca_dns_scan.json | 202 ------------------ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 -- 2 files changed, 225 deletions(-) delete mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json delete mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json deleted file mode 100644 index cad720cc93..0000000000 --- a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "domain": "cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - [ - "cyber.gc.ca. 7161 IN A 20.151.96.73" - ] - ], - "resolve_ips": [ - "20.151.96.73" - ], - "cname_record": null, - "mx_records": { - "hosts": [ - { - "preference": 10, - "hostname": "cyber-gc-ca.mail.protection.outlook.com", - "addresses": [ - "52.101.190.1", - "52.101.192.0", - "52.101.192.1", - "52.101.190.0" - ] - } - ], - "warnings": [] - }, - "ns_records": { - "hostnames": [ - "ns1.d-zone.ca", - "ns2.d-zone.ca", - "ns40.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns10.ent.global.gc.ca" - ], - "warnings": [] - }, - "dkim": {}, - "spf": { - "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", - "valid": true, - "dns_void_lookups": 0, - "warnings": [], - "parsed": { - "pass": [ - { - "value": "40.80.253.57", - "mechanism": "ip4" - }, - { - "value": "52.139.3.76", - "mechanism": "a" - }, - { - "value": "205.193.218.114", - "mechanism": "a" - }, - { - "value": "205.193.218.115", - "mechanism": "a" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [ - { - "domain": "spf.protection.outlook.com", - "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", - "dns_void_lookups": 0, - "parsed": { - "pass": [ - { - "value": "40.92.0.0/15", - "mechanism": "ip4" - }, - { - "value": "40.107.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.100.0.0/15", - "mechanism": "ip4" - }, - { - "value": "52.102.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.103.0.0/17", - "mechanism": "ip4" - }, - { - "value": "104.47.0.0/17", - "mechanism": "ip4" - }, - { - "value": "2a01:111:f400::/48", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403::/49", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:8000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:c000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:f000::/52", - "mechanism": "ip6" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "warnings": [], - "spf_default": "fail", - "lookups": 0 - } - ], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "spf_default": "fail", - "lookups": 3 - }, - "dmarc": { - "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", - "valid": true, - "location": "cyber.gc.ca", - "warnings": [], - "tags": { - "v": { - "value": "DMARC1", - "explicit": true - }, - "p": { - "value": "reject", - "explicit": true - }, - "pct": { - "value": 100, - "explicit": true - }, - "rua": { - "value": [ - { - "scheme": "mailto", - "address": "dmarc@cyber.gc.ca", - "size_limit": null, - "accepting": true - } - ], - "explicit": true - }, - "adkim": { - "value": "r", - "explicit": false - }, - "aspf": { - "value": "r", - "explicit": false - }, - "fo": { - "value": "0", - "explicit": false - }, - "rf": { - "value": "afrf", - "explicit": false - }, - "ri": { - "value": 86400, - "explicit": false - }, - "sp": { - "value": "reject", - "explicit": false - } - }, - "effective_policy_source": "p", - "effective_policy": "reject" - }, - "wildcard_sibling": false, - "wildcard_entry": false, - "zone_dnssec_enabled": null -} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json deleted file mode 100644 index 8f546a74a5..0000000000 --- a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "domain": "www.cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "www.cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], - ["cyber.gc.ca. 5550 IN A 20.151.96.73"] - ], - "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", - "ns_records": { - "hostnames": [ - "ns40.ent.global.gc.ca", - "ns10.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns2.d-zone.ca", - "ns1.d-zone.ca" - ], - "warnings": [] - } -} From 16dc260e13ba15f1899b3403ea1799ad898e3cf0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:05:11 -0300 Subject: [PATCH 30/45] update model.input to take in published dns_scanner_results payload --- .../internal/detect/classifier.go | 3 +- .../internal/detect/evidence.go | 17 ++++---- .../internal/messaging/consumer.go | 1 - .../internal/model/finding.go | 17 ++++---- .../internal/model/input.go | 41 +++++++++++-------- 5 files changed, 44 insertions(+), 35 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 28fb59649b..5d3e3ad110 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -22,12 +22,13 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} - cnameEvidence := ExtractCNAMEEvidence(input) + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, + DomainKey: input.DomainKey, RecordType: model.RecordTypeCNAME, Target: cnameEvidence.Target, Provider: cnameHit.Provider, diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index ba54926bb8..f035d01c76 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -20,26 +20,29 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { - if input.CnameRecord == nil { +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil { return nil } + domain := results.Domain + return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - NoResolve: input.ResolveChain == nil, + Domain: *domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, } } func ExtractNSEvidence(input model.Input) *NSEvidence { - if len(input.NsDelegations.Hosts) == 0 { + nsDelegations := input.Results.NsDelegations + if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } return &NSEvidence{} } -func ClassifyLameType(nsChecks []model.NsCheck) { +func ClassifyLameType(nsChecks []any) { return } diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go deleted file mode 100644 index cbb7a43f8b..0000000000 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ /dev/null @@ -1 +0,0 @@ -package messaging diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 57dbc83c67..66e98a2f35 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -8,13 +8,12 @@ const ( ) type Finding struct { - Domain string `json:"domain"` - DomainKey string `json:"domain_key"` - RecordType RecordType `json:"record_type"` - Target string `json:"target"` - Provider string `json:"provider"` - LameType string `json:"lame_type"` - Confidence string `json:"confidence"` - ReasonCode string `json:"reason_code"` - Remediation string `json:"remediation"` + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType RecordType `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 7bdd179f73..263e74928d 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,34 +1,30 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` - QueryAnswers QueryAnswers `json:"query_res"` - NsDelegations NsDelegations `json:"ns_delegations"` + DomainKey string `json:"domain_key"` + Results ScanResults `json:"results"` +} + +type ScanResults struct { + Domain *string `json:"domain"` + NsDelegations *NsDelegations `json:"ns_delegations"` + RegistrarContext *RegistrarContext `json:"registrar_context"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords *NsRecords `json:"ns_records"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` -} - -type QueryAnswers struct { - A string `json:"A"` - SOA string `json:"SOA"` - NS string `json:"NS"` - CNAME string `json:"CNAME"` + Error string `json:"error"` } type NsDelegations struct { Hosts []string `json:"ns_hosts"` Checks []NsCheck `json:"ns_checks"` Delegation Delegation `json:"ns_delegation"` + Error string `json:"error"` } type NsCheck struct { @@ -46,3 +42,14 @@ type Delegation struct { LameCount int `json:"lame_count"` LameType string `json:"lame_type"` } + +type RegistrarContext struct { + BaseDomain string `json:"base_domain"` + LookupSuccess bool `json:"lookup_success"` + RDAPURL string `json:"rdap_url"` + RegistrarName string `json:"registrar_name"` + RegistrarID string `json:"registrar_id"` + RDAPNameservers []string `json:"rdap_nameservers"` + DelegationMatchesRDAP *bool `json:"delegation_matches_rdap"` + Error string `json:"error"` +} From 50d653a6728c11ab11f96a22d282b9fbd443ef52 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:15:22 -0300 Subject: [PATCH 31/45] add logic to emit nameserver hijacking exploitability --- .../internal/detect/classifier.go | 26 +++-- .../internal/detect/evidence.go | 42 +++++--- .../internal/detect/rules.go | 95 ++++++++++++++++--- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5d3e3ad110..c3883063a0 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1,6 +1,8 @@ package detect -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) type Classifier struct { Matcher BodyFingerprintMatcher @@ -38,13 +40,21 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin } } - // nsEvidence := ExtractNSEvidence(input) - // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - // if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // findings = append(findings, model.Finding{ - // Domain: input.Domain, - // }) - // } + nsEvidence := ExtractNSEvidence(input.Results) + if nsEvidence != nil { + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(nsHit) { + findings = append(findings, model.Finding{ + Domain: nsEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeNS, + Target: nsHit.Host, + Provider: nsHit.Provider, + ReasonCode: string(nsHit.ReasonCode), + Confidence: ConfidenceForReason(nsHit.ReasonCode), + }) + } + } return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f035d01c76..7ea527de08 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,6 +1,7 @@ package detect import ( + "slices" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" @@ -21,34 +22,47 @@ type NSEvidence struct { } func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil { + if results.CnameRecord == nil || results.Domain == nil { return nil } - domain := results.Domain - return &CNAMEEvidence{ - Domain: *domain, + Domain: *results.Domain, Target: parseCname(*results.CnameRecord), NoResolve: len(results.ResolveChain) == 0, } } -func ExtractNSEvidence(input model.Input) *NSEvidence { - nsDelegations := input.Results.NsDelegations - if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { +func ExtractNSEvidence(input model.ScanResults) *NSEvidence { + nsDelegations := input.NsDelegations + if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } - return &NSEvidence{} -} -func ClassifyLameType(nsChecks []any) { - return + return &NSEvidence{ + Domain: *input.Domain, + NSHosts: parseHostnames(input.NsDelegations.Hosts), + NSDelegations: *input.NsDelegations, + } + } func parseCname(record string) string { - trimmed := strings.Trim(record, ".") - lower := strings.ToLower(trimmed) - recordTokens := strings.Split(lower, " ") + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") return recordTokens[len(recordTokens)-1] } + +func parseHostnames(hosts []string) []string { + var normalizedHosts = []string{} + for _, host := range hosts { + normalHost := normalizeHost(host) + normalizedHosts = append(normalizedHosts, normalHost) + } + slices.Sort(normalizedHosts) + return slices.Compact(normalizedHosts) +} + +func normalizeHost(host string) string { + return strings.Trim(strings.ToLower(host), ".") +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 44626a372b..fe08aabc33 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,5 +1,7 @@ package detect +import "strings" + type CNAMEHit struct { Matched bool Provider string @@ -8,11 +10,11 @@ type CNAMEHit struct { } type NSHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - ProviderVulnerable bool - RegistrarMismatch bool + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool } func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { @@ -24,6 +26,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider ReasonCode: "", NeedsNX: fp.Nxdomain, } + if hit.NeedsNX { hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN if evidence.NoResolve { @@ -38,6 +41,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } + return &hit } } @@ -45,19 +49,88 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return &NSHit{} + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best } func ShouldEmitCNAME(hit *CNAMEHit) bool { if hit == nil { return false } - if hit.Matched { - return true + return hit.Matched +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly } - return false } -func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { - return false +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } } From e20f1ea935bc89a3ad1a26e202b0c7b35badbd3c Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:20:24 -0300 Subject: [PATCH 32/45] refactor rules.go into separate cname and ns detection logic files --- .../internal/detect/classifier.go | 27 +++- .../internal/detect/cname_evidence.go | 31 ++++ .../internal/detect/cname_rules.go | 46 ++++++ .../internal/detect/fingerprint_source.go | 16 +++ .../detect/{evidence.go => ns_evidence.go} | 32 +---- .../internal/detect/ns_reasoning.go | 47 ++++++ .../internal/detect/ns_rules.go | 45 ++++++ .../internal/detect/rules.go | 136 ------------------ 8 files changed, 210 insertions(+), 170 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/cname_evidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source.go rename scanners/subdomain-takeover/internal/detect/{evidence.go => ns_evidence.go} (55%) create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules.go delete mode 100644 scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index c3883063a0..d3853749fd 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -6,10 +6,19 @@ import ( type Classifier struct { Matcher BodyFingerprintMatcher + Source FingerprintSource } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} +} + +func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { + if source == nil { + source = GlobalFingerprintSource{} + } + + return &Classifier{Matcher: matcher, Source: source} } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -18,15 +27,23 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { matcher = NewNoopBodyFingerprintMatcher() } - return Classify(input, matcher) + source := c.Source + if source == nil { + source = GlobalFingerprintSource{} + } + + return Classify(input, matcher, source) } -func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { findings := []model.Finding{} + cnameFingerprints := source.CNAME() + nsFingerprints := source.NS() + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, @@ -42,7 +59,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, diff --git a/scanners/subdomain-takeover/internal/detect/cname_evidence.go b/scanners/subdomain-takeover/internal/detect/cname_evidence.go new file mode 100644 index 0000000000..4466dcd9fc --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type CNAMEEvidence struct { + Domain string + Target string + NoResolve bool +} + +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil || results.Domain == nil { + return nil + } + + return &CNAMEEvidence{ + Domain: *results.Domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, + } +} + +func parseCname(record string) string { + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") + return recordTokens[len(recordTokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go new file mode 100644 index 0000000000..e69c5cddb0 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -0,0 +1,46 @@ +package detect + +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode ReasonCode + NeedsNX bool +} + +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + + if hit.NeedsNX { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN + } + } else { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + } + } + + return &hit + } + } + return nil +} + +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go new file mode 100644 index 0000000000..2ecc9981f8 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -0,0 +1,16 @@ +package detect + +type FingerprintSource interface { + CNAME() []CNAMEProviderFingerprint + NS() []NSProviderFingerprint +} + +type GlobalFingerprintSource struct{} + +func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { + return CNAMEProviderFingerprints +} + +func (GlobalFingerprintSource) NS() []NSProviderFingerprint { + return NSProviderFingerprints +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go similarity index 55% rename from scanners/subdomain-takeover/internal/detect/evidence.go rename to scanners/subdomain-takeover/internal/detect/ns_evidence.go index 7ea527de08..78ddb2f428 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -7,13 +7,6 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -// inputs from evidence.go -type CNAMEEvidence struct { - Domain string - Target string - NoResolve bool -} - type NSEvidence struct { Domain string NSHosts []string @@ -21,18 +14,6 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil || results.Domain == nil { - return nil - } - - return &CNAMEEvidence{ - Domain: *results.Domain, - Target: parseCname(*results.CnameRecord), - NoResolve: len(results.ResolveChain) == 0, - } -} - func ExtractNSEvidence(input model.ScanResults) *NSEvidence { nsDelegations := input.NsDelegations if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { @@ -44,21 +25,14 @@ func ExtractNSEvidence(input model.ScanResults) *NSEvidence { NSHosts: parseHostnames(input.NsDelegations.Hosts), NSDelegations: *input.NsDelegations, } - -} - -func parseCname(record string) string { - normalRecord := normalizeHost(record) - recordTokens := strings.Split(normalRecord, " ") - return recordTokens[len(recordTokens)-1] } func parseHostnames(hosts []string) []string { - var normalizedHosts = []string{} + normalizedHosts := make([]string, 0, len(hosts)) for _, host := range hosts { - normalHost := normalizeHost(host) - normalizedHosts = append(normalizedHosts, normalHost) + normalizedHosts = append(normalizedHosts, normalizeHost(host)) } + slices.Sort(normalizedHosts) return slices.Compact(normalizedHosts) } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go new file mode 100644 index 0000000000..953cee5580 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -0,0 +1,47 @@ +package detect + +import "strings" + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly + } +} + +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go new file mode 100644 index 0000000000..ff993ef28b --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -0,0 +1,45 @@ +package detect + +type NSHit struct { + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool +} + +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go deleted file mode 100644 index fe08aabc33..0000000000 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ /dev/null @@ -1,136 +0,0 @@ -package detect - -import "strings" - -type CNAMEHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - NeedsNX bool -} - -type NSHit struct { - Matched bool - Host string - Provider string - ReasonCode ReasonCode - RegistrarMismatch bool -} - -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { - for _, fp := range fingerprints { - if fp.ContainsTarget(evidence.Target) { - hit := CNAMEHit{ - Matched: false, - Provider: fp.Name, - ReasonCode: "", - NeedsNX: fp.Nxdomain, - } - - if hit.NeedsNX { - hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN - if evidence.NoResolve { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN - } - } else { - hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP - mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) - if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch - } - } - - return &hit - } - } - return nil -} - -func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { - return nil - } - - lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) - var best *NSHit - - for _, host := range evidence.NSHosts { - for _, fp := range fingerprints { - if fp.ContainsNSHost(host) { - reasonCode := getNSHijackReasonCode(lameType, fp.Status) - hit := &NSHit{ - Matched: isNSMatch(reasonCode), - Host: host, - Provider: fp.Name, - ReasonCode: reasonCode, - } - - if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { - best = hit - } - } - } - } - - return best -} - -func ShouldEmitCNAME(hit *CNAMEHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func ShouldEmitNSHijack(hit *NSHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { - switch normalizeLameType(lameType) { - case "full": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSFullLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - case "partial": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSPartialLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - default: - return ReasonNSProviderMatchOnly - } -} - -func isNSMatch(reasonCode ReasonCode) bool { - return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable -} - -func isExploitableProviderStatus(status NSProviderStatus) bool { - return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase -} - -func normalizeLameType(lameType string) string { - return strings.ToLower(strings.TrimSpace(lameType)) -} - -func nsReasonRank(reasonCode ReasonCode) int { - switch reasonCode { - case ReasonNSFullLameProviderVulnerable: - return 4 - case ReasonNSPartialLameProviderVulnerable: - return 3 - case ReasonNSLameProviderUnknown: - return 2 - case ReasonNSProviderMatchOnly: - return 1 - default: - return 0 - } -} From dc360ef065a24ce47a03e86d1e42b7845f26ccd0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:06 -0300 Subject: [PATCH 33/45] add debug logging --- .../subdomain-takeover/cmd/service/main.go | 17 ++++-- .../subdomain-takeover/internal/app/runner.go | 14 ++++- .../subdomain-takeover/internal/app/worker.go | 14 ++++- .../internal/bootstrap/nats.go | 8 ++- .../internal/detect/classifier.go | 60 +++++++++++++++++-- .../internal/detect/cname_rules.go | 31 ++++++++++ .../internal/detect/fingerprint_loader.go | 12 +++- .../internal/detect/logger.go | 9 +++ .../internal/detect/ns_rules.go | 28 ++++++++- .../internal/messaging/publisher.go | 28 ++++++++- 10 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/logger.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 0f588ec30d..78a2500bff 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -24,8 +24,17 @@ func main() { } logger = bootstrap.NewLogger(cfg.LogLevel) - - if err := detect.LoadFingerprints(); err != nil { + logger.Info(). + Str("nats_url", cfg.NATSURL). + Str("stream", cfg.NATSStream). + Str("subject_in", cfg.SubjectIn). + Str("subject_out", cfg.SubjectOut). + Str("durable", cfg.DurableName). + Int("worker_count", cfg.WorkerCount). + Str("log_level", cfg.LogLevel.String()). + Msg("service configuration loaded") + + if err := detect.LoadFingerprints(logger); err != nil { logger.Fatal().Err(err).Msg("failed to load fingerprints") } @@ -36,7 +45,7 @@ func main() { signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sig) - runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg, logger) if err != nil { logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") } @@ -45,7 +54,7 @@ func main() { pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) - classifier := detect.NewClassifier(matcher) + classifier := detect.NewClassifier(matcher).WithLogger(logger) worker := app.NewWorker(logger, pub, classifier) go func() { diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 6d7ee52e8d..b8c0121f87 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -26,11 +26,14 @@ type MessageHandler interface { func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter + nextErrCount := 0 if deps.WorkerCount < 1 { deps.WorkerCount = 1 } + logger.Info().Int("worker_count", deps.WorkerCount).Msg("runner started") + var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) @@ -50,11 +53,18 @@ Loop: msg, err := iter.Next() if err != nil { if ctx.Err() != nil { + logger.Info().Msg("runner stopping: context canceled") break Loop } - logger.Debug().Err(err).Msg("next returned, continuing") + nextErrCount++ + if nextErrCount%10 == 0 { + logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") + } else { + logger.Debug().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("next returned, continuing") + } continue } + nextErrCount = 0 select { case sem <- struct{}{}: @@ -80,6 +90,6 @@ Loop: case <-waitDone: logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") + logger.Warn().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index e282fab952..eca59f91bc 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -30,7 +30,11 @@ func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier Sca } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { - log := w.logger.With().Str("component", "worker").Logger() + log := w.logger.With(). + Str("component", "worker"). + Str("subject", msg.Subject()). + Int("msg_size", len(msg.Data())). + Logger() scan, err := decodeScan(msg.Data()) if err != nil { @@ -39,6 +43,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log = log.With().Str("domain_key", scan.DomainKey).Logger() + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") @@ -46,6 +52,10 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + if len(findings) == 0 { + log.Debug().Msg("classification produced no findings") + } + for _, finding := range findings { err = w.publisher.Publish(ctx, finding) if err != nil { @@ -60,6 +70,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log.Debug().Int("findings_count", len(findings)).Msg("message acknowledged") + return nil } diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go index 51edb6b060..ea2fc49e21 100644 --- a/scanners/subdomain-takeover/internal/bootstrap/nats.go +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -7,6 +7,7 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) type RuntimeDeps struct { @@ -15,18 +16,21 @@ type RuntimeDeps struct { JS jetstream.JetStream } -func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { +func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logger) (*RuntimeDeps, error) { + logger.Debug().Str("nats_url", cfg.NATSURL).Msg("connecting to nats") nc, err := nats.Connect(cfg.NATSURL) if err != nil { return nil, err } + logger.Debug().Msg("creating jetstream client") js, err := jetstream.New(nc) if err != nil { nc.Close() return nil, err } + logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NATSStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, @@ -36,6 +40,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, @@ -45,6 +50,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Msg("creating pull message iterator") iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) if err != nil { nc.Close() diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index d3853749fd..ffd1d527c6 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,15 +2,17 @@ package detect import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" ) type Classifier struct { Matcher BodyFingerprintMatcher Source FingerprintSource + logger zerolog.Logger } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} } func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { @@ -18,7 +20,13 @@ func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintS source = GlobalFingerprintSource{} } - return &Classifier{Matcher: matcher, Source: source} + return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} +} + +func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { + c.logger = logger.With().Str("component", "classifier").Logger() + SetLogger(logger) + return c } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -32,10 +40,15 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { source = GlobalFingerprintSource{} } - return Classify(input, matcher, source) + logger := c.logger + if logger.GetLevel() == zerolog.NoLevel { + logger = zerolog.Nop() + } + + return Classify(input, matcher, source, logger) } -func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} cnameFingerprints := source.CNAME() @@ -43,8 +56,15 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { + logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("emitting cname finding") findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, DomainKey: input.DomainKey, @@ -55,12 +75,34 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr Confidence: ConfidenceForReason(cnameHit.ReasonCode), }) } + if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("cname finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") } nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Int("ns_hosts", len(nsEvidence.NSHosts)). + Msg("ns evidence extracted") nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("emitting ns finding") findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, DomainKey: input.DomainKey, @@ -70,7 +112,17 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr ReasonCode: string(nsHit.ReasonCode), Confidence: ConfidenceForReason(nsHit.ReasonCode), }) + } else if nsHit != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("ns finding suppressed") } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") } return findings, nil diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index e69c5cddb0..b693a4cd70 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -10,6 +10,13 @@ type CNAMEHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("target", evidence.Target). + Str("provider", fp.Name). + Bool("needs_nx", fp.Nxdomain). + Msg("cname provider target matched") + hit := CNAMEHit{ Matched: false, Provider: fp.Name, @@ -26,12 +33,36 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("fingerprint_mode", string(mode)). + Msg("checking cname body fingerprint") if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname body fingerprint matched") + } else { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: body fingerprint missing") } } + if hit.NeedsNX && !hit.Matched { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: required nxdomain signal missing") + } + return &hit } } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go index 694bf833aa..da2101e51a 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" "sync" + + "github.com/rs/zerolog" ) //go:embed data/*.json @@ -16,33 +18,41 @@ var ( loadFingerprintsErr error ) -func LoadFingerprints() error { +func LoadFingerprints(logger zerolog.Logger) error { + log := logger.With().Str("component", "fingerprint_loader").Logger() + loadFingerprintsOnce.Do(func() { var cname []CNAMEProviderFingerprint var ns []NSProviderFingerprint if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := validateCNAMEFingerprints(cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } if err := validateNSFingerprints(ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } CNAMEProviderFingerprints = cname NSProviderFingerprints = ns + + log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") }) return loadFingerprintsErr diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go new file mode 100644 index 0000000000..c3d4a30293 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger.go @@ -0,0 +1,9 @@ +package detect + +import "github.com/rs/zerolog" + +var detectLogger = zerolog.Nop() + +func SetLogger(logger zerolog.Logger) { + detectLogger = logger.With().Str("component", "detect").Logger() +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index ff993ef28b..5485e9cc95 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -5,11 +5,11 @@ type NSHit struct { Host string Provider string ReasonCode ReasonCode - RegistrarMismatch bool } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -20,6 +20,7 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr for _, fp := range fingerprints { if fp.ContainsNSHost(host) { reasonCode := getNSHijackReasonCode(lameType, fp.Status) + rank := nsReasonRank(reasonCode) hit := &NSHit{ Matched: isNSMatch(reasonCode), Host: host, @@ -27,6 +28,17 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr ReasonCode: reasonCode, } + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", host). + Str("provider", fp.Name). + Str("provider_status", string(fp.Status)). + Str("lame_type", lameType). + Str("reason_code", string(reasonCode)). + Int("rank", rank). + Bool("emittable", hit.Matched). + Msg("ns candidate evaluated") + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { best = hit } @@ -34,6 +46,20 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr } } + if best == nil { + detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + return nil + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", best.Host). + Str("provider", best.Provider). + Str("reason_code", string(best.ReasonCode)). + Int("rank", nsReasonRank(best.ReasonCode)). + Bool("emittable", best.Matched). + Msg("ns best candidate selected") + return best } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 5b5585a75f..5626f45983 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -22,15 +22,37 @@ func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { payload, err := json.Marshal(finding) if err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Msg("marshal finding failed") return err } if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("publish failed") return err } - p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + p.logger.Debug(). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("finding published") return nil } From 92c33569e8990ce92c682a405d4caca5c16b8393 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:11 -0300 Subject: [PATCH 34/45] update README.md --- scanners/subdomain-takeover/README.md | 54 ++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 640b246776..7e28388320 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,16 +5,27 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates passive CNAME takeover risk using provider fingerprints +2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` ## Current detection scope -- Implemented: CNAME-based passive detection -- Not implemented yet: NS-based (Sitting Duck) detection +Implemented today: -The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. +- CNAME passive detection +- NS passive detection (vulnerable-only emission policy) +- Deterministic NS candidate selection when multiple providers/hosts match +- Nil-safe evidence extraction for CNAME and NS +- Classifier refactor to support injected fingerprint sources (`FingerprintSource`) + +Current NS behavior: + +- NS provider/host matches are evaluated and ranked +- Findings are emitted only for exploitable NS outcomes: + - `NS_FULL_LAME_PROVIDER_VULNERABLE` + - `NS_PARTIAL_LAME_PROVIDER_VULNERABLE` +- Non-exploitable NS matches (`NS_LAME_PROVIDER_UNKNOWN`, `NS_PROVIDER_MATCH_ONLY`) are classified but not emitted ## Passive-only detection policy @@ -28,7 +39,7 @@ As a result, confidence is conservative and evidence-driven: ## Confidence model (current) -Current production behavior for CNAME findings: +### CNAME - `suspected` - provider target matched, but required passive signal is missing @@ -38,6 +49,28 @@ Current production behavior for CNAME findings: - `confirmed` - reserved; not currently emitted by CNAME rules +### NS + +- `confirmed` + - full lame delegation + exploitable provider status +- `probable` + - partial lame delegation + exploitable provider status +- `suspected` + - reason codes exist for weaker NS states, but these are currently not emitted + +## Internal package layout + +Detection logic was split into focused files to reduce coupling: + +- `internal/detect/cname_rules.go` +- `internal/detect/ns_rules.go` +- `internal/detect/ns_reasoning.go` +- `internal/detect/cname_evidence.go` +- `internal/detect/ns_evidence.go` +- `internal/detect/fingerprint_source.go` + +`Classifier` now supports dependency injection via `NewClassifierWithSource(...)`, while `NewClassifier(...)` defaults to global loaded fingerprints. + ## Fingerprint data Provider fingerprint data is stored in: @@ -88,7 +121,7 @@ The worker uses explicit JetStream ack semantics: Publish a test request: ```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"example.gc.ca","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_delegations":{"ns_hosts":["ns1.example-dns-provider.net"],"ns_checks":[],"ns_delegation":{"total_ns":1,"authoritative_ok":0,"lame_count":1,"lame_type":"full"},"error":""}}}' ``` Watch findings: @@ -96,3 +129,12 @@ Watch findings: ```bash nats sub "scans.findings.upsert" ``` + +## Next steps / nice-to-haves + +1. Add table-driven tests for NS reason mapping, ranking, and emission gating. +2. Add classifier integration tests using injected `FingerprintSource`. +3. Document and implement explicit policy for `edge_case` and `registration_closed` NS provider statuses. +4. Add structured debug logs for NS matching decisions (host, provider, status, reason). +5. Add registrar-context mismatch handling (`RegistrarMismatch`) and confidence policy. +6. Add metrics (counts by reason code, emitted vs suppressed) for production observability. From ca11886b0d250b3fb40a171608a04dda9e869aa2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:00:44 -0300 Subject: [PATCH 35/45] add detect logic tests --- .../internal/detect/body_matcher_test.go | 63 +++++++++ .../internal/detect/classifier_test.go | 132 ++++++++++++++++++ .../internal/detect/cname_rules_test.go | 102 ++++++++++++++ .../internal/detect/confidence_test.go | 26 ++++ .../internal/detect/evidence_test.go | 128 +++++++++++++++++ .../detect/fingerprint_loader_test.go | 47 +++++++ .../detect/fingerprint_source_test.go | 23 +++ .../internal/detect/fingerprints_test.go | 47 +++++++ .../internal/detect/logger_test.go | 15 ++ .../internal/detect/ns_reasoning_test.go | 62 ++++++++ .../internal/detect/ns_rules_test.go | 110 +++++++++++++++ .../internal/detect/test_helpers_test.go | 32 +++++ 12 files changed, 787 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/classifier_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/confidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/evidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprints_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/logger_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/test_helpers_test.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go new file mode 100644 index 0000000000..a045a83a48 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go @@ -0,0 +1,63 @@ +package detect + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestNoopBodyFingerprintMatcher(t *testing.T) { + matcher := NewNoopBodyFingerprintMatcher() + if matcher.Contains("example.ca", "anything", FingerprintModeLiteral) { + t.Fatal("noop matcher should never match") + } +} + +func TestHTTPBodyFingerprintMatcher(t *testing.T) { + t.Run("returns false when matcher/client nil", func(t *testing.T) { + var matcher *HTTPBodyFingerprintMatcher + if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + t.Fatal("expected false for nil matcher") + } + + matcher = &HTTPBodyFingerprintMatcher{} + if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + t.Fatal("expected false for nil client") + } + }) + + t.Run("literal and regex matching", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("Hello from sample app. Error Code: 503")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + + if !matcher.Contains(domain, "sample app", FingerprintModeLiteral) { + t.Fatal("expected literal substring match") + } + if !matcher.Contains(domain, `Error Code: \d+`, FingerprintModeRegex) { + t.Fatal("expected regex match") + } + if matcher.Contains(domain, "does-not-exist", FingerprintModeLiteral) { + t.Fatal("did not expect missing literal to match") + } + }) + + t.Run("invalid regex returns false", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("anything")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + if matcher.Contains(domain, "(", FingerprintModeRegex) { + t.Fatal("expected false for invalid regex") + } + }) +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go new file mode 100644 index 0000000000..221c37ec25 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -0,0 +1,132 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +func TestClassify_ExpectedBehavior(t *testing.T) { + cnameFPs := []CNAMEProviderFingerprint{ + {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, + {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: FingerprintModeLiteral}, + } + nsFPs := []NSProviderFingerprint{ + {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "SafeDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, + } + + source := fakeSource{cname: cnameFPs, ns: nsFPs} + + t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + input := model.Input{ + DomainKey: "k1", + Results: model.ScanResults{ + Domain: strPtr("a.example.ca"), + CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.risky-dns.net"}, + Delegation: model.Delegation{ + LameType: "partial", + }, + }, + }, + } + + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 2 { + t.Fatalf("expected 2 findings, got %d", len(findings)) + } + + var sawCNAME, sawNS bool + for _, f := range findings { + if f.RecordType == model.RecordTypeCNAME { + sawCNAME = true + if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { + t.Fatalf("unexpected cname reason: %q", f.ReasonCode) + } + } + if f.RecordType == model.RecordTypeNS { + sawNS = true + if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { + t.Fatalf("unexpected ns reason: %q", f.ReasonCode) + } + } + } + if !sawCNAME || !sawNS { + t.Fatalf("missing expected finding types: cname=%v ns=%v", sawCNAME, sawNS) + } + }) + + t.Run("suppresses non-exploitable ns matches", func(t *testing.T) { + input := model.Input{ + DomainKey: "k2", + Results: model.ScanResults{ + Domain: strPtr("b.example.ca"), + CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.safe-dns.net"}, + Delegation: model.Delegation{ + LameType: "full", + }, + }, + }, + } + + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + return true + }} + + findings, err := Classify(input, matcher, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + + if len(findings) != 1 { + t.Fatalf("expected 1 finding, got %d", len(findings)) + } + if findings[0].RecordType != model.RecordTypeCNAME { + t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) + } + }) + + t.Run("returns no findings and no panic when evidence absent", func(t *testing.T) { + input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + }) +} + +func TestClassifier_MethodDefaults(t *testing.T) { + classifier := NewClassifier(nil) + if classifier == nil { + t.Fatal("expected classifier") + } + + input := model.Input{DomainKey: "k", Results: model.ScanResults{}} + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + + custom := NewClassifierWithSource(fakeMatcher{}, fakeSource{}) + if custom == nil { + t.Fatal("expected classifier with source") + } + + if custom.WithLogger(zerolog.Nop()) != custom { + t.Fatal("WithLogger should return same classifier pointer") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go new file mode 100644 index 0000000000..d3a85d9006 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -0,0 +1,102 @@ +package detect + +import "testing" + +func TestMatchCNAMEFingerprints(t *testing.T) { + fps := []CNAMEProviderFingerprint{ + { + Name: "Azure", + Cname: []string{"azurewebsites.net"}, + Nxdomain: true, + Fingerprint: "This web app is stopped", + }, + { + Name: "Ghost", + Cname: []string{"ghost.io"}, + Nxdomain: false, + Fingerprint: "The thing you were looking for is no longer here", + Mode: FingerprintModeLiteral, + }, + } + + t.Run("returns nil when no target match", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.not-a-provider.net", NoResolve: true} + if got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("nxdomain provider emits dangling reason when unresolved", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: true} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEDanglingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("nxdomain provider suppressed when resolve evidence exists", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + if domain != "a.example.ca" || mode != FingerprintModeLiteral { + t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) + } + return true + }} + got := MatchCNAMEFingerprints(evidence, fps, matcher) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEProviderFingerprintBodyMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, FingerprintMode) bool { return false }}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingBodyFP { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) +} + +func TestShouldEmitCNAME(t *testing.T) { + if ShouldEmitCNAME(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitCNAME(&CNAMEHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitCNAME(&CNAMEHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/confidence_test.go b/scanners/subdomain-takeover/internal/detect/confidence_test.go new file mode 100644 index 0000000000..d7efdd2061 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/confidence_test.go @@ -0,0 +1,26 @@ +package detect + +import "testing" + +func TestConfidenceForReason(t *testing.T) { + tests := []struct { + reason ReasonCode + want string + }{ + {ReasonCNAMEDanglingNXDOMAIN, ConfidenceProbable}, + {ReasonCNAMEProviderFingerprintBodyMatch, ConfidenceProbable}, + {ReasonCNAMETargetMatchMissingNXDOMAIN, ConfidenceSuspected}, + {ReasonCNAMETargetMatchMissingBodyFP, ConfidenceSuspected}, + {ReasonNSFullLameProviderVulnerable, ConfidenceConfirmed}, + {ReasonNSPartialLameProviderVulnerable, ConfidenceProbable}, + {ReasonNSLameProviderUnknown, ConfidenceSuspected}, + {ReasonNSProviderMatchOnly, ConfidenceSuspected}, + {ReasonCode("UNKNOWN_REASON"), ConfidenceSuspected}, + } + + for _, tt := range tests { + if got := ConfidenceForReason(tt.reason); got != tt.want { + t.Fatalf("ConfidenceForReason(%q)=%q want=%q", tt.reason, got, tt.want) + } + } +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence_test.go b/scanners/subdomain-takeover/internal/detect/evidence_test.go new file mode 100644 index 0000000000..1dc1006fe9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/evidence_test.go @@ -0,0 +1,128 @@ +package detect + +import ( + "reflect" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestExtractCNAMEEvidence(t *testing.T) { + t.Run("returns nil when cname record missing", func(t *testing.T) { + results := minimalResults("app.example.ca") + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when domain missing", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := model.ScanResults{CnameRecord: &record} + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("extracts normalized target and noresolve true", func(t *testing.T) { + record := "A.Example.CA. 300 IN CNAME Foo.AzureWebsites.NET." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.Target != "foo.azurewebsites.net" { + t.Fatalf("unexpected target: %q", got.Target) + } + if !got.NoResolve { + t.Fatal("expected NoResolve=true") + } + }) + + t.Run("sets noresolve false when resolve chain present", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + results.ResolveChain = [][]string{{"a.example.ca CNAME foo.azurewebsites.net"}} + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + if got.NoResolve { + t.Fatal("expected NoResolve=false") + } + }) +} + +func TestExtractNSEvidence(t *testing.T) { + t.Run("returns nil when domain missing", func(t *testing.T) { + input := model.ScanResults{NsDelegations: &model.NsDelegations{Hosts: []string{"ns1.example.net"}}} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when ns delegations missing", func(t *testing.T) { + input := minimalResults("app.example.ca") + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("normalizes, sorts, and deduplicates hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{ + Hosts: []string{"NS2.Example.Net.", "ns1.example.net", "ns2.example.net", ".ns1.example.net."}, + Delegation: model.Delegation{ + LameType: "partial", + }, + } + + got := ExtractNSEvidence(input) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + expectedHosts := []string{"ns1.example.net", "ns2.example.net"} + if !reflect.DeepEqual(got.NSHosts, expectedHosts) { + t.Fatalf("unexpected hosts: got=%v want=%v", got.NSHosts, expectedHosts) + } + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.NSDelegations.Delegation.LameType != "partial" { + t.Fatalf("unexpected lame type: %q", got.NSDelegations.Delegation.LameType) + } + }) +} + +func TestNormalizeHostAndParseHelpers(t *testing.T) { + if got := normalizeHost(".NS1.Example.NET."); got != "ns1.example.net" { + t.Fatalf("normalizeHost mismatch: %q", got) + } + + record := "x.example.ca. 300 IN CNAME target.provider.net." + if got := parseCname(record); got != "target.provider.net" { + t.Fatalf("parseCname mismatch: %q", got) + } + + hosts := []string{"ns3.example.net", "NS1.example.net.", "ns1.example.net", "ns2.example.net"} + expected := []string{"ns1.example.net", "ns2.example.net", "ns3.example.net"} + if got := parseHostnames(hosts); !reflect.DeepEqual(got, expected) { + t.Fatalf("parseHostnames mismatch: got=%v want=%v", got, expected) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go new file mode 100644 index 0000000000..726d95b4d9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go @@ -0,0 +1,47 @@ +package detect + +import ( + "sync" + "testing" + + "github.com/rs/zerolog" +) + +func TestLoadFingerprints(t *testing.T) { + origOnce := loadFingerprintsOnce + origErr := loadFingerprintsErr + origCNAME := CNAMEProviderFingerprints + origNS := NSProviderFingerprints + + loadFingerprintsOnce = sync.Once{} + loadFingerprintsErr = nil + CNAMEProviderFingerprints = nil + NSProviderFingerprints = nil + + t.Cleanup(func() { + loadFingerprintsOnce = origOnce + loadFingerprintsErr = origErr + CNAMEProviderFingerprints = origCNAME + NSProviderFingerprints = origNS + }) + + err := LoadFingerprints(zerolog.Nop()) + if err != nil { + t.Fatalf("LoadFingerprints returned error: %v", err) + } + if len(CNAMEProviderFingerprints) == 0 { + t.Fatal("expected cname fingerprints to be loaded") + } + if len(NSProviderFingerprints) == 0 { + t.Fatal("expected ns fingerprints to be loaded") + } +} + +func TestValidateFingerprintHelpers(t *testing.T) { + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "", Cname: []string{"x"}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname fingerprint name") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{""}}}); err == nil { + t.Fatal("expected error for empty ns host pattern") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go new file mode 100644 index 0000000000..fa9c8160ec --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go @@ -0,0 +1,23 @@ +package detect + +import "testing" + +func TestGlobalFingerprintSource(t *testing.T) { + origCNAME := CNAMEProviderFingerprints + origNS := NSProviderFingerprints + t.Cleanup(func() { + CNAMEProviderFingerprints = origCNAME + NSProviderFingerprints = origNS + }) + + CNAMEProviderFingerprints = []CNAMEProviderFingerprint{{Name: "A"}} + NSProviderFingerprints = []NSProviderFingerprint{{Name: "B"}} + + src := GlobalFingerprintSource{} + if got := src.CNAME(); len(got) != 1 || got[0].Name != "A" { + t.Fatalf("unexpected cname source: %+v", got) + } + if got := src.NS(); len(got) != 1 || got[0].Name != "B" { + t.Fatalf("unexpected ns source: %+v", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go b/scanners/subdomain-takeover/internal/detect/fingerprints_test.go new file mode 100644 index 0000000000..a31934890e --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints_test.go @@ -0,0 +1,47 @@ +package detect + +import "testing" + +func TestContainsNSHostAndWildcardMatch(t *testing.T) { + fp := NSProviderFingerprint{Name: "Test", HostPatterns: []string{"*.example.net", "ns.fixed.io"}} + + if !fp.ContainsNSHost("NS1.EXAMPLE.NET.") { + t.Fatal("expected wildcard match for ns1.example.net") + } + if !fp.ContainsNSHost("ns.fixed.io") { + t.Fatal("expected exact match for ns.fixed.io") + } + if fp.ContainsNSHost("other.net") { + t.Fatal("did not expect match for other.net") + } + + if !wildcardHostMatch("abc*xyz*io", "abc-123-xyz-final-io") { + t.Fatal("expected multi-star wildcard to match") + } + if wildcardHostMatch("abc*xyz", "zabc123xyz") { + t.Fatal("did not expect prefix mismatch to match") + } +} + +func TestContainsTargetAndNormalizeMode(t *testing.T) { + fp := CNAMEProviderFingerprint{Cname: []string{"azurewebsites.net"}} + if !fp.ContainsTarget("foo.azurewebsites.net") { + t.Fatal("expected suffix match") + } + if fp.ContainsTarget("foo.example.net") { + t.Fatal("did not expect suffix mismatch") + } + + if got := normalizeFingerprintMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { + t.Fatalf("unexpected mode: %q", got) + } + if got := normalizeFingerprintMode("", "service unavailable"); got != FingerprintModeLiteral { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := normalizeFingerprintMode("", "Error: .* not found"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := normalizeFingerprintMode("", "foo\\d+"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/logger_test.go b/scanners/subdomain-takeover/internal/detect/logger_test.go new file mode 100644 index 0000000000..2487426271 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger_test.go @@ -0,0 +1,15 @@ +package detect + +import ( + "io" + "testing" + + "github.com/rs/zerolog" +) + +func TestSetLogger(t *testing.T) { + SetLogger(zerolog.New(io.Discard)) + if got := detectLogger.GetLevel(); got == zerolog.Disabled { + t.Fatalf("expected configured logger, got level=%s", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go new file mode 100644 index 0000000000..3521974672 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -0,0 +1,62 @@ +package detect + +import "testing" + +func TestGetNSHijackReasonCode(t *testing.T) { + tests := []struct { + name string + lameType string + status NSProviderStatus + want ReasonCode + }{ + {name: "full vulnerable", lameType: "full", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "partial vulnerable", lameType: "partial", status: NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, + {name: "full vulnerable with purchase", lameType: "full", status: NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, + {name: "full not vulnerable", lameType: "full", status: NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, + {name: "partial registration closed", lameType: "partial", status: NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, + {name: "unknown lame type", lameType: "none", status: NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, + {name: "normalized lame type", lameType: " FULL ", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getNSHijackReasonCode(tt.lameType, tt.status) + if got != tt.want { + t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) + } + }) + } +} + +func TestNSReasoningHelpers(t *testing.T) { + if !isExploitableProviderStatus(NSStatusVulnerable) { + t.Fatal("expected vulnerable to be exploitable") + } + if !isExploitableProviderStatus(NSStatusVulnerableWithPurchase) { + t.Fatal("expected vulnerable_with_purchase to be exploitable") + } + if isExploitableProviderStatus(NSStatusNotVulnerable) { + t.Fatal("expected not_vulnerable to be non-exploitable") + } + + if got := normalizeLameType(" Partial "); got != "partial" { + t.Fatalf("normalizeLameType mismatch: %q", got) + } + + if !isNSMatch(ReasonNSFullLameProviderVulnerable) || !isNSMatch(ReasonNSPartialLameProviderVulnerable) { + t.Fatal("expected vulnerable reason codes to be emittable") + } + if isNSMatch(ReasonNSLameProviderUnknown) { + t.Fatal("expected unknown reason not to be emittable") + } + + if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { + t.Fatal("expected full lame rank > partial lame rank") + } + if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSLameProviderUnknown) { + t.Fatal("expected partial lame rank > unknown rank") + } + if nsReasonRank(ReasonNSLameProviderUnknown) <= nsReasonRank(ReasonNSProviderMatchOnly) { + t.Fatal("expected unknown rank > provider-only rank") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go new file mode 100644 index 0000000000..c12671323d --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -0,0 +1,110 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestMatchNSProviderRules(t *testing.T) { + nsFP := []NSProviderFingerprint{ + {Name: "UnknownDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + } + + t.Run("returns nil for missing hosts", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca"} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil for missing fingerprints", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}} + if got := MatchNSProviderRules(evidence, nil); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no provider matches", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.nomatch.net"}} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "RiskyDNS" { + t.Fatalf("unexpected provider: %q", got.Provider) + } + if got.ReasonCode != ReasonNSFullLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if !got.Matched { + t.Fatal("expected matched=true") + } + }) + + t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "partial", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "RiskyDNS" { + t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) + } + if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSLameProviderUnknown { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false for non-vulnerable state") + } + }) +} + +func TestShouldEmitNSHijack(t *testing.T) { + if ShouldEmitNSHijack(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitNSHijack(&NSHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitNSHijack(&NSHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go new file mode 100644 index 0000000000..f73fcca80f --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -0,0 +1,32 @@ +package detect + +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type fakeMatcher struct { + containsFn func(domain string, fingerprint string, mode FingerprintMode) bool +} + +func (f fakeMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if f.containsFn == nil { + return false + } + return f.containsFn(domain, fingerprint, mode) +} + +type fakeSource struct { + cname []CNAMEProviderFingerprint + ns []NSProviderFingerprint +} + +func (f fakeSource) CNAME() []CNAMEProviderFingerprint { return f.cname } +func (f fakeSource) NS() []NSProviderFingerprint { return f.ns } + +func strPtr(v string) *string { return &v } + +func boolPtr(v bool) *bool { return &v } + +func minimalResults(domain string) model.ScanResults { + return model.ScanResults{Domain: strPtr(domain)} +} From 2a63ac44a4c519f563f816881012cb46fd019498 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:44:03 -0300 Subject: [PATCH 36/45] app logic tests --- .../internal/app/runner_test.go | 87 ++++++++ .../internal/app/worker_test.go | 205 ++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/app/runner_test.go create mode 100644 scanners/subdomain-takeover/internal/app/worker_test.go diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go new file mode 100644 index 0000000000..d66c0e0b16 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -0,0 +1,87 @@ +package app + +import ( + "context" + "sync" + "testing" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakeMessagesIter struct { + mu sync.Mutex + msgs []jetstream.Msg + err error + idx int + stopCalled bool +} + +func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return nil, f.err + } + if f.idx >= len(f.msgs) { + return nil, jetstream.ErrMsgIteratorClosed + } + m := f.msgs[f.idx] + f.idx++ + return m, nil +} + +func (f *fakeMessagesIter) Stop() { f.stopCalled = true } +func (f *fakeMessagesIter) Drain() {} + +type fakeHandler struct { + mu sync.Mutex + count int +} + +func (f *fakeHandler) Handle(context.Context, jetstream.Msg) error { + f.mu.Lock() + f.count++ + f.mu.Unlock() + return nil +} + +// nolint:revive // nats Conn fields are unexported; this fake nil connection path only. +func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { + iter := &fakeMessagesIter{} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 2, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(context.Background(), deps) + + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} + +func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { + iter := &fakeMessagesIter{} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 0, + Iter: iter, + Worker: h, + NC: (*nats.Conn)(nil), + } + + Run(context.Background(), deps) + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go new file mode 100644 index 0000000000..c36915185f --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -0,0 +1,205 @@ +package app + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublisher struct { + err error + published []model.Finding +} + +func (f *fakePublisher) Publish(_ context.Context, finding model.Finding) error { + if f.err != nil { + return f.err + } + f.published = append(f.published, finding) + return nil +} + +type fakeClassifier struct { + findings []model.Finding + err error +} + +func (f fakeClassifier) Classify(model.Input) ([]model.Finding, error) { + if f.err != nil { + return nil, f.err + } + return f.findings, nil +} + +type fakeJSMsg struct { + data []byte + subject string + + ackErr error + nakErr error + termErr error + + ackCount int + nakCount int + termCount int +} + +func (m *fakeJSMsg) Metadata() (*jetstream.MsgMetadata, error) { return nil, nil } +func (m *fakeJSMsg) Data() []byte { return m.data } +func (m *fakeJSMsg) Headers() nats.Header { return nil } +func (m *fakeJSMsg) Subject() string { return m.subject } +func (m *fakeJSMsg) Reply() string { return "" } + +func (m *fakeJSMsg) Ack() error { + m.ackCount++ + return m.ackErr +} + +func (m *fakeJSMsg) DoubleAck(context.Context) error { return nil } + +func (m *fakeJSMsg) Nak() error { + m.nakCount++ + return m.nakErr +} + +func (m *fakeJSMsg) NakWithDelay(time.Duration) error { return nil } +func (m *fakeJSMsg) InProgress() error { return nil } + +func (m *fakeJSMsg) Term() error { + m.termCount++ + return m.termErr +} + +func (m *fakeJSMsg) TermWithReason(string) error { return nil } + +func TestWorkerHandle(t *testing.T) { + logger := zerolog.Nop() + + t.Run("decode error terminates message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte("{not-json"), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + if msg.nakCount != 0 || msg.ackCount != 0 { + t.Fatalf("unexpected ack/nak counts: ack=%d nak=%d", msg.ackCount, msg.nakCount) + } + }) + + t.Run("classification error naks message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{err: errors.New("classify failed")} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected classification error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("publish error naks message", func(t *testing.T) { + pub := &fakePublisher{err: errors.New("publish failed")} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected publish error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("successful processing publishes all findings and acks", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pub.published) != 2 { + t.Fatalf("expected 2 published findings, got %d", len(pub.published)) + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + if msg.nakCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected nak/term counts: nak=%d term=%d", msg.nakCount, msg.termCount) + } + }) + + t.Run("ack failure returns error", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte(`{"domain_key":"k","results":{}}`), + subject: "scans.dns_scanner_results", + ackErr: errors.New("ack failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected ack error") + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + }) + + t.Run("decode error still returned when term fails", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte("{bad-json"), + subject: "scans.dns_scanner_results", + termErr: errors.New("term failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + }) +} + +func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { + _, err := decodeScan([]byte("{\"domain_key\":\"k\",\"results\":{}}\n")) + if err != nil { + t.Fatalf("expected decode success, got error: %v", err) + } +} From 6be58701408d03bbaade6ee9153f15cbca1f0938 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:44:16 -0300 Subject: [PATCH 37/45] messaging logic tests --- .../internal/messaging/health_test.go | 13 +++ .../internal/messaging/publisher.go | 8 +- .../internal/messaging/publisher_test.go | 93 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/messaging/health_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher_test.go diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go new file mode 100644 index 0000000000..de46741d53 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -0,0 +1,13 @@ +package messaging + +import "testing" + +func TestCheckConnection(t *testing.T) { + err := CheckConnection(nil) + if err == nil { + t.Fatal("expected error for nil connection") + } + if err.Error() != "nats connection is nil" { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 5626f45983..7ba6e866be 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -11,11 +11,15 @@ import ( type Publisher struct { logger zerolog.Logger - js jetstream.JetStream + js publishClient subject string } -func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { +type publishClient interface { + Publish(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) +} + +func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publisher { return &Publisher{logger: logger, js: js, subject: subject} } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go new file mode 100644 index 0000000000..38de994a00 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -0,0 +1,93 @@ +package messaging + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublishClient struct { + err error + ack *jetstream.PubAck + called int + lastSubj string + lastBytes []byte +} + +func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.called++ + f.lastSubj = subj + f.lastBytes = append([]byte(nil), data...) + if f.err != nil { + return nil, f.err + } + if f.ack != nil { + return f.ack, nil + } + return &jetstream.PubAck{}, nil +} + +func TestNewPublisher(t *testing.T) { + client := &fakePublishClient{} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.upsert") + if p == nil { + t.Fatal("expected publisher instance") + } + if p.subject != "scans.findings.upsert" { + t.Fatalf("unexpected subject: %q", p.subject) + } +} + +func TestPublisherPublish(t *testing.T) { + finding := model.Finding{ + Domain: "a.example.ca", + DomainKey: "123", + RecordType: model.RecordTypeNS, + Target: "ns1.risky-dns.net", + Provider: "RiskyDNS", + Confidence: "probable", + ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", + } + + t.Run("publishes serialized finding", func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + if client.lastSubj != "scans.findings.upsert" { + t.Fatalf("unexpected subject: %q", client.lastSubj) + } + + var got model.Finding + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + if got != finding { + t.Fatalf("unexpected payload: got=%+v want=%+v", got, finding) + } + }) + + t.Run("returns publish client error", func(t *testing.T) { + client := &fakePublishClient{err: errors.New("publish failed")} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected publish error") + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + }) +} From 65ee95ddcd36eb6d229a0f2214c10823632c2264 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 12:26:12 -0300 Subject: [PATCH 38/45] add CNAME rtype to record_exists check --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 40567ba136..3a2c6fb5a3 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -212,7 +212,12 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: + for query_type in [ + dns.rdatatype.A, + dns.rdatatype.CNAME, + dns.rdatatype.SOA, + dns.rdatatype.NS, + ]: rtype = get_dns_return_type(domain, query_type) if rtype == "NOERROR": dns_answer_return_types.append(rtype) @@ -329,7 +334,7 @@ def scan_domain(domain, dkim_selectors=None): registrar_domain = scan_result.base_domain or zone_apex or domain scan_result.registrar_context = get_registrar_context( base_domain=registrar_domain, - ns_hosts=ns_delegations.get("ns_hosts", []), + ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), ) # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records From 64f8fd950b1ce8f2dd171e1b4581789a74515726 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:12:56 -0300 Subject: [PATCH 39/45] refactor fingerprints into separate package --- scanners/subdomain-takeover/README.md | 19 +++++++-- .../subdomain-takeover/cmd/service/main.go | 3 +- .../internal/app/runner_test.go | 2 +- .../internal/app/worker_test.go | 2 +- .../internal/detect/body_matcher.go | 12 +++--- .../internal/detect/body_matcher_test.go | 16 ++++---- .../internal/detect/classifier.go | 8 ++-- .../internal/detect/classifier_test.go | 13 +++--- .../internal/detect/cname_rules.go | 8 ++-- .../internal/detect/cname_rules_test.go | 16 +++++--- .../internal/detect/fingerprint_source.go | 14 ++++--- .../detect/fingerprint_source_test.go | 27 ++++++------ .../internal/detect/ns_reasoning.go | 12 ++++-- .../internal/detect/ns_reasoning_test.go | 28 +++++++------ .../internal/detect/ns_rules.go | 18 ++++---- .../internal/detect/ns_rules_test.go | 7 ++-- .../internal/detect/test_helpers_test.go | 13 +++--- .../data/cname_fingerprints.json | 0 .../data/ns_fingerprints.json | 0 .../loader.go} | 41 ++++++++++++------- .../loader_test.go} | 35 ++++++++-------- .../fingerprints.go => fingerprints/types.go} | 7 +--- .../types_test.go} | 10 ++--- 23 files changed, 178 insertions(+), 133 deletions(-) rename scanners/subdomain-takeover/internal/{detect => fingerprints}/data/cname_fingerprints.json (100%) rename scanners/subdomain-takeover/internal/{detect => fingerprints}/data/ns_fingerprints.json (100%) rename scanners/subdomain-takeover/internal/{detect/fingerprint_loader.go => fingerprints/loader.go} (79%) rename scanners/subdomain-takeover/internal/{detect/fingerprint_loader_test.go => fingerprints/loader_test.go} (50%) rename scanners/subdomain-takeover/internal/{detect/fingerprints.go => fingerprints/types.go} (92%) rename scanners/subdomain-takeover/internal/{detect/fingerprints_test.go => fingerprints/types_test.go} (75%) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 7e28388320..0345e1ec2c 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -75,10 +75,10 @@ Detection logic was split into focused files to reduce coupling: Provider fingerprint data is stored in: -- `internal/detect/data/cname_fingerprints.json` -- `internal/detect/data/ns_fingerprints.json` +- `internal/fingerprints/data/cname_fingerprints.json` +- `internal/fingerprints/data/ns_fingerprints.json` -The service embeds and validates these files at startup (`detect.LoadFingerprints`). +The service embeds and validates these files at startup (`fingerprints.Load`). For CNAME body matching: @@ -99,6 +99,19 @@ go mod tidy go run ./cmd/service ``` +## Local commands + +This service includes a local `Makefile` for common workflows: + +```bash +make help # list targets +make run # run the service +make test # run tests +make lint # fmt-check + vet +make build # build bin/subdomain-takeover +make ci # lint + test + build +``` + Environment variables: - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 78a2500bff..816ab0cc80 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -11,6 +11,7 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/rs/zerolog" ) @@ -34,7 +35,7 @@ func main() { Str("log_level", cfg.LogLevel.String()). Msg("service configuration loaded") - if err := detect.LoadFingerprints(logger); err != nil { + if err := fingerprints.Load(logger); err != nil { logger.Fatal().Err(err).Msg("failed to load fingerprints") } diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go index d66c0e0b16..7d076aa67d 100644 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -33,7 +33,7 @@ func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { return m, nil } -func (f *fakeMessagesIter) Stop() { f.stopCalled = true } +func (f *fakeMessagesIter) Stop() { f.stopCalled = true } func (f *fakeMessagesIter) Drain() {} type fakeHandler struct { diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go index c36915185f..fc771e4e15 100644 --- a/scanners/subdomain-takeover/internal/app/worker_test.go +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -13,7 +13,7 @@ import ( ) type fakePublisher struct { - err error + err error published []model.Finding } diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go index 622b2f7c51..b28d4b496e 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -7,10 +7,12 @@ import ( "regexp" "strings" "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" ) type BodyFingerprintMatcher interface { - Contains(domain string, fingerprint string, mode FingerprintMode) bool + Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool } type HTTPBodyFingerprintMatcher struct { @@ -29,11 +31,11 @@ func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { return &NoopBodyFingerprintMatcher{} } -func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { return false } -func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { if m == nil || m.client == nil { return false } @@ -51,9 +53,9 @@ func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, } bodyText := string(body) - resolvedMode := normalizeFingerprintMode(mode, fingerprint) + resolvedMode := fingerprints.NormalizeMode(mode, fingerprint) - if resolvedMode == FingerprintModeRegex { + if resolvedMode == fingerprints.FingerprintModeRegex { re, err := regexp.Compile(fingerprint) if err != nil { return false diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go index a045a83a48..59df870656 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go @@ -6,11 +6,13 @@ import ( "strings" "testing" "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" ) func TestNoopBodyFingerprintMatcher(t *testing.T) { matcher := NewNoopBodyFingerprintMatcher() - if matcher.Contains("example.ca", "anything", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "anything", fingerprints.FingerprintModeLiteral) { t.Fatal("noop matcher should never match") } } @@ -18,12 +20,12 @@ func TestNoopBodyFingerprintMatcher(t *testing.T) { func TestHTTPBodyFingerprintMatcher(t *testing.T) { t.Run("returns false when matcher/client nil", func(t *testing.T) { var matcher *HTTPBodyFingerprintMatcher - if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { t.Fatal("expected false for nil matcher") } matcher = &HTTPBodyFingerprintMatcher{} - if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { t.Fatal("expected false for nil client") } }) @@ -37,13 +39,13 @@ func TestHTTPBodyFingerprintMatcher(t *testing.T) { domain := strings.TrimPrefix(server.URL, "http://") matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) - if !matcher.Contains(domain, "sample app", FingerprintModeLiteral) { + if !matcher.Contains(domain, "sample app", fingerprints.FingerprintModeLiteral) { t.Fatal("expected literal substring match") } - if !matcher.Contains(domain, `Error Code: \d+`, FingerprintModeRegex) { + if !matcher.Contains(domain, `Error Code: \d+`, fingerprints.FingerprintModeRegex) { t.Fatal("expected regex match") } - if matcher.Contains(domain, "does-not-exist", FingerprintModeLiteral) { + if matcher.Contains(domain, "does-not-exist", fingerprints.FingerprintModeLiteral) { t.Fatal("did not expect missing literal to match") } }) @@ -56,7 +58,7 @@ func TestHTTPBodyFingerprintMatcher(t *testing.T) { domain := strings.TrimPrefix(server.URL, "http://") matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) - if matcher.Contains(domain, "(", FingerprintModeRegex) { + if matcher.Contains(domain, "(", fingerprints.FingerprintModeRegex) { t.Fatal("expected false for invalid regex") } }) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index ffd1d527c6..34ccd35246 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -51,13 +51,13 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} - cnameFingerprints := source.CNAME() - nsFingerprints := source.NS() + cnameProviderFingerprints := source.CNAME() + nsProviderFingerprints := source.NS() cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameProviderFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { logger.Debug(). Str("domain_key", input.DomainKey). @@ -94,7 +94,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr Str("domain", nsEvidence.Domain). Int("ns_hosts", len(nsEvidence.NSHosts)). Msg("ns evidence extracted") - nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsProviderFingerprints) if ShouldEmitNSHijack(nsHit) { logger.Debug(). Str("domain_key", input.DomainKey). diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go index 221c37ec25..62f19ec04a 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -3,18 +3,19 @@ package detect import ( "testing" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/rs/zerolog" ) func TestClassify_ExpectedBehavior(t *testing.T) { - cnameFPs := []CNAMEProviderFingerprint{ + cnameFPs := []fingerprints.CNAMEProviderFingerprint{ {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, - {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: FingerprintModeLiteral}, + {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, } - nsFPs := []NSProviderFingerprint{ - {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, - {Name: "SafeDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, + nsFPs := []fingerprints.NSProviderFingerprint{ + {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, } source := fakeSource{cname: cnameFPs, ns: nsFPs} @@ -77,7 +78,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { }, } - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { return true }} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index b693a4cd70..bc1023182e 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -1,5 +1,7 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type CNAMEHit struct { Matched bool Provider string @@ -7,8 +9,8 @@ type CNAMEHit struct { NeedsNX bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { - for _, fp := range fingerprints { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []fingerprints.CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range providerFingerprints { if fp.ContainsTarget(evidence.Target) { detectLogger.Debug(). Str("domain", evidence.Domain). @@ -32,7 +34,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP - mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + mode := fingerprints.NormalizeMode(fp.Mode, fp.Fingerprint) detectLogger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go index d3a85d9006..a8c2e217c8 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -1,9 +1,13 @@ package detect -import "testing" +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) func TestMatchCNAMEFingerprints(t *testing.T) { - fps := []CNAMEProviderFingerprint{ + fps := []fingerprints.CNAMEProviderFingerprint{ { Name: "Azure", Cname: []string{"azurewebsites.net"}, @@ -15,7 +19,7 @@ func TestMatchCNAMEFingerprints(t *testing.T) { Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "The thing you were looking for is no longer here", - Mode: FingerprintModeLiteral, + Mode: fingerprints.FingerprintModeLiteral, }, } @@ -56,8 +60,8 @@ func TestMatchCNAMEFingerprints(t *testing.T) { t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { - if domain != "a.example.ca" || mode != FingerprintModeLiteral { + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if domain != "a.example.ca" || mode != fingerprints.FingerprintModeLiteral { t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) } return true @@ -76,7 +80,7 @@ func TestMatchCNAMEFingerprints(t *testing.T) { t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, FingerprintMode) bool { return false }}) + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return false }}) if got == nil { t.Fatal("expected hit, got nil") } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go index 2ecc9981f8..b6b202e1d7 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -1,16 +1,18 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type FingerprintSource interface { - CNAME() []CNAMEProviderFingerprint - NS() []NSProviderFingerprint + CNAME() []fingerprints.CNAMEProviderFingerprint + NS() []fingerprints.NSProviderFingerprint } type GlobalFingerprintSource struct{} -func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { - return CNAMEProviderFingerprints +func (GlobalFingerprintSource) CNAME() []fingerprints.CNAMEProviderFingerprint { + return fingerprints.CNAME() } -func (GlobalFingerprintSource) NS() []NSProviderFingerprint { - return NSProviderFingerprints +func (GlobalFingerprintSource) NS() []fingerprints.NSProviderFingerprint { + return fingerprints.NS() } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go index fa9c8160ec..34f45f1c3b 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go @@ -1,23 +1,22 @@ package detect -import "testing" +import ( + "testing" -func TestGlobalFingerprintSource(t *testing.T) { - origCNAME := CNAMEProviderFingerprints - origNS := NSProviderFingerprints - t.Cleanup(func() { - CNAMEProviderFingerprints = origCNAME - NSProviderFingerprints = origNS - }) + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) - CNAMEProviderFingerprints = []CNAMEProviderFingerprint{{Name: "A"}} - NSProviderFingerprints = []NSProviderFingerprint{{Name: "B"}} +func TestGlobalFingerprintSource(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } src := GlobalFingerprintSource{} - if got := src.CNAME(); len(got) != 1 || got[0].Name != "A" { - t.Fatalf("unexpected cname source: %+v", got) + if got := src.CNAME(); len(got) == 0 { + t.Fatal("expected cname source to be populated") } - if got := src.NS(); len(got) != 1 || got[0].Name != "B" { - t.Fatalf("unexpected ns source: %+v", got) + if got := src.NS(); len(got) == 0 { + t.Fatal("expected ns source to be populated") } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go index 953cee5580..a5b2485984 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -1,8 +1,12 @@ package detect -import "strings" +import ( + "strings" -func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) + +func getNSHijackReasonCode(lameType string, providerStatus fingerprints.NSProviderStatus) ReasonCode { switch normalizeLameType(lameType) { case "full": if isExploitableProviderStatus(providerStatus) { @@ -23,8 +27,8 @@ func isNSMatch(reasonCode ReasonCode) bool { return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable } -func isExploitableProviderStatus(status NSProviderStatus) bool { - return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { + return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase } func normalizeLameType(lameType string) string { diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go index 3521974672..28b85a4ed4 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -1,21 +1,25 @@ package detect -import "testing" +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) func TestGetNSHijackReasonCode(t *testing.T) { tests := []struct { name string lameType string - status NSProviderStatus + status fingerprints.NSProviderStatus want ReasonCode }{ - {name: "full vulnerable", lameType: "full", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, - {name: "partial vulnerable", lameType: "partial", status: NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, - {name: "full vulnerable with purchase", lameType: "full", status: NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, - {name: "full not vulnerable", lameType: "full", status: NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, - {name: "partial registration closed", lameType: "partial", status: NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, - {name: "unknown lame type", lameType: "none", status: NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, - {name: "normalized lame type", lameType: " FULL ", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "full vulnerable", lameType: "full", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "partial vulnerable", lameType: "partial", status: fingerprints.NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, + {name: "full vulnerable with purchase", lameType: "full", status: fingerprints.NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, + {name: "full not vulnerable", lameType: "full", status: fingerprints.NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, + {name: "partial registration closed", lameType: "partial", status: fingerprints.NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, + {name: "unknown lame type", lameType: "none", status: fingerprints.NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, + {name: "normalized lame type", lameType: " FULL ", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, } for _, tt := range tests { @@ -29,13 +33,13 @@ func TestGetNSHijackReasonCode(t *testing.T) { } func TestNSReasoningHelpers(t *testing.T) { - if !isExploitableProviderStatus(NSStatusVulnerable) { + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerable) { t.Fatal("expected vulnerable to be exploitable") } - if !isExploitableProviderStatus(NSStatusVulnerableWithPurchase) { + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerableWithPurchase) { t.Fatal("expected vulnerable_with_purchase to be exploitable") } - if isExploitableProviderStatus(NSStatusNotVulnerable) { + if isExploitableProviderStatus(fingerprints.NSStatusNotVulnerable) { t.Fatal("expected not_vulnerable to be non-exploitable") } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index 5485e9cc95..8168cc415c 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -1,15 +1,17 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type NSHit struct { - Matched bool - Host string - Provider string - ReasonCode ReasonCode + Matched bool + Host string + Provider string + ReasonCode ReasonCode } -func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { - detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") +func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprints.NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(providerFingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(providerFingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -17,7 +19,7 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr var best *NSHit for _, host := range evidence.NSHosts { - for _, fp := range fingerprints { + for _, fp := range providerFingerprints { if fp.ContainsNSHost(host) { reasonCode := getNSHijackReasonCode(lameType, fp.Status) rank := nsReasonRank(reasonCode) diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index c12671323d..858a60e15b 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -3,13 +3,14 @@ package detect import ( "testing" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) func TestMatchNSProviderRules(t *testing.T) { - nsFP := []NSProviderFingerprint{ - {Name: "UnknownDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, - {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + nsFP := []fingerprints.NSProviderFingerprint{ + {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, } t.Run("returns nil for missing hosts", func(t *testing.T) { diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go index f73fcca80f..057f7101eb 100644 --- a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -1,14 +1,15 @@ package detect import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) type fakeMatcher struct { - containsFn func(domain string, fingerprint string, mode FingerprintMode) bool + containsFn func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool } -func (f fakeMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (f fakeMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { if f.containsFn == nil { return false } @@ -16,12 +17,12 @@ func (f fakeMatcher) Contains(domain string, fingerprint string, mode Fingerprin } type fakeSource struct { - cname []CNAMEProviderFingerprint - ns []NSProviderFingerprint + cname []fingerprints.CNAMEProviderFingerprint + ns []fingerprints.NSProviderFingerprint } -func (f fakeSource) CNAME() []CNAMEProviderFingerprint { return f.cname } -func (f fakeSource) NS() []NSProviderFingerprint { return f.ns } +func (f fakeSource) CNAME() []fingerprints.CNAMEProviderFingerprint { return f.cname } +func (f fakeSource) NS() []fingerprints.NSProviderFingerprint { return f.ns } func strPtr(v string) *string { return &v } diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json similarity index 100% rename from scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json rename to scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json similarity index 100% rename from scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json rename to scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/fingerprints/loader.go similarity index 79% rename from scanners/subdomain-takeover/internal/detect/fingerprint_loader.go rename to scanners/subdomain-takeover/internal/fingerprints/loader.go index da2101e51a..9bcec10792 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import ( "embed" @@ -11,55 +11,66 @@ import ( ) //go:embed data/*.json -var fingerprintFS embed.FS +var dataFS embed.FS var ( - loadFingerprintsOnce sync.Once - loadFingerprintsErr error + loadOnce sync.Once + loadErr error + + cnameProviderFingerprints []CNAMEProviderFingerprint + nsProviderFingerprints []NSProviderFingerprint ) -func LoadFingerprints(logger zerolog.Logger) error { +func Load(logger zerolog.Logger) error { log := logger.With().Str("component", "fingerprint_loader").Logger() - loadFingerprintsOnce.Do(func() { + loadOnce.Do(func() { var cname []CNAMEProviderFingerprint var ns []NSProviderFingerprint if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := validateCNAMEFingerprints(cname); err != nil { log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := validateNSFingerprints(ns); err != nil { log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } - CNAMEProviderFingerprints = cname - NSProviderFingerprints = ns + cnameProviderFingerprints = cname + nsProviderFingerprints = ns log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") }) - return loadFingerprintsErr + return loadErr +} + +func CNAME() []CNAMEProviderFingerprint { + return cnameProviderFingerprints +} + +func NS() []NSProviderFingerprint { + return nsProviderFingerprints } func loadJSON(path string, out any) error { - b, err := fingerprintFS.ReadFile(path) + b, err := dataFS.ReadFile(path) if err != nil { return fmt.Errorf("read %s: %w", path, err) } @@ -90,7 +101,7 @@ func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) } - fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + fingerprints[i].Mode = NormalizeMode(fp.Mode, fp.Fingerprint) } return nil diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go similarity index 50% rename from scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go rename to scanners/subdomain-takeover/internal/fingerprints/loader_test.go index 726d95b4d9..68dff54530 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import ( "sync" @@ -7,32 +7,31 @@ import ( "github.com/rs/zerolog" ) -func TestLoadFingerprints(t *testing.T) { - origOnce := loadFingerprintsOnce - origErr := loadFingerprintsErr - origCNAME := CNAMEProviderFingerprints - origNS := NSProviderFingerprints +func TestLoad(t *testing.T) { + origErr := loadErr + origCNAME := cnameProviderFingerprints + origNS := nsProviderFingerprints - loadFingerprintsOnce = sync.Once{} - loadFingerprintsErr = nil - CNAMEProviderFingerprints = nil - NSProviderFingerprints = nil + loadOnce = sync.Once{} + loadErr = nil + cnameProviderFingerprints = nil + nsProviderFingerprints = nil t.Cleanup(func() { - loadFingerprintsOnce = origOnce - loadFingerprintsErr = origErr - CNAMEProviderFingerprints = origCNAME - NSProviderFingerprints = origNS + loadOnce = sync.Once{} + loadErr = origErr + cnameProviderFingerprints = origCNAME + nsProviderFingerprints = origNS }) - err := LoadFingerprints(zerolog.Nop()) + err := Load(zerolog.Nop()) if err != nil { - t.Fatalf("LoadFingerprints returned error: %v", err) + t.Fatalf("Load returned error: %v", err) } - if len(CNAMEProviderFingerprints) == 0 { + if len(CNAME()) == 0 { t.Fatal("expected cname fingerprints to be loaded") } - if len(NSProviderFingerprints) == 0 { + if len(NS()) == 0 { t.Fatal("expected ns fingerprints to be loaded") } } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/fingerprints/types.go similarity index 92% rename from scanners/subdomain-takeover/internal/detect/fingerprints.go rename to scanners/subdomain-takeover/internal/fingerprints/types.go index d1491995cd..ac275d8430 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/fingerprints/types.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import "strings" @@ -17,9 +17,6 @@ const ( FingerprintModeRegex FingerprintMode = "regex" ) -var CNAMEProviderFingerprints []CNAMEProviderFingerprint -var NSProviderFingerprints []NSProviderFingerprint - type NSProviderStatus string const ( @@ -95,7 +92,7 @@ func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { return false } -func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { +func NormalizeMode(mode FingerprintMode, fingerprint string) FingerprintMode { if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { return mode } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go b/scanners/subdomain-takeover/internal/fingerprints/types_test.go similarity index 75% rename from scanners/subdomain-takeover/internal/detect/fingerprints_test.go rename to scanners/subdomain-takeover/internal/fingerprints/types_test.go index a31934890e..96be0d1183 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/types_test.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import "testing" @@ -32,16 +32,16 @@ func TestContainsTargetAndNormalizeMode(t *testing.T) { t.Fatal("did not expect suffix mismatch") } - if got := normalizeFingerprintMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { + if got := NormalizeMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { t.Fatalf("unexpected mode: %q", got) } - if got := normalizeFingerprintMode("", "service unavailable"); got != FingerprintModeLiteral { + if got := NormalizeMode("", "service unavailable"); got != FingerprintModeLiteral { t.Fatalf("unexpected inferred mode: %q", got) } - if got := normalizeFingerprintMode("", "Error: .* not found"); got != FingerprintModeRegex { + if got := NormalizeMode("", "Error: .* not found"); got != FingerprintModeRegex { t.Fatalf("unexpected inferred mode: %q", got) } - if got := normalizeFingerprintMode("", "foo\\d+"); got != FingerprintModeRegex { + if got := NormalizeMode("", "foo\\d+"); got != FingerprintModeRegex { t.Fatalf("unexpected inferred mode: %q", got) } } From ce87343ebc87a780f73e52cb3d0f05c7fb243ecc Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:13:07 -0300 Subject: [PATCH 40/45] add Makefile for local development --- scanners/subdomain-takeover/Makefile | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 scanners/subdomain-takeover/Makefile diff --git a/scanners/subdomain-takeover/Makefile b/scanners/subdomain-takeover/Makefile new file mode 100644 index 0000000000..72345ab0b2 --- /dev/null +++ b/scanners/subdomain-takeover/Makefile @@ -0,0 +1,47 @@ +.PHONY: help run test test-race build fmt fmt-check vet lint tidy ci + +GO ?= go +SERVICE_BIN ?= subdomain-takeover +BUILD_DIR ?= bin + +help: + @printf "Targets:\n" + @printf " make run - Run the service\n" + @printf " make test - Run all tests\n" + @printf " make test-race - Run tests with race detector\n" + @printf " make build - Build service binary\n" + @printf " make fmt - Format Go files\n" + @printf " make fmt-check - Check formatting (no changes)\n" + @printf " make vet - Run go vet\n" + @printf " make lint - Run fmt-check + vet\n" + @printf " make tidy - Tidy modules\n" + @printf " make ci - Lint, test, build\n" + +run: + $(GO) run ./cmd/service + +test: + $(GO) test ./... + +test-race: + $(GO) test -race ./... + +build: + mkdir -p $(BUILD_DIR) + CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(SERVICE_BIN) ./cmd/service + +fmt: + $(GO) fmt ./... + +fmt-check: + @test -z "$$($(GO)fmt -l .)" || (printf "Unformatted files found. Run 'make fmt'.\n" && exit 1) + +vet: + $(GO) vet ./... + +lint: fmt-check vet + +tidy: + $(GO) mod tidy + +ci: lint test build From 6ecde7dfc0d461bc957f07da7d5a6e9e82b5383a Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:13:15 -0300 Subject: [PATCH 41/45] add cloudbuild.yaml --- scanners/subdomain-takeover/cloudbuild.yaml | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scanners/subdomain-takeover/cloudbuild.yaml diff --git a/scanners/subdomain-takeover/cloudbuild.yaml b/scanners/subdomain-takeover/cloudbuild.yaml new file mode 100644 index 0000000000..64bd3d7902 --- /dev/null +++ b/scanners/subdomain-takeover/cloudbuild.yaml @@ -0,0 +1,43 @@ +steps: + - name: "golang:1.25" + id: ci-checks + dir: scanners/subdomain-takeover + entrypoint: "bash" + args: + - "-c" + - | + make ci + + - name: "gcr.io/cloud-builders/docker" + id: generate-image-name + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + echo "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/subdomain-takeover:$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename + + - name: "gcr.io/cloud-builders/docker" + id: build-scanner + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + image=$(cat /workspace/imagename) + docker build -t $image . + + - name: "gcr.io/cloud-builders/docker" + id: push-scanner-if-master + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + if [[ "$BRANCH_NAME" == "master" ]] + then + image=$(cat /workspace/imagename) + docker push $image + else + exit 0 + fi From 763e4f70ab36cb9345a629b73151f9aa3257b121 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 24 Jul 2026 17:49:39 -0300 Subject: [PATCH 42/45] add registrar mismatch rules to NS hijack detection logic --- .../subdomain-takeover/internal/app/runner.go | 4 +- .../internal/app/runner_test.go | 47 +++++- .../internal/detect/classifier_test.go | 8 +- .../internal/detect/confidence.go | 4 + .../internal/detect/ns_evidence.go | 3 +- .../internal/detect/ns_reasoning.go | 70 +++++++-- .../internal/detect/ns_reasoning_test.go | 95 +++++++++--- .../internal/detect/ns_rules.go | 14 +- .../internal/detect/ns_rules_test.go | 55 ++++++- .../internal/detect/reason_codes.go | 2 + .../internal/detect/registrar_helpers.go | 139 ++++++++++++++++++ .../internal/fingerprints/loader_test.go | 48 ++++++ .../internal/messaging/health.go | 11 ++ .../internal/messaging/health_test.go | 41 +++++- .../internal/messaging/publisher.go | 4 +- .../internal/messaging/publisher_test.go | 20 +++ 16 files changed, 516 insertions(+), 49 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/registrar_helpers.go diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index b8c0121f87..924427b855 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -23,6 +23,8 @@ type MessageHandler interface { Handle(ctx context.Context, msg jetstream.Msg) error } +var checkConnection = messaging.CheckConnection + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter @@ -39,7 +41,7 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - if err := messaging.CheckConnection(deps.NC); err != nil { + if err := checkConnection(deps.NC); err != nil { logger.Error().Err(err).Msg("NATS connection unhealthy") break Loop } diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go index 7d076aa67d..b732ecd76a 100644 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -2,8 +2,10 @@ package app import ( "context" + "errors" "sync" "testing" + "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -69,7 +71,19 @@ func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { } func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { - iter := &fakeMessagesIter{} + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + + checkCalls := 0 + checkConnection = func(_ *nats.Conn) error { + checkCalls++ + if checkCalls > 1 { + return errors.New("stop") + } + return nil + } + + iter := &fakeMessagesIter{msgs: []jetstream.Msg{&fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"}}} h := &fakeHandler{} deps := RunnerDeps{ @@ -77,10 +91,39 @@ func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { WorkerCount: 0, Iter: iter, Worker: h, - NC: (*nats.Conn)(nil), + NC: nil, } Run(context.Background(), deps) + if h.count != 1 { + t.Fatalf("expected one handled message, got %d", h.count) + } +} + +func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + checkConnection = func(_ *nats.Conn) error { return nil } + + iter := &fakeMessagesIter{err: errors.New("next failed")} + h := &fakeHandler{} + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Millisecond) + cancel() + }() + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 1, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(ctx, deps) + if h.count != 0 { t.Fatalf("expected no handled messages, got %d", h.count) } diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go index 62f19ec04a..e09a055c7d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -14,18 +14,24 @@ func TestClassify_ExpectedBehavior(t *testing.T) { {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, } nsFPs := []fingerprints.NSProviderFingerprint{ - {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, } source := fakeSource{cname: cnameFPs, ns: nsFPs} t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + rdapMatch := true input := model.Input{ DomainKey: "k1", Results: model.ScanResults{ Domain: strPtr("a.example.ca"), CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + RegistrarContext: &model.RegistrarContext{ + LookupSuccess: true, + RegistrarName: "Namecheap", + DelegationMatchesRDAP: &rdapMatch, + }, NsDelegations: &model.NsDelegations{ Hosts: []string{"ns1.risky-dns.net"}, Delegation: model.Delegation{ diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index 9865a561db..06fcd68e53 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -23,6 +23,10 @@ func ConfidenceForReason(reasonCode ReasonCode) string { return ConfidenceConfirmed case ReasonNSPartialLameProviderVulnerable: return ConfidenceProbable + case ReasonNSRegistrarProviderMatch: + return ConfidenceSuspected + case ReasonNSRegistrarContextInsufficient: + return ConfidenceSuspected case ReasonNSLameProviderUnknown: return ConfidenceSuspected case ReasonNSProviderMatchOnly: diff --git a/scanners/subdomain-takeover/internal/detect/ns_evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go index 78ddb2f428..7115fdbe2d 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_evidence.go +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -11,7 +11,7 @@ type NSEvidence struct { Domain string NSHosts []string NSDelegations model.NsDelegations - // Registrar model.RegistrarContext // if/when added + Registrar *model.RegistrarContext } func ExtractNSEvidence(input model.ScanResults) *NSEvidence { @@ -24,6 +24,7 @@ func ExtractNSEvidence(input model.ScanResults) *NSEvidence { Domain: *input.Domain, NSHosts: parseHostnames(input.NsDelegations.Hosts), NSDelegations: *input.NsDelegations, + Registrar: input.RegistrarContext, } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go index a5b2485984..bc0a672aca 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -4,33 +4,73 @@ import ( "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func getNSHijackReasonCode(lameType string, providerStatus fingerprints.NSProviderStatus) ReasonCode { +func getNSHijackReasonCode(lameType string, fingerprint fingerprints.NSProviderFingerprint, registrar *model.RegistrarContext) ReasonCode { + if !isLameType(lameType) { + return ReasonNSProviderMatchOnly + } + + if !isExploitableProviderStatus(fingerprint.Status) { + return ReasonNSLameProviderUnknown + } + + registrarReason := getRegistrarReasonCode(fingerprint.Name, registrar) + if registrarReason != "" { + return registrarReason + } + switch normalizeLameType(lameType) { case "full": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSFullLameProviderVulnerable - } - return ReasonNSLameProviderUnknown + return ReasonNSFullLameProviderVulnerable case "partial": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSPartialLameProviderVulnerable - } - return ReasonNSLameProviderUnknown + return ReasonNSPartialLameProviderVulnerable default: return ReasonNSProviderMatchOnly } } -func isNSMatch(reasonCode ReasonCode) bool { - return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +func getRegistrarReasonCode(provider string, registrar *model.RegistrarContext) ReasonCode { + if registrar == nil { + return ReasonNSRegistrarContextInsufficient + } + + if !registrar.LookupSuccess || registrar.RegistrarName == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrar.DelegationMatchesRDAP != nil && !*registrar.DelegationMatchesRDAP { + return ReasonNSRegistrarContextInsufficient + } + + registrarKey := canonicalProviderKey(registrar.RegistrarName) + providerKey := canonicalProviderKey(provider) + + if registrarKey == "" || providerKey == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrarKey == providerKey { + return ReasonNSRegistrarProviderMatch + } + + return "" +} + +func isRegistrarMismatch(provider string, registrar *model.RegistrarContext) bool { + return getRegistrarReasonCode(provider, registrar) == "" } func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase } +func isLameType(lameType string) bool { + ltNorm := normalizeLameType(lameType) + return ltNorm == "full" || ltNorm == "partial" +} + func normalizeLameType(lameType string) string { return strings.ToLower(strings.TrimSpace(lameType)) } @@ -41,10 +81,14 @@ func nsReasonRank(reasonCode ReasonCode) int { return 4 case ReasonNSPartialLameProviderVulnerable: return 3 - case ReasonNSLameProviderUnknown: + case ReasonNSRegistrarProviderMatch: return 2 - case ReasonNSProviderMatchOnly: + case ReasonNSRegistrarContextInsufficient: + return 2 + case ReasonNSLameProviderUnknown: return 1 + case ReasonNSProviderMatchOnly: + return 0 default: return 0 } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go index 28b85a4ed4..3b8fccb923 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -4,27 +4,82 @@ import ( "testing" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) func TestGetNSHijackReasonCode(t *testing.T) { + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + invalidRegistrar := &model.RegistrarContext{LookupSuccess: false, RegistrarName: "Namecheap"} + tests := []struct { - name string - lameType string - status fingerprints.NSProviderStatus - want ReasonCode + name string + lameType string + provider fingerprints.NSProviderFingerprint + registrar *model.RegistrarContext + want ReasonCode }{ - {name: "full vulnerable", lameType: "full", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, - {name: "partial vulnerable", lameType: "partial", status: fingerprints.NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, - {name: "full vulnerable with purchase", lameType: "full", status: fingerprints.NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, - {name: "full not vulnerable", lameType: "full", status: fingerprints.NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, - {name: "partial registration closed", lameType: "partial", status: fingerprints.NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, - {name: "unknown lame type", lameType: "none", status: fingerprints.NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, - {name: "normalized lame type", lameType: " FULL ", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + { + name: "full vulnerable with registrar mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "partial vulnerable with registrar mismatch", + lameType: "partial", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSPartialLameProviderVulnerable, + }, + { + name: "full vulnerable with purchase and mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerableWithPurchase}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "full non vulnerable", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusNotVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSLameProviderUnknown, + }, + { + name: "unknown lame type", + lameType: "none", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSProviderMatchOnly, + }, + { + name: "same registrar/provider suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: digitalOceanRegistrar, + want: ReasonNSRegistrarProviderMatch, + }, + { + name: "missing registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: nil, + want: ReasonNSRegistrarContextInsufficient, + }, + { + name: "invalid registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: invalidRegistrar, + want: ReasonNSRegistrarContextInsufficient, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := getNSHijackReasonCode(tt.lameType, tt.status) + got := getNSHijackReasonCode(tt.lameType, tt.provider, tt.registrar) if got != tt.want { t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) } @@ -47,20 +102,20 @@ func TestNSReasoningHelpers(t *testing.T) { t.Fatalf("normalizeLameType mismatch: %q", got) } - if !isNSMatch(ReasonNSFullLameProviderVulnerable) || !isNSMatch(ReasonNSPartialLameProviderVulnerable) { - t.Fatal("expected vulnerable reason codes to be emittable") + if !isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}) { + t.Fatal("expected registrar mismatch for different providers") } - if isNSMatch(ReasonNSLameProviderUnknown) { - t.Fatal("expected unknown reason not to be emittable") + if isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}) { + t.Fatal("expected same provider to fail mismatch check") } if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { t.Fatal("expected full lame rank > partial lame rank") } - if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSLameProviderUnknown) { - t.Fatal("expected partial lame rank > unknown rank") + if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSRegistrarProviderMatch) { + t.Fatal("expected partial lame rank > registrar-provider-match rank") } - if nsReasonRank(ReasonNSLameProviderUnknown) <= nsReasonRank(ReasonNSProviderMatchOnly) { - t.Fatal("expected unknown rank > provider-only rank") + if nsReasonRank(ReasonNSRegistrarProviderMatch) <= nsReasonRank(ReasonNSProviderMatchOnly) { + t.Fatal("expected registrar gate rank > provider-only rank") } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index 8168cc415c..32de3049e3 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -21,27 +21,29 @@ func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprin for _, host := range evidence.NSHosts { for _, fp := range providerFingerprints { if fp.ContainsNSHost(host) { - reasonCode := getNSHijackReasonCode(lameType, fp.Status) - rank := nsReasonRank(reasonCode) hit := &NSHit{ - Matched: isNSMatch(reasonCode), + Matched: false, Host: host, Provider: fp.Name, - ReasonCode: reasonCode, + ReasonCode: getNSHijackReasonCode(lameType, fp, evidence.Registrar), } + hit.Matched = hit.ReasonCode == ReasonNSFullLameProviderVulnerable || hit.ReasonCode == ReasonNSPartialLameProviderVulnerable + + rank := nsReasonRank(hit.ReasonCode) + detectLogger.Debug(). Str("domain", evidence.Domain). Str("host", host). Str("provider", fp.Name). Str("provider_status", string(fp.Status)). Str("lame_type", lameType). - Str("reason_code", string(reasonCode)). + Str("reason_code", string(hit.ReasonCode)). Int("rank", rank). Bool("emittable", hit.Matched). Msg("ns candidate evaluated") - if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + if best == nil || rank > nsReasonRank(best.ReasonCode) { best = hit } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index 858a60e15b..497dc4160f 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -10,9 +10,12 @@ import ( func TestMatchNSProviderRules(t *testing.T) { nsFP := []fingerprints.NSProviderFingerprint{ {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, - {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, } + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + t.Run("returns nil for missing hosts", func(t *testing.T) { evidence := NSEvidence{Domain: "a.example.ca"} if got := MatchNSProviderRules(evidence, nsFP); got != nil { @@ -38,6 +41,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", }}, @@ -46,7 +50,7 @@ func TestMatchNSProviderRules(t *testing.T) { if got == nil { t.Fatal("expected hit, got nil") } - if got.Provider != "RiskyDNS" { + if got.Provider != "Digital Ocean" { t.Fatalf("unexpected provider: %q", got.Provider) } if got.ReasonCode != ReasonNSFullLameProviderVulnerable { @@ -61,6 +65,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "partial", }}, @@ -69,7 +74,7 @@ func TestMatchNSProviderRules(t *testing.T) { if got == nil { t.Fatal("expected hit, got nil") } - if got.Provider != "RiskyDNS" { + if got.Provider != "Digital Ocean" { t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) } if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { @@ -81,6 +86,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.unknown-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", }}, @@ -96,6 +102,49 @@ func TestMatchNSProviderRules(t *testing.T) { t.Fatal("expected matched=false for non-vulnerable state") } }) + + t.Run("suppresses vulnerable provider when registrar matches provider", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: digitalOceanRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarProviderMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar and provider match") + } + }) + + t.Run("suppresses vulnerable provider when registrar context missing", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarContextInsufficient { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar context is missing") + } + }) } func TestShouldEmitNSHijack(t *testing.T) { diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go index f080b18329..6a79589ded 100644 --- a/scanners/subdomain-takeover/internal/detect/reason_codes.go +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -10,6 +10,8 @@ const ( ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSRegistrarProviderMatch ReasonCode = "NS_REGISTRAR_PROVIDER_MATCH" + ReasonNSRegistrarContextInsufficient ReasonCode = "NS_REGISTRAR_CONTEXT_INSUFFICIENT" ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" ) diff --git a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go new file mode 100644 index 0000000000..a31f942744 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go @@ -0,0 +1,139 @@ +package detect + +import "strings" + +func normalizeOrgName(raw string) string { + s := strings.ToLower(strings.TrimSpace(raw)) + if s == "" { + return "" + } + + // Replace punctuation/separators with spaces. + replacer := strings.NewReplacer( + ".", " ", + ",", " ", + ";", " ", + ":", " ", + "(", " ", + ")", " ", + "[", " ", + "]", " ", + "{", " ", + "}", " ", + "/", " ", + "\\", " ", + "-", " ", + "_", " ", + "&", " and ", + "'", "", + "\"", "", + ) + s = replacer.Replace(s) + + // Collapse repeated whitespace. + s = strings.Join(strings.Fields(s), " ") + if s == "" { + return "" + } + + // Remove common legal/company suffixes and generic noise words. + stop := map[string]struct{}{ + "inc": {}, "incorporated": {}, "llc": {}, "l l c": {}, "ltd": {}, "limited": {}, + "corp": {}, "corporation": {}, "co": {}, "company": {}, "gmbh": {}, "ag": {}, + "sa": {}, "s a": {}, "sas": {}, "plc": {}, "pte": {}, "bv": {}, "nv": {}, + "the": {}, "group": {}, "holdings": {}, "technologies": {}, "technology": {}, + "services": {}, "service": {}, + } + + tokens := strings.Fields(s) + out := make([]string, 0, len(tokens)) + for _, t := range tokens { + if _, drop := stop[t]; drop { + continue + } + out = append(out, t) + } + + return strings.Join(out, " ") +} +func canonicalProviderKey(raw string) string { + + var canonicalProviderAliases = map[string][]string{ + "aws-route53": { + "aws route 53", "route 53", "route53", "amazon route 53", "amazon web services", "aws", + }, + "azure-dns": { + "azure", "azure dns", "microsoft", "microsoft azure", + }, + "cloudflare": { + "cloudflare", "cloudflare inc", + }, + "digitalocean": { + "digital ocean", "digitalocean", + }, + "dnsmadeeasy": { + "dns made easy", "dnsmadeeasy", + }, + "dnsimple": { + "dnsimple", + }, + "domaincom": { + "domain com", "domain.com", "domain", + }, + "dreamhost": { + "dreamhost", + }, + "easydns": { + "easydns", "easy dns", + }, + "gandi": { + "gandi", "gandi sas", "gandi.net", + }, + "google-cloud-dns": { + "google cloud", "google cloud dns", "google domains", "googledomains", "google", + }, + "hurricane-electric": { + "hurricane electric", "he net", "he", + }, + "linode": { + "linode", "akamai linode", "akamai", + }, + "namecom": { + "name.com", "name com", "namecom", + }, + "namecheap": { + "namecheap", + }, + "network-solutions": { + "network solutions", "web.com", "webcom", + }, + "ns1": { + "ns1", "nsone", "nsone.net", + }, + "reg-ru": { + "reg ru", "reg.ru", + }, + "tierranet": { + "tierranet", "tierra net", "domaindiscover", + }, + "ultradns": { + "ultradns", "neustar ultradns", "neustar", + }, + "yahoo-smb": { + "yahoo small business", "yahoo", "yns", + }, + } + + s := normalizeOrgName(raw) // lowercase, trim, collapse spaces, strip punctuation/legal suffixes + if s == "" { + return "" + } + for key, aliases := range canonicalProviderAliases { + for _, a := range aliases { + if s == a || strings.Contains(s, a) { + return key + } + } + } + return "" +} diff --git a/scanners/subdomain-takeover/internal/fingerprints/loader_test.go b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go index 68dff54530..f28d2a0d1c 100644 --- a/scanners/subdomain-takeover/internal/fingerprints/loader_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go @@ -40,7 +40,55 @@ func TestValidateFingerprintHelpers(t *testing.T) { if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "", Cname: []string{"x"}, Fingerprint: "fp"}}); err == nil { t.Fatal("expected error for missing cname fingerprint name") } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: nil, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname patterns") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{""}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for empty cname pattern") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{"example.net"}, Fingerprint: ""}}); err == nil { + t.Fatal("expected error for missing fingerprint") + } + + validCNAME := []CNAMEProviderFingerprint{{ + Name: "ok", + Cname: []string{"example.net"}, + Fingerprint: "service unavailable", + }} + if err := validateCNAMEFingerprints(validCNAME); err != nil { + t.Fatalf("expected valid cname fingerprints, got error: %v", err) + } + if validCNAME[0].Mode != FingerprintModeLiteral { + t.Fatalf("expected normalized literal mode, got %q", validCNAME[0].Mode) + } + + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "", HostPatterns: []string{"*.example.net"}}}); err == nil { + t.Fatal("expected error for missing ns name") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: nil}}); err == nil { + t.Fatal("expected error for missing ns host patterns") + } if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{""}}}); err == nil { t.Fatal("expected error for empty ns host pattern") } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{"*.example.net"}}}); err != nil { + t.Fatalf("expected valid ns fingerprints, got error: %v", err) + } +} + +func TestLoadJSON(t *testing.T) { + t.Run("returns read error for missing file", func(t *testing.T) { + var out []CNAMEProviderFingerprint + err := loadJSON("data/missing.json", &out) + if err == nil { + t.Fatal("expected read error") + } + }) + + t.Run("returns decode error for invalid target", func(t *testing.T) { + err := loadJSON("data/cname_fingerprints.json", nil) + if err == nil { + t.Fatal("expected decode error") + } + }) } diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go index 593ca1faff..551e72d825 100644 --- a/scanners/subdomain-takeover/internal/messaging/health.go +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -7,10 +7,21 @@ import ( "github.com/nats-io/nats.go" ) +type natsConnectionState interface { + IsClosed() bool + IsConnected() bool + Status() nats.Status +} + func CheckConnection(nc *nats.Conn) error { if nc == nil { return errors.New("nats connection is nil") } + + return checkConnectionState(nc) +} + +func checkConnectionState(nc natsConnectionState) error { if nc.IsClosed() { return errors.New("nats connection is closed") } diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go index de46741d53..06829d6757 100644 --- a/scanners/subdomain-takeover/internal/messaging/health_test.go +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -1,6 +1,22 @@ package messaging -import "testing" +import ( + "testing" + + "github.com/nats-io/nats.go" +) + +type fakeConnState struct { + closed bool + connected bool + status nats.Status +} + +func (f fakeConnState) IsClosed() bool { return f.closed } +func (f fakeConnState) IsConnected() bool { return f.connected } +func (f fakeConnState) Status() nats.Status { + return f.status +} func TestCheckConnection(t *testing.T) { err := CheckConnection(nil) @@ -11,3 +27,26 @@ func TestCheckConnection(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestCheckConnectionState(t *testing.T) { + t.Run("closed connection returns closed error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{closed: true}) + if err == nil || err.Error() != "nats connection is closed" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("not connected returns status error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: false, status: nats.DISCONNECTED}) + if err == nil || err.Error() != "nats not connected (status=DISCONNECTED)" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("connected state returns nil", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: true, status: nats.CONNECTED}) + if err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 7ba6e866be..c903f05e00 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -9,6 +9,8 @@ import ( "github.com/rs/zerolog" ) +var marshalFinding = json.Marshal + type Publisher struct { logger zerolog.Logger js publishClient @@ -24,7 +26,7 @@ func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publ } func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { - payload, err := json.Marshal(finding) + payload, err := marshalFinding(finding) if err != nil { p.logger.Error(). Err(err). diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index 38de994a00..8595f65017 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -44,6 +44,9 @@ func TestNewPublisher(t *testing.T) { } func TestPublisherPublish(t *testing.T) { + origMarshal := marshalFinding + t.Cleanup(func() { marshalFinding = origMarshal }) + finding := model.Finding{ Domain: "a.example.ca", DomainKey: "123", @@ -90,4 +93,21 @@ func TestPublisherPublish(t *testing.T) { t.Fatalf("expected one publish call, got %d", client.called) } }) + + t.Run("returns marshal error and does not publish", func(t *testing.T) { + marshalFinding = func(v any) ([]byte, error) { + return nil, errors.New("marshal failed") + } + + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected marshal error") + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) } From dce38a050928fb69d0376ea8835fbbadbd6dd3de Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 24 Jul 2026 17:51:25 -0300 Subject: [PATCH 43/45] fix formatting --- .../internal/detect/ns_rules_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index 497dc4160f..61b41f7e47 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -39,8 +39,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.risky-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", @@ -63,8 +63,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "partial", @@ -84,8 +84,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", From b0ea22978e4266178f73343982ac4607875c08ab Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 29 Jul 2026 15:25:23 -0300 Subject: [PATCH 44/45] change subject out from upsert to subdomain_takeover --- scanners/subdomain-takeover/README.md | 8 +++++--- .../subdomain-takeover/internal/config/config.go | 2 +- .../internal/messaging/publisher_test.go | 12 ++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0345e1ec2c..0677e91e3c 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -7,7 +7,7 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) 2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) -4. Publishes findings to `scans.findings.upsert` +4. Publishes findings to `scans.findings.subdomain_takeover` ## Current detection scope @@ -33,6 +33,7 @@ This service uses passive checks only. It does not attempt account takeover, resource claiming, or any active validation. As a result, confidence is conservative and evidence-driven: + - `suspected`: weak or incomplete takeover indicators - `probable`: strong passive indicators with known exploitable conditions - `confirmed`: reserved for deterministic passive signatures only @@ -113,10 +114,11 @@ make ci # lint + test + build ``` Environment variables: + - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) - `SUBJECT_IN` (default: `scans.dns_scanner_results`) -- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `SUBJECT_OUT` (default: `scans.findings.subdomain_takeover`) - `DURABLE_NAME` (default: `subdomain_takeover`) - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) @@ -140,7 +142,7 @@ nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"e Watch findings: ```bash -nats sub "scans.findings.upsert" +nats sub "scans.findings.subdomain_takeover" ``` ## Next steps / nice-to-haves diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 1b18e68117..053993d380 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -9,7 +9,7 @@ type Config struct { NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` - SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.subdomain_takeover"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index 8595f65017..5e7f83b835 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -34,11 +34,11 @@ func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, func TestNewPublisher(t *testing.T) { client := &fakePublishClient{} - p := NewPublisher(zerolog.Nop(), client, "scans.findings.upsert") + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain_takeover") if p == nil { t.Fatal("expected publisher instance") } - if p.subject != "scans.findings.upsert" { + if p.subject != "scans.findings.subdomain_takeover" { t.Fatalf("unexpected subject: %q", p.subject) } } @@ -59,7 +59,7 @@ func TestPublisherPublish(t *testing.T) { t.Run("publishes serialized finding", func(t *testing.T) { client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err != nil { @@ -68,7 +68,7 @@ func TestPublisherPublish(t *testing.T) { if client.called != 1 { t.Fatalf("expected one publish call, got %d", client.called) } - if client.lastSubj != "scans.findings.upsert" { + if client.lastSubj != "scans.findings.subdomain_takeover" { t.Fatalf("unexpected subject: %q", client.lastSubj) } @@ -83,7 +83,7 @@ func TestPublisherPublish(t *testing.T) { t.Run("returns publish client error", func(t *testing.T) { client := &fakePublishClient{err: errors.New("publish failed")} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err == nil { @@ -100,7 +100,7 @@ func TestPublisherPublish(t *testing.T) { } client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err == nil { From 433c1d7a8b45041a0898a512ef156ab0a9912313 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 11:28:06 -0300 Subject: [PATCH 45/45] fix: gate RDAP scanning behind feature flag and reduce RDAP scan timeout --- scanners/dns-scanner/.env.example | 2 + .../dns-scanner/dns_scanner/dns_scanner.py | 21 +++-- .../dns-scanner/dns_scanner/ns_registrar.py | 30 ++++--- .../dns-scanner/tests/test_dns_scanner.py | 81 +++++++++++++---- .../dns-scanner/tests/test_ns_registrar.py | 88 +++++++++++++++++++ .../tests/test_scan_domain_registrar_flag.py | 66 ++++++++++++++ 6 files changed, 254 insertions(+), 34 deletions(-) create mode 100644 scanners/dns-scanner/tests/test_ns_registrar.py create mode 100644 scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py diff --git a/scanners/dns-scanner/.env.example b/scanners/dns-scanner/.env.example index 6d3cf0ff2c..9146d956e5 100644 --- a/scanners/dns-scanner/.env.example +++ b/scanners/dns-scanner/.env.example @@ -1,2 +1,4 @@ NATS_SERVERS= SCAN_TIMEOUT= +ENABLE_RDAP_LOOKUP=false +RDAP_TIMEOUT_SEC=3 diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 3a2c6fb5a3..b9c53af991 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,13 +12,18 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner -from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context +from dns_scanner.ns_registrar import ( + check_ns_delegations, + get_registrar_context, + build_registrar_context, +) logger = logging.getLogger(__name__) TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +ENABLE_RDAP_LOOKUP = os.getenv("ENABLE_RDAP_LOOKUP", "false").lower() == "true" @dataclass @@ -332,10 +337,16 @@ def scan_domain(domain, dkim_selectors=None): ) registrar_domain = scan_result.base_domain or zone_apex or domain - scan_result.registrar_context = get_registrar_context( - base_domain=registrar_domain, - ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), - ) + if ENABLE_RDAP_LOOKUP: + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), + ) + else: + scan_result.registrar_context = build_registrar_context( + base_domain=registrar_domain, + error="rdap_lookup_disabled", + ) # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py index 4e60d31dc4..7178a2774f 100644 --- a/scanners/dns-scanner/dns_scanner/ns_registrar.py +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -7,6 +7,20 @@ from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) +RDAP_TIMEOUT_SEC = float(os.getenv("RDAP_TIMEOUT_SEC", "3")) + + +def build_registrar_context(base_domain, error=None): + return { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": error, + } def probe_nameserver( @@ -52,8 +66,9 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s ns_hosts = ns_records.get("hostnames", []) if len(ns_hosts) == 0: + ns_lookup_name = zone_apex or domain try: - ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_res = resolver.resolve(ns_lookup_name, dns.rdatatype.NS) ns_hosts = [host.to_text() for host in ns_res] except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): ns_hosts = [] @@ -126,16 +141,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s def get_registrar_context(base_domain, ns_hosts=None): - context = { - "base_domain": base_domain, - "lookup_success": False, - "rdap_url": None, - "registrar_name": None, - "registrar_id": None, - "rdap_nameservers": [], - "delegation_matches_rdap": None, - "error": None, - } + context = build_registrar_context(base_domain) if not base_domain: context["error"] = "missing_base_domain" @@ -145,7 +151,7 @@ def get_registrar_context(base_domain, ns_hosts=None): context["rdap_url"] = rdap_url try: - response = requests.get(rdap_url, timeout=TIMEOUT) + response = requests.get(rdap_url, timeout=RDAP_TIMEOUT_SEC) response.raise_for_status() payload = response.json() except Exception as e: diff --git a/scanners/dns-scanner/tests/test_dns_scanner.py b/scanners/dns-scanner/tests/test_dns_scanner.py index 0b7429ebf3..921d4a0db8 100644 --- a/scanners/dns-scanner/tests/test_dns_scanner.py +++ b/scanners/dns-scanner/tests/test_dns_scanner.py @@ -1,24 +1,71 @@ -import pytest -from pretend import stub -from starlette.testclient import TestClient -from server import Server +from dns_scanner import dns_scanner as scanner_mod -def test_scan(): - client_stub = stub(post=lambda url, json: None) +class FakeResolver: + timeout = None + lifetime = None - test_app = Server(server_client=client_stub) + def resolve(self, *args, **kwargs): + raise scanner_mod.NoAnswer - test_client = TestClient(test_app) - test_payload = { - "user_key": 1, - "domain": "cyber.gc.ca", - "domain_key": "domains/1", - "selectors": ["selector1", "selector2"], - "shared_id": 1234 - } +class FakeDMARCScanner: + def __init__(self, domain): + self.domain = domain - res = test_client.post("/", json=test_payload) + def run(self): + return { + "base_domain": "example.com", + "mx": {"hosts": [], "warnings": []}, + "spf": {}, + "dmarc": {}, + "ns": {"hostnames": ["ns1.example.com."], "errors": []}, + } - assert "Scan completed" == res.text + +class FakeDKIMScanner: + def __init__(self, domain, selectors): + self.domain = domain + self.selectors = selectors + + def run(self): + return {"selectors": self.selectors} + + +def test_scan_domain_returns_expected_shape(monkeypatch): + monkeypatch.setattr(scanner_mod, "ENABLE_RDAP_LOOKUP", False) + monkeypatch.setattr( + scanner_mod, + "get_dns_return_type", + lambda domain, query_type: "NOERROR", + ) + monkeypatch.setattr(scanner_mod, "find_zone_apex", lambda domain: "example.com") + monkeypatch.setattr(scanner_mod, "DMARCScanner", FakeDMARCScanner) + monkeypatch.setattr(scanner_mod, "DKIMScanner", FakeDKIMScanner) + monkeypatch.setattr(scanner_mod.dns.resolver, "Resolver", lambda: FakeResolver()) + monkeypatch.setattr( + scanner_mod, + "get_wildcard_status", + lambda domain, resolver, a_records: { + "wildcard_entry": False, + "wildcard_sibling": False, + }, + ) + monkeypatch.setattr( + scanner_mod, + "check_ns_delegations", + lambda domain, zone_apex, ns_records: { + "ns_hosts": ns_records.get("hostnames", []), + "ns_checks": [], + "ns_delegation": {"lame_type": "none"}, + }, + ) + + result = scanner_mod.scan_domain("mail.example.com", dkim_selectors=["selector1"]) + + assert result["record_exists"] is True + assert result["rcode"] == "NOERROR" + assert result["zone_apex"] == "example.com" + assert result["base_domain"] == "example.com" + assert result["dkim"] == {"selectors": ["selector1"]} + assert result["registrar_context"]["error"] == "rdap_lookup_disabled" diff --git a/scanners/dns-scanner/tests/test_ns_registrar.py b/scanners/dns-scanner/tests/test_ns_registrar.py new file mode 100644 index 0000000000..c04da55a5f --- /dev/null +++ b/scanners/dns-scanner/tests/test_ns_registrar.py @@ -0,0 +1,88 @@ +import dns + +from dns_scanner import ns_registrar + + +class FakeHost: + def __init__(self, value): + self.value = value + + def to_text(self): + return self.value + + +class FakeResponse: + def __init__(self, rcode, authoritative): + self._rcode = rcode + self.flags = dns.flags.AA if authoritative else 0 + + def rcode(self): + return self._rcode + + +class RecordingResolver: + def __init__(self): + self.calls = [] + + def resolve(self, name, query_type): + self.calls.append((name, query_type)) + if query_type == dns.rdatatype.NS: + return [FakeHost("ns1.example.com.")] + raise AssertionError(f"Unexpected resolve call: {(name, query_type)}") + + +def test_check_ns_delegations_fallback_queries_zone_apex(monkeypatch): + resolver = RecordingResolver() + + monkeypatch.setattr(ns_registrar, "get_ns_ip", lambda host, resolver: "192.0.2.1") + def fake_probe(where, qname, qtype, recursion_desired, timeout): + assert qname == "example.com" + assert qtype == "SOA" + return FakeResponse(dns.rcode.NOERROR, True) + + monkeypatch.setattr(ns_registrar, "probe_nameserver", fake_probe) + + result = ns_registrar.check_ns_delegations( + domain="mail.example.com", + zone_apex="example.com", + ns_records={"hostnames": [], "errors": []}, + resolver=resolver, + ) + + assert resolver.calls[0] == ("example.com", dns.rdatatype.NS) + assert result["ns_delegation"]["lame_type"] == "none" + assert result["ns_delegation"]["authoritative_ok"] == 1 + + +def test_build_registrar_context_defaults(): + context = ns_registrar.build_registrar_context("example.com") + + assert context["base_domain"] == "example.com" + assert context["lookup_success"] is False + assert context["error"] is None + assert context["rdap_nameservers"] == [] + + +def test_get_registrar_context_uses_rdap_timeout(monkeypatch): + captured = {} + + class FakeRdapResponse: + def raise_for_status(self): + return None + + def json(self): + return {"nameservers": [], "entities": []} + + def fake_get(url, timeout): + captured["url"] = url + captured["timeout"] = timeout + return FakeRdapResponse() + + monkeypatch.setattr(ns_registrar, "RDAP_TIMEOUT_SEC", 2.5) + monkeypatch.setattr(ns_registrar.requests, "get", fake_get) + + context = ns_registrar.get_registrar_context("example.com", ns_hosts=[]) + + assert captured["url"] == "https://rdap.org/domain/example.com" + assert captured["timeout"] == 2.5 + assert context["lookup_success"] is True diff --git a/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py b/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py new file mode 100644 index 0000000000..f1aa2cbfca --- /dev/null +++ b/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py @@ -0,0 +1,66 @@ +from dns_scanner import dns_scanner as scanner_mod + + +class FakeResolver: + timeout = None + lifetime = None + + def resolve(self, *args, **kwargs): + raise scanner_mod.NoAnswer + + +class FakeDMARCScanner: + def __init__(self, domain): + self.domain = domain + + def run(self): + return { + "base_domain": "example.com", + "mx": {"hosts": [], "warnings": []}, + "spf": {}, + "dmarc": {}, + "ns": {"hostnames": [], "errors": []}, + } + + +class FakeDKIMScanner: + def __init__(self, domain, selectors): + self.domain = domain + self.selectors = selectors + + def run(self): + return {} + + +def test_scan_domain_skips_rdap_lookup_when_disabled(monkeypatch): + monkeypatch.setattr(scanner_mod, "ENABLE_RDAP_LOOKUP", False) + monkeypatch.setattr(scanner_mod, "get_dns_return_type", lambda domain, query_type: "NOERROR") + monkeypatch.setattr(scanner_mod, "find_zone_apex", lambda domain: "example.com") + monkeypatch.setattr( + scanner_mod, + "check_ns_delegations", + lambda domain, zone_apex, ns_records: {"ns_hosts": [], "ns_checks": [], "ns_delegation": {}}, + ) + monkeypatch.setattr( + scanner_mod, + "get_registrar_context", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("should not be called")), + ) + monkeypatch.setattr(scanner_mod, "DMARCScanner", FakeDMARCScanner) + monkeypatch.setattr(scanner_mod, "DKIMScanner", FakeDKIMScanner) + monkeypatch.setattr(scanner_mod.dns.resolver, "Resolver", lambda: FakeResolver()) + monkeypatch.setattr( + scanner_mod, + "get_wildcard_status", + lambda domain, resolver, a_records: { + "wildcard_entry": False, + "wildcard_sibling": False, + }, + ) + + result = scanner_mod.scan_domain("mail.example.com", dkim_selectors=[]) + + assert result["record_exists"] is True + assert result["registrar_context"]["base_domain"] == "example.com" + assert result["registrar_context"]["lookup_success"] is False + assert result["registrar_context"]["error"] == "rdap_lookup_disabled"