From 85310ada9dfd999ee085aa2b827d9545cbd38deb Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 08:48:24 -0300 Subject: [PATCH 01/34] init subdomain-takeover + NATS skeleton --- scanners/subdomain-takeover/.env.example | 7 + scanners/subdomain-takeover/Dockerfile | 22 ++ scanners/subdomain-takeover/README.md | 137 ++++++++++++ .../subdomain-takeover/cmd/service/main.go | 92 ++++++++ scanners/subdomain-takeover/config.go | 37 ++++ .../fixtures/cyber-gc-ca_dns_scan.json | 202 ++++++++++++++++++ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 ++ scanners/subdomain-takeover/go.mod | 19 ++ scanners/subdomain-takeover/go.sum | 21 ++ scanners/subdomain-takeover/model/finding.go | 0 scanners/subdomain-takeover/model/types.go | 0 scanners/subdomain-takeover/nats/consumer.go | 0 scanners/subdomain-takeover/nats/publisher.go | 0 .../subdomain-takeover/scanner/confidence.go | 0 .../subdomain-takeover/scanner/dns_checks.go | 0 .../subdomain-takeover/scanner/scanner.go | 0 16 files changed, 560 insertions(+) create mode 100644 scanners/subdomain-takeover/.env.example create mode 100644 scanners/subdomain-takeover/Dockerfile create mode 100644 scanners/subdomain-takeover/README.md create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/config.go create mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/go.mod create mode 100644 scanners/subdomain-takeover/go.sum create mode 100644 scanners/subdomain-takeover/model/finding.go create mode 100644 scanners/subdomain-takeover/model/types.go create mode 100644 scanners/subdomain-takeover/nats/consumer.go create mode 100644 scanners/subdomain-takeover/nats/publisher.go create mode 100644 scanners/subdomain-takeover/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/scanner/scanner.go diff --git a/scanners/subdomain-takeover/.env.example b/scanners/subdomain-takeover/.env.example new file mode 100644 index 0000000000..31d2f06813 --- /dev/null +++ b/scanners/subdomain-takeover/.env.example @@ -0,0 +1,7 @@ +NATS_URL= +NATS_STREAM= +SUBJECT_IN= +SUBJECT_OUT= +DURABLE_NAME= +WORKER_COUNT= +LOG_LEVEL= diff --git a/scanners/subdomain-takeover/Dockerfile b/scanners/subdomain-takeover/Dockerfile new file mode 100644 index 0000000000..6c7f3eafaa --- /dev/null +++ b/scanners/subdomain-takeover/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache git ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/subdomain-takeover ./cmd/service + +# Runtime stage +FROM alpine:3.22 + +RUN addgroup -S scanner && adduser -S scanner -G scanner +WORKDIR /app +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /out/subdomain-takeover /app/subdomain-takeover + +USER scanner +ENTRYPOINT ["/app/subdomain-takeover"] diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md new file mode 100644 index 0000000000..0bfcdaf9ba --- /dev/null +++ b/scanners/subdomain-takeover/README.md @@ -0,0 +1,137 @@ +# subdomain-takeover-service + +Go microservice that consumes DNS scan results and emits normalized subdomain takeover findings. + +## What it does + +1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) +2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +3. Assigns confidence (`suspected`, `probable`, `confirmed`) +4. Publishes findings to `scans.findings.upsert` + +## Passive-only detection policy + +This service uses passive checks only. +It does not attempt account takeover, resource claiming, or any active validation. + +As a result, confidence is conservative and evidence-driven: +- `suspected`: weak or incomplete takeover indicators +- `probable`: strong passive indicators with known exploitable conditions +- `confirmed`: reserved for deterministic passive signatures only + +## Sitting Duck DNS hijack coverage + +The scanner includes passive detection for Sitting Duck style DNS hijack risk. + +A Sitting Duck takeover is considered possible when all of the following are true: + +1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. +2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). +3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. + +### Variations covered + +- Partially lame delegation: some NS records are lame, others still answer. +- Full lame delegation: all authoritative NS are lame. +- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. + +If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. + +## Confidence model + +This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. + +- `suspected` + - dangling/suspicious DNS pattern only, or + - lame delegation evidence without known exploitable-provider confirmation +- `probable` + - DNS pattern + known provider unclaimed signature, or + - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match +- `confirmed` + - explicit passive takeover indicator with deterministic evidence + - if deterministic passive proof is missing, cap at `probable` + +## Required evidence for findings + +Each finding should include: +- `domain` +- `domain_key` +- `record_type` (`CNAME` or `NS`) +- `target` or `ns_host` +- `provider` +- `lame_type` (`partial` or `full`) for Sitting Duck findings +- per-NS resolver outcomes where applicable +- `confidence` +- `reason_code` +- remediation guidance + +## Remediation guidance + +Typical remediation: +1. Fix delegation at registrar and authoritative DNS provider. +2. Remove stale or dead NS entries. +3. Reclaim or securely rebind abandoned DNS zones or accounts. +4. Validate ownership controls for DNS provider account workflows. + +## Prerequisites + +- Go 1.24+ +- NATS with JetStream enabled +- `nats` CLI (optional, for local testing) + +## Quick start + +```bash +go mod tidy +go run ./cmd/service +``` + +Environment variables: +- `NATS_URL` (default: `nats://localhost:4222`) +- `NATS_STREAM` (default: `SCANS`) +- `SUBJECT_IN` (default: `scans.dns_scanner_results`) +- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `DURABLE_NAME` (default: `subdomain_takeover`) +- `WORKER_COUNT` (default: `5`) +- `LOG_LEVEL` (default: `info`) + +Local test flow: + +Publish a test request: +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' +``` + +Watch findings: +```bash +nats sub "scans.findings.upsert" +``` + +## Development milestones + +1. Build config + NATS consume/publish loop. +2. Add DNS evidence extraction. +3. Add provider fingerprint rule engine. +4. Add confidence scoring and remediation text. +5. Add tests for classification rules and message handling. + +--- + +## External Go modules to install + +Start minimal: + +```bash +go get github.com/nats-io/nats.go +go get github.com/miekg/dns +go get github.com/rs/zerolog +go get golang.org/x/sync/errgroup +go get github.com/stretchr/testify +``` + +What each is for: +- `nats.go`: JetStream consume/publish +- `miekg/dns`: robust DNS lookups and record parsing +- `zerolog`: structured logs without heavy setup +- `errgroup`: clean goroutine lifecycle management +- `testify`: easier assertions in unit tests diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..175a930507 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + }() + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break + } + + select { + case <-ctx.Done(): + return + default: + continue + } + + // sem <- struct{}{} + // go func() { + // defer func() { + // <-sem + // }() + // msg, err := iter.Next() + // if err != nil { + // logger.Error().Msgf("next err:", err.Error()) + // continue + // } + // fmt.Printf("Processing msg: %s\n", string(msg.Data())) + // // doWork() + // msg.Ack() + // }() + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/config.go new file mode 100644 index 0000000000..b14c80710a --- /dev/null +++ b/scanners/subdomain-takeover/config.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + + "github.com/kelseyhightower/envconfig" + "github.com/rs/zerolog" +) + +type Config struct { + NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` + WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` + LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` + Logger zerolog.Logger +} + +func initConfig() *Config { + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + panic(err) + } + + zerolog.SetGlobalLevel(cfg.LogLevel) + + // Create multiple output steams for zerolog + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + + logger := zerolog.New(multi).With().Timestamp().Logger() + + cfg.Logger = logger + + return &cfg +} diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..cad720cc93 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json @@ -0,0 +1,202 @@ +{ + "domain": "cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + [ + "cyber.gc.ca. 7161 IN A 20.151.96.73" + ] + ], + "resolve_ips": [ + "20.151.96.73" + ], + "cname_record": null, + "mx_records": { + "hosts": [ + { + "preference": 10, + "hostname": "cyber-gc-ca.mail.protection.outlook.com", + "addresses": [ + "52.101.190.1", + "52.101.192.0", + "52.101.192.1", + "52.101.190.0" + ] + } + ], + "warnings": [] + }, + "ns_records": { + "hostnames": [ + "ns1.d-zone.ca", + "ns2.d-zone.ca", + "ns40.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns10.ent.global.gc.ca" + ], + "warnings": [] + }, + "dkim": {}, + "spf": { + "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", + "valid": true, + "dns_void_lookups": 0, + "warnings": [], + "parsed": { + "pass": [ + { + "value": "40.80.253.57", + "mechanism": "ip4" + }, + { + "value": "52.139.3.76", + "mechanism": "a" + }, + { + "value": "205.193.218.114", + "mechanism": "a" + }, + { + "value": "205.193.218.115", + "mechanism": "a" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [ + { + "domain": "spf.protection.outlook.com", + "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", + "dns_void_lookups": 0, + "parsed": { + "pass": [ + { + "value": "40.92.0.0/15", + "mechanism": "ip4" + }, + { + "value": "40.107.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.100.0.0/15", + "mechanism": "ip4" + }, + { + "value": "52.102.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.103.0.0/17", + "mechanism": "ip4" + }, + { + "value": "104.47.0.0/17", + "mechanism": "ip4" + }, + { + "value": "2a01:111:f400::/48", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403::/49", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:8000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:c000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:f000::/52", + "mechanism": "ip6" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "warnings": [], + "spf_default": "fail", + "lookups": 0 + } + ], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "spf_default": "fail", + "lookups": 3 + }, + "dmarc": { + "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", + "valid": true, + "location": "cyber.gc.ca", + "warnings": [], + "tags": { + "v": { + "value": "DMARC1", + "explicit": true + }, + "p": { + "value": "reject", + "explicit": true + }, + "pct": { + "value": 100, + "explicit": true + }, + "rua": { + "value": [ + { + "scheme": "mailto", + "address": "dmarc@cyber.gc.ca", + "size_limit": null, + "accepting": true + } + ], + "explicit": true + }, + "adkim": { + "value": "r", + "explicit": false + }, + "aspf": { + "value": "r", + "explicit": false + }, + "fo": { + "value": "0", + "explicit": false + }, + "rf": { + "value": "afrf", + "explicit": false + }, + "ri": { + "value": 86400, + "explicit": false + }, + "sp": { + "value": "reject", + "explicit": false + } + }, + "effective_policy_source": "p", + "effective_policy": "reject" + }, + "wildcard_sibling": false, + "wildcard_entry": false, + "zone_dnssec_enabled": null +} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..8f546a74a5 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json @@ -0,0 +1,23 @@ +{ + "domain": "www.cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "www.cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], + ["cyber.gc.ca. 5550 IN A 20.151.96.73"] + ], + "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", + "ns_records": { + "hostnames": [ + "ns40.ent.global.gc.ca", + "ns10.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns2.d-zone.ca", + "ns1.d-zone.ca" + ], + "warnings": [] + } +} diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod new file mode 100644 index 0000000000..f4bfb6b567 --- /dev/null +++ b/scanners/subdomain-takeover/go.mod @@ -0,0 +1,19 @@ +module github.com/canada-ca/tracker/scanners/subdomain-takeover + +go 1.25.0 + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/klauspost/compress v1.18.5 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum new file mode 100644 index 0000000000..470146821e --- /dev/null +++ b/scanners/subdomain-takeover/go.sum @@ -0,0 +1,21 @@ +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/nats/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/nats/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 From 794cc25ac7db66544e6638fc2e2b00ff058312ae Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:22:07 -0300 Subject: [PATCH 02/34] finish NATS consumer logic --- scanners/subdomain-takeover/main.go | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go new file mode 100644 index 0000000000..102e03b573 --- /dev/null +++ b/scanners/subdomain-takeover/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err) + } + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + +Loop: + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break Loop + } + + select { + case <-ctx.Done(): + break Loop + default: + } + + msg, err := iter.Next() + if err != nil { + if ctx.Err() != nil { + break Loop + } + logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + fmt.Println(string(m.Data())) + m.Ack() + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} From c64e5e4c69eee06506c3e02a46e8f5fc4ede82fc Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:24:14 -0300 Subject: [PATCH 03/34] move main.go to top dir --- .../subdomain-takeover/cmd/service/main.go | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 scanners/subdomain-takeover/cmd/service/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go deleted file mode 100644 index 175a930507..0000000000 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - }() - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break - } - - select { - case <-ctx.Done(): - return - default: - continue - } - - // sem <- struct{}{} - // go func() { - // defer func() { - // <-sem - // }() - // msg, err := iter.Next() - // if err != nil { - // logger.Error().Msgf("next err:", err.Error()) - // continue - // } - // fmt.Printf("Processing msg: %s\n", string(msg.Data())) - // // doWork() - // msg.Ack() - // }() - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From bb6da4966a0792dafcb40673908d72bf8660a586 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 16:24:41 -0300 Subject: [PATCH 04/34] refactor main loop into app/runner.go --- .../subdomain-takeover/cmd/service/main.go | 50 ++++++++ .../subdomain-takeover/internal/app/runner.go | 89 +++++++++++++++ .../subdomain-takeover/internal/app/worker.go | 1 + .../{ => internal/config}/config.go | 4 +- .../detect/classifier.go} | 0 .../detect}/confidence.go | 0 .../types.go => internal/detect/evidence.go} | 0 .../detect/remediation.go} | 0 .../publisher.go => internal/detect/rules.go} | 0 .../logging/logger.go} | 0 .../messaging/client.go} | 0 .../internal/messaging/consumer.go | 0 .../internal/messaging/publisher.go | 0 .../internal/model/finding.go | 0 .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 scanners/subdomain-takeover/main.go | 107 ------------------ 19 files changed, 142 insertions(+), 109 deletions(-) create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/internal/app/runner.go create mode 100644 scanners/subdomain-takeover/internal/app/worker.go rename scanners/subdomain-takeover/{ => internal/config}/config.go (95%) rename scanners/subdomain-takeover/{model/finding.go => internal/detect/classifier.go} (100%) rename scanners/subdomain-takeover/{scanner => internal/detect}/confidence.go (100%) rename scanners/subdomain-takeover/{model/types.go => internal/detect/evidence.go} (100%) rename scanners/subdomain-takeover/{nats/consumer.go => internal/detect/remediation.go} (100%) rename scanners/subdomain-takeover/{nats/publisher.go => internal/detect/rules.go} (100%) rename scanners/subdomain-takeover/{scanner/dns_checks.go => internal/logging/logger.go} (100%) rename scanners/subdomain-takeover/{scanner/scanner.go => internal/messaging/client.go} (100%) create mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher.go create mode 100644 scanners/subdomain-takeover/internal/model/finding.go create mode 100644 scanners/subdomain-takeover/internal/model/types.go create mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go delete mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..4fafed7523 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + cfg := config.InitConfig() + logger := cfg.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + nc, _ := nats.Connect(cfg.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NatsStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err).Msg("Error creating message iterator") + } + + app.Run( + app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Cancel: cancel, + Context: ctx, + }) + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go new file mode 100644 index 0000000000..f02d48800d --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -0,0 +1,89 @@ +package app + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RunnerDeps struct { + Logger zerolog.Logger + WorkerCount int + Iter jetstream.MessagesContext + Cancel context.CancelFunc + Context context.Context +} + +func Run(deps RunnerDeps) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, deps.WorkerCount) + + go func() { + <-sig + deps.Logger.Info().Msg("Shutdown requested...") + deps.Cancel() + deps.Iter.Stop() + }() + +Loop: + for { + // if nc.IsClosed() { + // deps.Logger.Error().Msg("Connection to NATS is closed.") + // break Loop + // } + + select { + case <-deps.Context.Done(): + break Loop + default: + } + + msg, err := deps.Iter.Next() + if err != nil { + if deps.Context.Err() != nil { + break Loop + } + deps.Logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-deps.Context.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + deps.Logger.Info().Msg(string(m.Data())) + err := m.Ack() + if err != nil { + deps.Logger.Err(err).Msg("Ack error") + } + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + deps.Logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + deps.Logger.Info().Msg("drain timeout") + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go new file mode 100644 index 0000000000..4879f7a483 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -0,0 +1 @@ +package app diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/internal/config/config.go similarity index 95% rename from scanners/subdomain-takeover/config.go rename to scanners/subdomain-takeover/internal/config/config.go index b14c80710a..28d2ddf824 100644 --- a/scanners/subdomain-takeover/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,4 +1,4 @@ -package main +package config import ( "os" @@ -18,7 +18,7 @@ type Config struct { Logger zerolog.Logger } -func initConfig() *Config { +func InitConfig() *Config { var cfg Config if err := envconfig.Process("", &cfg); err != nil { panic(err) diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/internal/detect/classifier.go similarity index 100% rename from scanners/subdomain-takeover/model/finding.go rename to scanners/subdomain-takeover/internal/detect/classifier.go diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go similarity index 100% rename from scanners/subdomain-takeover/scanner/confidence.go rename to scanners/subdomain-takeover/internal/detect/confidence.go diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/internal/detect/evidence.go similarity index 100% rename from scanners/subdomain-takeover/model/types.go rename to scanners/subdomain-takeover/internal/detect/evidence.go diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/internal/detect/remediation.go similarity index 100% rename from scanners/subdomain-takeover/nats/consumer.go rename to scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/internal/detect/rules.go similarity index 100% rename from scanners/subdomain-takeover/nats/publisher.go rename to scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/logging/logger.go similarity index 100% rename from scanners/subdomain-takeover/scanner/dns_checks.go rename to scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/internal/messaging/client.go similarity index 100% rename from scanners/subdomain-takeover/scanner/scanner.go rename to scanners/subdomain-takeover/internal/messaging/client.go diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go deleted file mode 100644 index 102e03b573..0000000000 --- a/scanners/subdomain-takeover/main.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err) - } - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - iter.Stop() - }() - -Loop: - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break Loop - } - - select { - case <-ctx.Done(): - break Loop - default: - } - - msg, err := iter.Next() - if err != nil { - if ctx.Err() != nil { - break Loop - } - logger.Debug().Err(err).Msg("next returned, continuing") - continue - } - - select { - case sem <- struct{}{}: - case <-ctx.Done(): - break Loop - } - - wg.Add(1) - go func(m jetstream.Msg) { - defer wg.Done() - defer func() { <-sem }() - fmt.Println(string(m.Data())) - m.Ack() - }(msg) - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From ba669c435966f36a0cf933a2ea020703e4f6b291 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:00:18 -0300 Subject: [PATCH 05/34] init worker and publisher objects --- .../subdomain-takeover/cmd/service/main.go | 60 ++++++++++++++----- .../subdomain-takeover/internal/app/runner.go | 45 ++++++-------- .../subdomain-takeover/internal/app/worker.go | 50 ++++++++++++++++ .../internal/messaging/client.go | 0 .../internal/messaging/consumer.go | 1 + .../internal/messaging/publisher.go | 36 +++++++++++ .../internal/model/finding.go | 13 ++++ .../internal/model/input.go | 17 ++++++ .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 12 files changed, 180 insertions(+), 42 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/client.go create mode 100644 scanners/subdomain-takeover/internal/model/input.go delete mode 100644 scanners/subdomain-takeover/internal/model/types.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 4fafed7523..5275a170f2 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -2,47 +2,79 @@ package main import ( "context" + "os" + "os/signal" + "syscall" "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) +func checkErr(err error, log zerolog.Logger, msg string, exit bool) { + if err != nil { + log.Err(err).Msg(msg) + if exit { + os.Exit(1) + } + } + +} + func main() { cfg := config.InitConfig() logger := cfg.Logger ctx, cancel := context.WithCancel(context.Background()) - nc, _ := nats.Connect(cfg.NatsUrl) - js, _ := jetstream.New(nc) + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + nc, err := nats.Connect(cfg.NatsUrl) + checkErr(err, logger, "", true) + + js, err := jetstream.New(nc) + checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NatsStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, }) + checkErr(err, logger, "", true) - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, }) + checkErr(err, logger, "", true) iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err).Msg("Error creating message iterator") + checkErr(err, logger, "", true) + + pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) + worker := app.NewWorker(logger, *pub) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + + deps := app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Worker: *worker, } - app.Run( - app.RunnerDeps{ - Logger: logger, - WorkerCount: cfg.WorkerCount, - Iter: iter, - Cancel: cancel, - Context: ctx, - }) + app.Run(ctx, deps) logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) nc.Flush() diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index f02d48800d..a41650355d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -2,10 +2,7 @@ package app import ( "context" - "os" - "os/signal" "sync" - "syscall" "time" "github.com/nats-io/nats.go/jetstream" @@ -16,49 +13,45 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Cancel context.CancelFunc - Context context.Context + Worker Worker } -func Run(deps RunnerDeps) { - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) +func Run(ctx context.Context, deps RunnerDeps) { + logger := deps.Logger + iter := deps.Iter + + if deps.WorkerCount < 1 { + deps.WorkerCount = 1 + } var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) - go func() { - <-sig - deps.Logger.Info().Msg("Shutdown requested...") - deps.Cancel() - deps.Iter.Stop() - }() - Loop: for { // if nc.IsClosed() { - // deps.Logger.Error().Msg("Connection to NATS is closed.") + // logger.Error().Msg("Connection to NATS is closed.") // break Loop // } select { - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop default: } - msg, err := deps.Iter.Next() + msg, err := iter.Next() if err != nil { - if deps.Context.Err() != nil { + if ctx.Err() != nil { break Loop } - deps.Logger.Debug().Err(err).Msg("next returned, continuing") + logger.Debug().Err(err).Msg("next returned, continuing") continue } select { case sem <- struct{}{}: - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop } @@ -66,11 +59,7 @@ Loop: go func(m jetstream.Msg) { defer wg.Done() defer func() { <-sem }() - deps.Logger.Info().Msg(string(m.Data())) - err := m.Ack() - if err != nil { - deps.Logger.Err(err).Msg("Ack error") - } + deps.Worker.Handle(ctx, m) }(msg) } @@ -82,8 +71,8 @@ Loop: select { case <-waitDone: - deps.Logger.Info().Msg("all workers drained") + logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - deps.Logger.Info().Msg("drain timeout") + logger.Info().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 4879f7a483..44e5452724 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -1 +1,51 @@ package app + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Worker struct { + logger zerolog.Logger + pub messaging.Publisher +} + +func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { + return &Worker{logger: logger, pub: pub} +} + +func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { + log := w.logger.With().Str("component", "worker").Logger() + // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) + if err != nil { + log.Err(err).Msg("Decoding error") + } + log.Info().Msg(scan.Domain) + + // var finding model.Finding + // err := w.pub.Publish(ctx, finding) + // if err != nil { + // return err + // } + + if err := msg.Ack(); err != nil { + log.Err(err).Msg("Ack error") + return err + } + + return nil +} + +func decodeScan(data []byte) (model.Input, error) { + var scan model.Input + trimmed := bytes.Trim(data, "\n") + err := json.Unmarshal(trimmed, &scan) + return scan, err +} diff --git a/scanners/subdomain-takeover/internal/messaging/client.go b/scanners/subdomain-takeover/internal/messaging/client.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go index e69de29bb2..cbb7a43f8b 100644 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ b/scanners/subdomain-takeover/internal/messaging/consumer.go @@ -0,0 +1 @@ +package messaging diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index e69de29bb2..5b5585a75f 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -0,0 +1,36 @@ +package messaging + +import ( + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Publisher struct { + logger zerolog.Logger + js jetstream.JetStream + subject string +} + +func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} +} + +func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { + payload, err := json.Marshal(finding) + if err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + return err + } + + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + return err + } + + p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index e69de29bb2..5d387c0f1d 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -0,0 +1,13 @@ +package model + +type Finding struct { + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType string `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` + Remediation string `json:"remediation"` +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go new file mode 100644 index 0000000000..8d52ec9b96 --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -0,0 +1,17 @@ +package model + +type Input struct { + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + RCode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord string `json:"cname_record"` + NSRecords NSRecords `json:"ns_records"` +} + +type NSRecords struct { + Hostnames []string `json:"hostnames"` + Warnings []string `json:"warnings"` +} diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go deleted file mode 100644 index e69de29bb2..0000000000 From b4e9b8cbbaed8c62e95d1dd6d0f6858fdb932ca9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:14:17 -0300 Subject: [PATCH 06/34] add health check at start of run loop --- .../subdomain-takeover/cmd/service/main.go | 1 + .../subdomain-takeover/internal/app/runner.go | 11 ++++++---- .../internal/messaging/health.go | 21 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/messaging/health.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 5275a170f2..66085ffffa 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -72,6 +72,7 @@ func main() { WorkerCount: cfg.WorkerCount, Iter: iter, Worker: *worker, + NC: nc, } app.Run(ctx, deps) diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index a41650355d..42539405f0 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) @@ -14,6 +16,7 @@ type RunnerDeps struct { WorkerCount int Iter jetstream.MessagesContext Worker Worker + NC *nats.Conn } func Run(ctx context.Context, deps RunnerDeps) { @@ -29,10 +32,10 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - // if nc.IsClosed() { - // logger.Error().Msg("Connection to NATS is closed.") - // break Loop - // } + if err := messaging.CheckConnection(deps.NC); err != nil { + logger.Error().Err(err).Msg("NATS connection unhealthy") + break Loop + } select { case <-ctx.Done(): diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go new file mode 100644 index 0000000000..593ca1faff --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -0,0 +1,21 @@ +package messaging + +import ( + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +func CheckConnection(nc *nats.Conn) error { + if nc == nil { + return errors.New("nats connection is nil") + } + if nc.IsClosed() { + return errors.New("nats connection is closed") + } + if !nc.IsConnected() { + return fmt.Errorf("nats not connected (status=%s)", nc.Status().String()) + } + return nil +} From 4afcd60c5b261f23506f60c6de112ca7b6261e10 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 3 Jul 2026 11:58:39 -0300 Subject: [PATCH 07/34] add init logic for cname takeover scanning --- .../subdomain-takeover/internal/app/worker.go | 3 + .../internal/detect/classifier.go | 1 + .../internal/detect/confidence.go | 1 + .../internal/detect/evidence.go | 31 +++ .../internal/detect/fingerprints.go | 247 ++++++++++++++++++ .../internal/detect/remediation.go | 1 + .../internal/detect/rules.go | 29 ++ .../internal/model/input.go | 8 +- 8 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprints.go diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 44e5452724..7d000378e7 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" @@ -29,6 +30,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { } log.Info().Msg(scan.Domain) + detect.GetEvidence(scan) + // var finding model.Finding // err := w.pub.Publish(ctx, finding) // if err != nil { diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index e69de29bb2..c835b7444b 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetEvidence(input model.Input) []model.Finding { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cname := parseCname(*input.CnameRecord) + cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) + evidence = append(evidence, *nsEvidence) + } + + return evidence +} + +func parseCname(record string) string { + record_tokens := strings.Split(strings.Trim(record, "."), " ") + return record_tokens[len(record_tokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go new file mode 100644 index 0000000000..5d13a5489f --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -0,0 +1,247 @@ +package detect + +import "strings" + +type Fingerprint struct { + Cname []string + Name string + Nxdomain bool + Fingerprint string +} + +func (f *Fingerprint) ContainsCname(target string) bool { + for _, cname := range f.Cname { + if strings.HasSuffix(target, cname) { + return true + } + } + return false +} + +var Fingerprints = []Fingerprint{ + { + Cname: []string{ + "elasticbeanstalk.com", + }, + Nxdomain: true, + Name: "AWS/Elastic Beanstalk", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "s3.amazonaws.com", + }, + Nxdomain: false, + Name: "AWS/S3", + Fingerprint: "The specified bucket does not exist", + }, + { + Cname: []string{ + "agilecrm.com", + }, + Nxdomain: false, + Name: "Agile CRM", + Fingerprint: "Sorry, this page is no longer available.", + }, + { + Cname: []string{ + "airee.ru", + }, + Nxdomain: false, + Name: "Airee.ru", + Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", + }, + { + Cname: []string{ + "animaapp.io", + }, + Nxdomain: false, + Name: "Anima", + Fingerprint: "The page you were looking for does not exist.", + }, + { + Cname: []string{ + "bitbucket.io", + }, + Nxdomain: false, + Name: "Bitbucket", + Fingerprint: "Repository not found", + }, + { + Cname: []string{ + "trydiscourse.com", + }, + Nxdomain: true, + Name: "Discourse", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "furyns.com", + }, + Nxdomain: false, + Name: "Gemfury", + Fingerprint: "404: This page could not be found.", + }, + { + Cname: []string{ + "ghost.io", + }, + Nxdomain: false, + Name: "Ghost", + Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", + }, + { + Cname: []string{ + "hatenablog.com", + }, + Nxdomain: false, + Name: "HatenaBlog", + Fingerprint: "404 Blog is not found", + }, + { + Cname: []string{ + "helpjuice.com", + }, + Nxdomain: false, + Name: "Help Juice", + Fingerprint: "We could not find what you're looking for.", + }, + { + Cname: []string{ + "helpscoutdocs.com", + }, + Nxdomain: false, + Name: "Help Scout", + Fingerprint: "No settings were found for this company:", + }, + { + Cname: []string{ + "helprace.com", + }, + Nxdomain: false, + Name: "Helprace", + Fingerprint: "HTTP_STATUS=301", + }, + { + Cname: []string{ + "youtrack.cloud", + }, + Nxdomain: false, + Name: "JetBrains", + Fingerprint: "is not a registered InCloud YouTrack", + }, + { + Cname: []string{ + "launchrock.com", + }, + Nxdomain: false, + Name: "LaunchRock", + Fingerprint: "HTTP_STATUS=500", + }, + { + Cname: []string{ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com", + }, + Nxdomain: true, + Name: "Microsoft Azure", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "ngrok.io", + }, + Nxdomain: false, + Name: "Ngrok", + Fingerprint: "Tunnel .*.ngrok.io not found", + }, + { + Cname: []string{ + "readme.io", + }, + Nxdomain: false, + Name: "Readme.io", + Fingerprint: "The creators of this project are still working on making everything perfect!", + }, + { + Cname: []string{ + "52.16.160.97", + }, + Nxdomain: false, + Name: "SmartJobBoard", + Fingerprint: "This job board website is either expired or its domain name is invalid.", + }, + { + Cname: []string{ + "s.strikinglydns.com", + }, + Nxdomain: false, + Name: "Strikingly", + Fingerprint: "PAGE NOT FOUND.", + }, + { + Cname: []string{ + "na-west1.surge.sh", + }, + Nxdomain: false, + Name: "Surge.sh", + Fingerprint: "project not found", + }, + { + Cname: []string{ + "surveysparrow.com", + }, + Nxdomain: false, + Name: "SurveySparrow", + Fingerprint: "Account not found.", + }, + { + Cname: []string{ + "read.uberflip.com", + }, + Nxdomain: false, + Name: "Uberflip", + Fingerprint: "The URL you've accessed does not provide a hub.", + }, + { + Cname: []string{ + "stats.uptimerobot.com", + }, + Nxdomain: false, + Name: "Uptimerobot", + Fingerprint: "page not found", + }, + { + Cname: []string{ + "wordpress.com", + }, + Nxdomain: false, + Name: "Wordpress", + Fingerprint: "Do you want to register .*.wordpress.com?", + }, + { + Cname: []string{ + "worksites.net", + "69.164.223.206", + }, + Nxdomain: false, + Name: "Worksites", + Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", + }, +} diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index e69de29bb2..ced758ce62 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -0,0 +1,29 @@ +package detect + +import ( + "fmt" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if rcode == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + } + } else { + fmt.Println("check for other fingerprint") + } + } + } + + return nil +} + +func getNsTakeoverEvidence(hostnames []string) *model.Finding { + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 8d52ec9b96..57a4da346a 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -5,13 +5,13 @@ type Input struct { BaseDomain string `json:"base_domain"` ZoneApex string `json:"zone_apex"` RecordExists bool `json:"record_exists"` - RCode string `json:"rcode"` + Rcode string `json:"rcode"` ResolveChain [][]string `json:"resolve_chain"` - CnameRecord string `json:"cname_record"` - NSRecords NSRecords `json:"ns_records"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` } -type NSRecords struct { +type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } From 26f6811ef2c6ac4532de022cfc041d54632ab72e Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 6 Jul 2026 15:51:16 -0300 Subject: [PATCH 08/34] return dict of rcode answers for each dns query type --- .../dns-scanner/dns_scanner/dns_scanner.py | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 573b986ea8..e1a720542f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -17,7 +17,7 @@ TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") -DNSSEC_NAMESERVER_HOSTNAME= os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") @dataclass @@ -27,6 +27,7 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None + query_res: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -50,9 +51,9 @@ def find_zone_apex(domain, resolver=None): while name != dns.name.root: logger.debug(f"Checking for SOA at {name}") try: - answers = resolver.resolve(name, 'SOA') + answers = resolver.resolve(name, "SOA") logger.debug(f"Found SOA for {domain} at {name}: {answers[0]}") - zone_apex = str(name).rstrip('.') + zone_apex = str(name).rstrip(".") return zone_apex except NoAnswer: # Go up one level @@ -105,7 +106,13 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker """ try: q = dns.message.make_query(domain, dns.rdatatype.DNSKEY, want_dnssec=True) - resp = dns.query.tls(q, where=nameserver_ip, timeout=TIMEOUT, server_hostname=nameserver_hostname, verify=True) + resp = dns.query.tls( + q, + where=nameserver_ip, + timeout=TIMEOUT, + server_hostname=nameserver_hostname, + verify=True, + ) if resp.rcode() != dns.rcode.NOERROR: return None except Timeout: @@ -114,7 +121,9 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker ) return None except Exception as e: - logger.error(f"Error while running minimal DNSSEC check for domain '{domain}': {e}") + logger.error( + f"Error while running minimal DNSSEC check for domain '{domain}': {e}" + ) return None # Check if AD (Authenticated Data) flag is set, showing that the data is DNSSEC validated return bool(resp.flags & dns.flags.AD) @@ -201,8 +210,15 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: + query_res = {} + for query_type in [ + dns.rdatatype.A, + dns.rdatatype.SOA, + dns.rdatatype.NS, + dns.rdatatype.CNAME, + ]: rtype = get_dns_return_type(domain, query_type) + query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) break @@ -221,6 +237,7 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) + scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -284,10 +301,16 @@ def scan_domain(domain, dkim_selectors=None): logger.debug(f"Skipping DNSSEC check for {domain} - No zone apex found") zone_dnssec_enabled = None elif not DNSSEC_NAMESERVER_IP or not DNSSEC_NAMESERVER_HOSTNAME: - logger.debug(f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set") + logger.debug( + f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set" + ) zone_dnssec_enabled = None else: - zone_dnssec_enabled = dnssec_check_with_ttl(domain=zone_apex, nameserver_ip=DNSSEC_NAMESERVER_IP, nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME) + zone_dnssec_enabled = dnssec_check_with_ttl( + domain=zone_apex, + nameserver_ip=DNSSEC_NAMESERVER_IP, + nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME, + ) scan_result.zone_dnssec_enabled = zone_dnssec_enabled From 314d207332ce402e452fbcca4b71ed004d5d6ff0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:01:38 -0300 Subject: [PATCH 09/34] add function to query and process ns delegations --- .../dns-scanner/dns_scanner/dns_scanner.py | 112 +++++++++++++++++- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index e1a720542f..7441cf816f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,6 +28,7 @@ class DNSScanResult: record_exists: bool = None rcode: str = None query_res: dict = None + ns_delegations: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -145,7 +146,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): result = {"wildcard_entry": False, "wildcard_sibling": False} try: wildcard_sibling_domain = re.sub(r"^[^.]+", "*", domain) - wildcard_record = dns.resolver.resolve( + wildcard_record = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.A, raise_on_no_answer=False, @@ -157,7 +158,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): try: # check for mail-only subdomain (e.g. mail.example.com) mx_records = resolver.resolve(qname=domain, rdtype=dns.rdatatype.MX) - wildcard_mx = dns.resolver.resolve( + wildcard_mx = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.MX, raise_on_no_answer=False, @@ -194,6 +195,104 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result +def dns_query_direct( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) + if not recursion_desired: + query.flags &= ~dns.flags.RD + return dns.query.udp(query, where=where, timeout=timeout) + + +def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): + ns_hosts = ns_records.get("hostnames", []) + # Always return structured output, even when empty + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], # per-NS evidence rows + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", # none | partial | full + }, + } + + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": domain, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -221,10 +320,8 @@ def scan_domain(domain, dkim_selectors=None): query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) - break elif rtype is None: dns_answer_return_types.append(None) - continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -320,12 +417,17 @@ def scan_domain(domain, dkim_selectors=None): dmarc_scanner = DMARCScanner(domain) dmarc_scan_result = dmarc_scanner.run() scan_result.base_domain = dmarc_scan_result.get("base_domain", "") - scan_result.ns_records = dmarc_scan_result.get("ns", {}) scan_result.mx_records = dmarc_scan_result.get("mx", {}) scan_result.spf = dmarc_scan_result.get("spf", {}) scan_result.dmarc = dmarc_scan_result.get("dmarc", {}) logger.debug(f"DMARC scan elapsed time: {time.monotonic() - dmarc_start_time}") + ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) + scan_result.ns_records = ns_records + # check nameserver delegations + ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + scan_result.ns_delegations = ns_delegations + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 From ddefa536c64b32ef42d3661af333029583440786 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:30:06 -0300 Subject: [PATCH 10/34] use zone apex for probe qname and recheck for ns hostnames --- .../dns-scanner/dns_scanner/dns_scanner.py | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 7441cf816f..f82113bdeb 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -204,8 +204,42 @@ def dns_query_direct( return dns.query.udp(query, where=where, timeout=timeout) -def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + # Always return structured output, even when empty output = { "ns_hosts": ns_hosts, @@ -217,18 +251,14 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): "lame_type": "none", # none | partial | full }, } - if len(ns_hosts) == 0: output["ns_delegation"]["lame_type"] = "unknown" return output - if resolver is None: - resolver = dns.resolver.get_default_resolver() - for host in ns_hosts: row = { "ns_host": host, - "qname": domain, + "qname": qname, "qtype": "SOA", "rcode": None, "answered_authoritatively": False, @@ -237,22 +267,7 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): } try: - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - + ns_ip = get_ns_ip(host, resolver) if ns_ip is None: row["error"] = "ns_ip_resolution_failed" output["ns_delegation"]["lame_count"] += 1 @@ -425,7 +440,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + ns_delegations = check_ns_delegations( + domain=domain, zone_apex=zone_apex, ns_records=ns_records + ) scan_result.ns_delegations = ns_delegations # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records From 0155dd634646c35117267014aded10340f6510a9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:35:01 -0300 Subject: [PATCH 11/34] better func names --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index f82113bdeb..da2e3aca1b 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -195,7 +195,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def dns_query_direct( +def probe_nameserver( where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int ): query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) @@ -274,7 +274,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s output["ns_checks"].append(row) continue - res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) row["rcode"] = dns.rcode.to_text(res.rcode()) row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) From 9101a702d5c23ace565088f7234d2c43e95da848 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:23:43 -0300 Subject: [PATCH 12/34] add new input fields --- .../internal/detect/classifier.go | 20 + .../internal/detect/evidence.go | 48 ++- .../internal/detect/fingerprints.go | 345 ++++++++++++++++++ .../internal/detect/remediation.go | 4 + .../internal/detect/rules.go | 30 +- .../internal/model/input.go | 47 ++- 6 files changed, 452 insertions(+), 42 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index ad6c03d18d..5e4a157c0d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1 +1,21 @@ package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + +func Classify(input model.Input) ([]model.Finding, error) { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cnameEvidence := ExtractCNAMEEvidence(input) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := ExtractNSEvidence(input) + evidence = append(evidence, *nsEvidence) + } + + return evidence, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index c835b7444b..f0ce3424e0 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,31 +1,49 @@ package detect import ( + "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetEvidence(input model.Input) []model.Finding { - evidence := []model.Finding{} - - // parse CNAME chain - if input.CnameRecord != nil { - cname := parseCname(*input.CnameRecord) - cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) - evidence = append(evidence, *cnameEvidence) +func ExtractCNAMEEvidence(input model.Input) *model.Finding { + cname := parseCname(*input.CnameRecord) + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if input.QueryAnswers.A == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + return &model.Finding{ + Domain: input.Domain, + RecordType: "CNAME", + Provider: fp.Name, + Confidence: "confirmed", + Target: cname, + } + } + } else { + fmt.Println("check for other fingerprint") + } + } } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) - evidence = append(evidence, *nsEvidence) - } + return nil +} + +func ExtractNSEvidence(input model.Input) *model.Finding { + return nil +} - return evidence +func ClassifyLameType(nsChecks []model.NsCheck) { + return } func parseCname(record string) string { - record_tokens := strings.Split(strings.Trim(record, "."), " ") + trimmed := strings.Trim(record, ".") + lower := strings.ToLower(trimmed) + record_tokens := strings.Split(lower, " ") return record_tokens[len(record_tokens)-1] } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 5d13a5489f..9c4e642e1b 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -9,6 +9,351 @@ type Fingerprint struct { Fingerprint string } +type NSProviderStatus string + +const ( + NSStatusVulnerable NSProviderStatus = "vulnerable" + NSStatusNotVulnerable NSProviderStatus = "not_vulnerable" + NSStatusEdgeCase NSProviderStatus = "edge_case" + NSStatusVulnerableWithPurchase NSProviderStatus = "vulnerable_with_purchase" + NSStatusRegistrationClosed NSProviderStatus = "registration_closed" +) + +type NSProviderFingerprint struct { + Name string + ProviderURL string + Status NSProviderStatus + HostPatterns []string + InstructionsURL string + PrivateDNS bool +} + +func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, pattern := range f.HostPatterns { + if wildcardHostMatch(pattern, host) { + return true + } + } + return false +} + +func wildcardHostMatch(pattern, host string) bool { + pattern = strings.ToLower(strings.TrimSuffix(pattern, ".")) + if !strings.Contains(pattern, "*") { + return host == pattern + } + + parts := strings.Split(pattern, "*") + if len(parts) == 2 { + return strings.HasPrefix(host, parts[0]) && strings.HasSuffix(host, parts[1]) + } + + idx := 0 + for i, part := range parts { + if part == "" { + continue + } + + pos := strings.Index(host[idx:], part) + if pos < 0 { + return false + } + + if i == 0 && !strings.HasPrefix(host, part) { + return false + } + + idx += pos + len(part) + } + + last := parts[len(parts)-1] + if last != "" && !strings.HasSuffix(host, last) { + return false + } + + return true +} + +var NSProviderFingerprints = []NSProviderFingerprint{ + { + Name: "000Domains", + ProviderURL: "https://000domains.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", + }, + { + Name: "AWS Route 53", + ProviderURL: "https://aws.amazon.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", + }, + { + Name: "Azure (Microsoft)", + ProviderURL: "https://azure.microsoft.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", + }, + { + Name: "BigCommerce", + ProviderURL: "https://bigcommerce.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", + }, + { + Name: "Bizland", + ProviderURL: "https://bizland.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", + }, + { + Name: "ClouDNS", + ProviderURL: "https://cloudns.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.cloudns.net"}, + }, + { + Name: "Cloudflare", + ProviderURL: "https://cloudflare.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.ns.cloudflare.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", + }, + { + Name: "Digital Ocean", + ProviderURL: "https://digitalocean.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", + }, + { + Name: "DNSMadeEasy", + ProviderURL: "https://dnsmadeeasy.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns*.dnsmadeeasy.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", + }, + { + Name: "DNSimple", + ProviderURL: "https://dnsimple.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", + }, + { + Name: "Domain.com", + ProviderURL: "https://domain.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", + }, + { + Name: "DomainPeople", + ProviderURL: "https://domainpeople.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", + }, + { + Name: "Dotster", + ProviderURL: "https://dotster.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", + }, + { + Name: "Dreamhost", + ProviderURL: "https://dreamhost.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", + }, + { + Name: "EasyDNS", + ProviderURL: "https://easydns.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", + }, + { + Name: "Gandi.net", + ProviderURL: "https://gandi.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, + }, + { + Name: "Google Cloud", + ProviderURL: "https://cloud.google.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns-cloud-*.googledomains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", + }, + { + Name: "Hostinger (old NS)", + ProviderURL: "https://hostinger.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, + }, + { + Name: "Hover", + ProviderURL: "https://hover.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", + }, + { + Name: "Hurricane Electric", + ProviderURL: "https://dns.he.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", + }, + { + Name: "Linode", + ProviderURL: "https://linode.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", + }, + { + Name: "MediaTemple (mt)", + ProviderURL: "https://mediatemple.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", + }, + { + Name: "MyDomain", + ProviderURL: "https://mydomain.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", + }, + { + Name: "Name.com", + ProviderURL: "https://name.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, + InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", + }, + { + Name: "Namecheap", + ProviderURL: "https://namecheap.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, + }, + { + Name: "Network Solutions", + ProviderURL: "https://networksolutions.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.worldnic.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", + }, + { + Name: "NS1", + ProviderURL: "https://nsone.net/", + Status: NSStatusRegistrationClosed, + HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", + }, + { + Name: "TierraNet", + ProviderURL: "https://tierra.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", + }, + { + Name: "Reg.ru", + ProviderURL: "https://reg.ru/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", + }, + { + Name: "UltraDNS", + ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", + }, + { + Name: "Yahoo Small Business", + ProviderURL: "https://yahoosmallbusiness.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", + }, + { + Name: "Activision", + ProviderURL: "https://activision.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.activision.com"}, + PrivateDNS: true, + }, + { + Name: "Adobe", + ProviderURL: "https://adobe.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"adobe-dns-0*.adobe.com"}, + PrivateDNS: true, + }, + { + Name: "Apple", + ProviderURL: "https://apple.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, + PrivateDNS: true, + }, + { + Name: "Automattic", + ProviderURL: "https://automattic.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.automattic.com"}, + PrivateDNS: true, + }, + { + Name: "Capital One", + ProviderURL: "https://capitalone.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.capitalone.com"}, + PrivateDNS: true, + }, + { + Name: "Disney", + ProviderURL: "https://disney.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, + PrivateDNS: true, + }, + { + Name: "Google", + ProviderURL: "https://google.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.google.com"}, + PrivateDNS: true, + }, + { + Name: "Lowe's", + ProviderURL: "https://lowes.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"authns*.lowes.com"}, + PrivateDNS: true, + }, + { + Name: "T-Mobile", + ProviderURL: "https://tmobileus.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, + PrivateDNS: true, + }, +} + func (f *Fingerprint) ContainsCname(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index ad6c03d18d..9d577baa7f 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -1 +1,5 @@ package detect + +func RemediationForReason(reasonCode string, provider string, lameType string) { + return +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index ced758ce62..1d2bb6c10a 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,29 +1,21 @@ package detect import ( - "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if rcode == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - } - } else { - fmt.Println("check for other fingerprint") - } - } - } +func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { + return +} + +func MatchNSProviderRules(evidence model.Finding, providerRules []string) { + return +} - return nil +func ShouldEmitCNAME() { + return } -func getNsTakeoverEvidence(hostnames []string) *model.Finding { - return nil +func ShouldEmitNSHijack() { + return } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 57a4da346a..7bdd179f73 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,17 +1,48 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + Rcode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` + QueryAnswers QueryAnswers `json:"query_res"` + NsDelegations NsDelegations `json:"ns_delegations"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } + +type QueryAnswers struct { + A string `json:"A"` + SOA string `json:"SOA"` + NS string `json:"NS"` + CNAME string `json:"CNAME"` +} + +type NsDelegations struct { + Hosts []string `json:"ns_hosts"` + Checks []NsCheck `json:"ns_checks"` + Delegation Delegation `json:"ns_delegation"` +} + +type NsCheck struct { + Host string `json:"ns_host"` + Qname string `json:"qname"` + Qtype string `json:"qtype"` + Rcode string `json:"rcode"` + AnsweredAuthoritatively bool `json:"answered_authoritatively"` + Error *string `json:"error"` + Timeout bool `json:"timeout"` +} +type Delegation struct { + TotalHosts int `json:"total_ns"` + OkCount int `json:"authoritative_ok"` + LameCount int `json:"lame_count"` + LameType string `json:"lame_type"` +} From c213301480bc39bc8d5706a1d9571519dcbbb8e5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:24:50 -0300 Subject: [PATCH 13/34] add registrar checks and move new logic to ns_registrar.py --- .../dns-scanner/dns_scanner/dns_scanner.py | 121 +---------- .../dns-scanner/dns_scanner/ns_registrar.py | 195 ++++++++++++++++++ 2 files changed, 203 insertions(+), 113 deletions(-) create mode 100644 scanners/dns-scanner/dns_scanner/ns_registrar.py diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index da2e3aca1b..06b72869d5 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,6 +12,7 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner +from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context logger = logging.getLogger(__name__) @@ -29,6 +30,7 @@ class DNSScanResult: rcode: str = None query_res: dict = None ns_delegations: dict = None + registrar_context: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -195,119 +197,6 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def probe_nameserver( - where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int -): - query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) - if not recursion_desired: - query.flags &= ~dns.flags.RD - return dns.query.udp(query, where=where, timeout=timeout) - - -def get_ns_ip(host: str, resolver): - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - return ns_ip - - -def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): - if resolver is None: - resolver = dns.resolver.get_default_resolver() - - qname = zone_apex - if not zone_apex: - qname = domain - - ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: - try: - ns_res = resolver.resolve(domain, dns.rdatatype.NS) - ns_hosts = [host.to_text() for host in ns_res] - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_hosts = [] - - # Always return structured output, even when empty - output = { - "ns_hosts": ns_hosts, - "ns_checks": [], # per-NS evidence rows - "ns_delegation": { - "total_ns": len(ns_hosts), - "authoritative_ok": 0, - "lame_count": 0, - "lame_type": "none", # none | partial | full - }, - } - if len(ns_hosts) == 0: - output["ns_delegation"]["lame_type"] = "unknown" - return output - - for host in ns_hosts: - row = { - "ns_host": host, - "qname": qname, - "qtype": "SOA", - "rcode": None, - "answered_authoritatively": False, - "error": None, - "timeout": False, - } - - try: - ns_ip = get_ns_ip(host, resolver) - if ns_ip is None: - row["error"] = "ns_ip_resolution_failed" - output["ns_delegation"]["lame_count"] += 1 - output["ns_checks"].append(row) - continue - - res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) - row["rcode"] = dns.rcode.to_text(res.rcode()) - row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) - - if row["answered_authoritatively"] and row["rcode"] in [ - "NOERROR", - "NXDOMAIN", - ]: - output["ns_delegation"]["authoritative_ok"] += 1 - else: - output["ns_delegation"]["lame_count"] += 1 - except Timeout: - row["timeout"] = True - row["error"] = "timeout" - output["ns_delegation"]["lame_count"] += 1 - except Exception as e: - row["error"] = str(e) - output["ns_delegation"]["lame_count"] += 1 - - output["ns_checks"].append(row) - - ok = output["ns_delegation"]["authoritative_ok"] - total = output["ns_delegation"]["total_ns"] - - if ok == total: - output["ns_delegation"]["lame_type"] = "none" - elif ok == 0: - output["ns_delegation"]["lame_type"] = "full" - else: - output["ns_delegation"]["lame_type"] = "partial" - - return output - - def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -445,6 +334,12 @@ def scan_domain(domain, dkim_selectors=None): ) scan_result.ns_delegations = ns_delegations + registrar_domain = scan_result.base_domain or zone_apex or domain + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=ns_delegations.get("ns_hosts", []), + ) + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py new file mode 100644 index 0000000000..9b48e7754f --- /dev/null +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -0,0 +1,195 @@ +import os + +import dns +import dns.resolver +import requests +from dns.exception import Timeout +from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers + +TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) + + +def probe_nameserver( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query( + qname, + dns.rdatatype.from_text(qtype), + use_edns=True, + recursion_desired=recursion_desired, + ) + return dns.query.udp(query, where=where, timeout=timeout) + + +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", + }, + } + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": qname, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = get_ns_ip(host, resolver) + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = probe_nameserver(ns_ip, qname, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + +def get_registrar_context(base_domain, ns_hosts=None): + context = { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": None, + } + + if not base_domain: + context["error"] = "missing_base_domain" + return context + + rdap_url = f"https://rdap.org/domain/{base_domain}" + context["rdap_url"] = rdap_url + + try: + response = requests.get(rdap_url, timeout=TIMEOUT) + response.raise_for_status() + payload = response.json() + except Exception as e: + context["error"] = str(e) + return context + + context["lookup_success"] = True + + nameservers = payload.get("nameservers", []) + context["rdap_nameservers"] = [ + (ns.get("ldhName") or "").rstrip(".").lower() + for ns in nameservers + if ns.get("ldhName") + ] + + entities = payload.get("entities", []) + for entity in entities: + roles = [r.lower() for r in entity.get("roles", [])] + if "registrar" not in roles: + continue + + context["registrar_id"] = entity.get("handle") + + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) == 2 and isinstance(vcard[1], list): + for entry in vcard[1]: + if not isinstance(entry, list) or len(entry) < 4: + continue + key = entry[0] + value = entry[3] + if key in {"fn", "org"} and value: + context["registrar_name"] = value + break + + if context["registrar_name"] is None: + public_ids = entity.get("publicIds", []) + if public_ids: + context["registrar_name"] = public_ids[0].get("identifier") + + break + + if ns_hosts is not None: + normalized_hosts = {h.rstrip(".").lower() for h in ns_hosts if h} + rdap_hosts = set(context["rdap_nameservers"]) + if rdap_hosts: + context["delegation_matches_rdap"] = normalized_hosts == rdap_hosts + + return context From 8442f3adda185fa1c842fb5eb51b952d5cc9eeaa Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 18:23:44 -0300 Subject: [PATCH 14/34] establish Classify() workflow --- .../subdomain-takeover/internal/app/worker.go | 20 ++++--- .../internal/detect/classifier.go | 25 +++++---- .../internal/detect/evidence.go | 52 ++++++++++--------- .../internal/detect/fingerprints.go | 6 +-- .../internal/detect/rules.go | 44 +++++++++++++--- 5 files changed, 93 insertions(+), 54 deletions(-) diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 7d000378e7..40176d6c88 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -28,15 +28,21 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { if err != nil { log.Err(err).Msg("Decoding error") } - log.Info().Msg(scan.Domain) + // log.Info().Msg(scan.Domain) - detect.GetEvidence(scan) + findings, err := detect.Classify(scan) + if err != nil { + log.Err(err).Msg("classify error") + return err + } - // var finding model.Finding - // err := w.pub.Publish(ctx, finding) - // if err != nil { - // return err - // } + for _, finding := range findings { + err = w.pub.Publish(ctx, finding) + if err != nil { + log.Err(err).Msg("publish error") + return err + } + } if err := msg.Ack(); err != nil { log.Err(err).Msg("Ack error") diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5e4a157c0d..f14a5b7f35 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -3,19 +3,22 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" func Classify(input model.Input) ([]model.Finding, error) { - evidence := []model.Finding{} + findings := []model.Finding{} - // parse CNAME chain - if input.CnameRecord != nil { - cnameEvidence := ExtractCNAMEEvidence(input) - evidence = append(evidence, *cnameEvidence) + cnameEvidence := ExtractCNAMEEvidence(input) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) + if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := ExtractNSEvidence(input) - evidence = append(evidence, *nsEvidence) + nsEvidence := ExtractNSEvidence(input) + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(*nsEvidence, nsHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - - return evidence, nil + return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f0ce3424e0..fb40318cfc 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,40 +1,42 @@ package detect import ( - "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func ExtractCNAMEEvidence(input model.Input) *model.Finding { - cname := parseCname(*input.CnameRecord) - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if input.QueryAnswers.A == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - return &model.Finding{ - Domain: input.Domain, - RecordType: "CNAME", - Provider: fp.Name, - Confidence: "confirmed", - Target: cname, - } - } - } else { - fmt.Println("check for other fingerprint") - } - } +// inputs from evidence.go +type CNAMEEvidence struct { + Domain string + Target string + QueryAnswer model.QueryAnswers // or just needed fields +} + +type NSEvidence struct { + Domain string + NSHosts []string + NSDelegations model.NsDelegations + // Registrar model.RegistrarContext // if/when added +} + +func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { + if input.CnameRecord == nil { + return nil } - return nil + return &CNAMEEvidence{ + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + QueryAnswer: input.QueryAnswers, + } } -func ExtractNSEvidence(input model.Input) *model.Finding { - return nil +func ExtractNSEvidence(input model.Input) *NSEvidence { + if len(input.NsDelegations.Hosts) == 0 { + return nil + } + return &NSEvidence{} } func ClassifyLameType(nsChecks []model.NsCheck) { diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 9c4e642e1b..b7098341a2 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -2,7 +2,7 @@ package detect import "strings" -type Fingerprint struct { +type CNAMEProviderFingerprint struct { Cname []string Name string Nxdomain bool @@ -354,7 +354,7 @@ var NSProviderFingerprints = []NSProviderFingerprint{ }, } -func (f *Fingerprint) ContainsCname(target string) bool { +func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { return true @@ -363,7 +363,7 @@ func (f *Fingerprint) ContainsCname(target string) bool { return false } -var Fingerprints = []Fingerprint{ +var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ { Cname: []string{ "elasticbeanstalk.com", diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 1d2bb6c10a..15edb479d6 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,21 +1,49 @@ package detect import ( - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "fmt" ) -func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { - return +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + NeedsNX bool } -func MatchNSProviderRules(evidence model.Finding, providerRules []string) { - return +type NSHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + ProviderVulnerable bool + RegistrarMismatch bool } -func ShouldEmitCNAME() { - return +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if evidence.QueryAnswer.A == "NXDOMAIN" { + fmt.Println("Bingo") + } + } // else { + // fmt.Println("check for other fingerprint") + // } + } + } } -func ShouldEmitNSHijack() { +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { return } + +func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { + return false +} + +func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { + return false +} From c4c1582a16b045ade48036f8a4d53a57df9c400a Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 10:08:51 -0300 Subject: [PATCH 15/34] match cname fingerprints and emit finding --- .../internal/detect/classifier.go | 14 ++++- .../internal/detect/rules.go | 62 ++++++++++++++++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index f14a5b7f35..0caea61196 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -7,15 +7,25 @@ func Classify(input model.Input) ([]model.Finding, error) { cnameEvidence := ExtractCNAMEEvidence(input) cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + if ShouldEmitCNAME(cnameHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ - Domain: input.Domain, + Domain: cnameEvidence.Domain, + RecordType: "CNAME", + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: cnameHit.ReasonCode, + Confidence: "", + Remediation: "", }) } nsEvidence := ExtractNSEvidence(input) nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ Domain: input.Domain, }) diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 15edb479d6..9cb5ac5738 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -2,13 +2,15 @@ package detect import ( "fmt" + "io" + "net/http" + "strings" ) type CNAMEHit struct { Matched bool Provider string ReasonCode string - RuleID string NeedsNX bool } @@ -16,7 +18,6 @@ type NSHit struct { Matched bool Provider string ReasonCode string - RuleID string ProviderVulnerable bool RegistrarMismatch bool } @@ -24,26 +25,67 @@ type NSHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + if hit.NeedsNX { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" if evidence.QueryAnswer.A == "NXDOMAIN" { - fmt.Println("Bingo") + hit.Matched = true + hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" } - } // else { - // fmt.Println("check for other fingerprint") - // } + } else { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.Matched = true + hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + } + } + return &hit } } + return nil } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return + return &NSHit{} } -func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + if hit.Matched { + return true + } return false } func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } + +func urlBodyContainsFingerprint(domain string, fp string) bool { + // curl domain and look for matching fp in body + url := fmt.Sprintf("http://%s", domain) + res, err := http.Get(url) + if err != nil { + // log error + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + // log err + return false + } + + if strings.Contains(string(body), fp) { + return true + } + return false +} From 61b90a5c44884f12ae4be2566ebcf91035d820f4 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 12:39:40 -0300 Subject: [PATCH 16/34] revert changes to record gate --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 16 ++++------------ scanners/dns-scanner/dns_scanner/ns_registrar.py | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 06b72869d5..40567ba136 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,7 +28,6 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None - query_res: dict = None ns_delegations: dict = None registrar_context: dict = None resolve_chain: list[list[str]] = None @@ -213,19 +212,14 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - query_res = {} - for query_type in [ - dns.rdatatype.A, - dns.rdatatype.SOA, - dns.rdatatype.NS, - dns.rdatatype.CNAME, - ]: + for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: rtype = get_dns_return_type(domain, query_type) - query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) + break elif rtype is None: dns_answer_return_types.append(None) + continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -238,7 +232,6 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) - scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -329,10 +322,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations( + scan_result.ns_delegations = check_ns_delegations( domain=domain, zone_apex=zone_apex, ns_records=ns_records ) - scan_result.ns_delegations = ns_delegations registrar_domain = scan_result.base_domain or zone_apex or domain scan_result.registrar_context = get_registrar_context( diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py index 9b48e7754f..4e60d31dc4 100644 --- a/scanners/dns-scanner/dns_scanner/ns_registrar.py +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -16,8 +16,9 @@ def probe_nameserver( qname, dns.rdatatype.from_text(qtype), use_edns=True, - recursion_desired=recursion_desired, ) + if not recursion_desired: + query.flags &= ~dns.flags.RD return dns.query.udp(query, where=where, timeout=timeout) @@ -50,7 +51,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s qname = domain ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: + if len(ns_hosts) == 0: try: ns_res = resolver.resolve(domain, dns.rdatatype.NS) ns_hosts = [host.to_text() for host in ns_res] From c09a79000e6540d8562eefeeea251ef79c318859 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:14:24 -0300 Subject: [PATCH 17/34] run go mod tidy --- scanners/subdomain-takeover/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod index f4bfb6b567..ca7b20f4ac 100644 --- a/scanners/subdomain-takeover/go.mod +++ b/scanners/subdomain-takeover/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats.go v1.52.0 github.com/rs/zerolog v1.35.1 ) @@ -11,7 +12,6 @@ require ( github.com/klauspost/compress v1.18.5 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/nats-io/nats.go v1.52.0 // indirect github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.49.0 // indirect From 7e55e8828483d9bd9e19af66d1ba4f4bb6b87f36 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:14 -0300 Subject: [PATCH 18/34] move logger out of config.go --- .../internal/bootstrap/logger.go | 13 +++++++++++ .../internal/config/config.go | 22 +++++-------------- .../internal/logging/logger.go | 0 3 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/logger.go delete mode 100644 scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/internal/bootstrap/logger.go b/scanners/subdomain-takeover/internal/bootstrap/logger.go new file mode 100644 index 0000000000..3721d23e40 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/logger.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "os" + + "github.com/rs/zerolog" +) + +func NewLogger(level zerolog.Level) zerolog.Logger { + zerolog.SetGlobalLevel(level) + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + return zerolog.New(multi).With().Timestamp().Logger() +} diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 28d2ddf824..1b18e68117 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,37 +1,25 @@ package config import ( - "os" - "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog" ) type Config struct { - NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` - NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` - Logger zerolog.Logger } -func InitConfig() *Config { +func Load() (*Config, error) { var cfg Config if err := envconfig.Process("", &cfg); err != nil { - panic(err) + return nil, err } - zerolog.SetGlobalLevel(cfg.LogLevel) - - // Create multiple output steams for zerolog - multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) - - logger := zerolog.New(multi).With().Timestamp().Logger() - - cfg.Logger = logger - - return &cfg + return &cfg, nil } diff --git a/scanners/subdomain-takeover/internal/logging/logger.go b/scanners/subdomain-takeover/internal/logging/logger.go deleted file mode 100644 index e69de29bb2..0000000000 From 7af7e0f892161ba16f1bc6c75495742b671cada8 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:57 -0300 Subject: [PATCH 19/34] bootstrap nats and add input interfaces --- .../subdomain-takeover/internal/app/runner.go | 6 +- .../subdomain-takeover/internal/app/worker.go | 47 +++++++++++---- .../internal/bootstrap/nats.go | 59 +++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/nats.go diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 42539405f0..6d7ee52e8d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -15,10 +15,14 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Worker Worker + Worker MessageHandler NC *nats.Conn } +type MessageHandler interface { + Handle(ctx context.Context, msg jetstream.Msg) error +} + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 40176d6c88..e282fab952 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -4,48 +4,59 @@ import ( "bytes" "context" "encoding/json" + "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) +type FindingPublisher interface { + Publish(ctx context.Context, finding model.Finding) error +} + +type ScanClassifier interface { + Classify(input model.Input) ([]model.Finding, error) +} + type Worker struct { - logger zerolog.Logger - pub messaging.Publisher + logger zerolog.Logger + publisher FindingPublisher + classifier ScanClassifier } -func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { - return &Worker{logger: logger, pub: pub} +func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { + return &Worker{logger: logger, publisher: publisher, classifier: classifier} } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { log := w.logger.With().Str("component", "worker").Logger() - // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) if err != nil { - log.Err(err).Msg("Decoding error") + log.Err(err).Msg("decode error") + w.term(msg, log, err) + return err } - // log.Info().Msg(scan.Domain) - findings, err := detect.Classify(scan) + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") + w.nak(msg, log, err) return err } for _, finding := range findings { - err = w.pub.Publish(ctx, finding) + err = w.publisher.Publish(ctx, finding) if err != nil { log.Err(err).Msg("publish error") + w.nak(msg, log, err) return err } } if err := msg.Ack(); err != nil { - log.Err(err).Msg("Ack error") + log.Err(err).Msg("ack error") return err } @@ -58,3 +69,15 @@ func decodeScan(data []byte) (model.Input, error) { err := json.Unmarshal(trimmed, &scan) return scan, err } + +func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Nak(); err != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") + } +} + +func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Term(); err != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") + } +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go new file mode 100644 index 0000000000..51edb6b060 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -0,0 +1,59 @@ +package bootstrap + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type RuntimeDeps struct { + NC *nats.Conn + Iter jetstream.MessagesContext + JS jetstream.JetStream +} + +func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { + nc, err := nats.Connect(cfg.NATSURL) + if err != nil { + return nil, err + } + + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + if err != nil { + nc.Close() + return nil, err + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + if err != nil { + nc.Close() + return nil, err + } + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + nc.Close() + return nil, err + } + + return &RuntimeDeps{ + NC: nc, + Iter: iter, + JS: js, + }, nil +} From 7e4cdd9ba87fa811e0332be912622ab8d8269f31 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:17:37 -0300 Subject: [PATCH 20/34] add better error handling to main.go during startup and shutdown --- .../subdomain-takeover/cmd/service/main.go | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 66085ffffa..0f588ec30d 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -8,76 +8,70 @@ import ( "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) -func checkErr(err error, log zerolog.Logger, msg string, exit bool) { +func main() { + logger := bootstrap.NewLogger(zerolog.InfoLevel) + + cfg, err := config.Load() if err != nil { - log.Err(err).Msg(msg) - if exit { - os.Exit(1) - } + logger.Fatal().Err(err).Msg("failed to load config") } -} + logger = bootstrap.NewLogger(cfg.LogLevel) -func main() { - cfg := config.InitConfig() - logger := cfg.Logger + if err := detect.LoadFingerprints(); err != nil { + logger.Fatal().Err(err).Msg("failed to load fingerprints") + } ctx, cancel := context.WithCancel(context.Background()) + defer cancel() sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) - nc, err := nats.Connect(cfg.NatsUrl) - checkErr(err, logger, "", true) - - js, err := jetstream.New(nc) - checkErr(err, logger, "", true) - - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: cfg.NatsStream, - Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, - }) - checkErr(err, logger, "", true) - - cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: cfg.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - checkErr(err, logger, "", true) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + if err != nil { + logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") + } - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) - pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) - worker := app.NewWorker(logger, *pub) + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) + classifier := detect.NewClassifier(matcher) + worker := app.NewWorker(logger, pub, classifier) go func() { <-sig logger.Info().Msg("Shutdown requested...") cancel() - iter.Stop() + runtimeDeps.Iter.Stop() }() deps := app.RunnerDeps{ Logger: logger, WorkerCount: cfg.WorkerCount, - Iter: iter, - Worker: *worker, - NC: nc, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, } app.Run(ctx, deps) - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() + logger.Info().Msgf("Disconnecting from NATS at %s", runtimeDeps.NC.ConnectedUrl()) + if err := runtimeDeps.NC.Flush(); err != nil { + logger.Error().Err(err).Msg("failed to flush nats connection") + } + runtimeDeps.NC.Close() + + if err := ctx.Err(); err != nil && err != context.Canceled { + logger.Error().Err(err).Msg("service exited with context error") + } } From 1d19257e48a5e24e6863cb27b8b9723f08f7a172 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:22 -0300 Subject: [PATCH 21/34] move fingerprint data fixtures to own json files --- .../detect/data/cname_fingerprints.json | 227 ++++++++ .../internal/detect/data/ns_fingerprints.json | 416 +++++++++++++ .../internal/detect/fingerprint_loader.go | 107 ++++ .../internal/detect/fingerprints.go | 546 +----------------- 4 files changed, 781 insertions(+), 515 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_loader.go diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json new file mode 100644 index 0000000000..cbf3aa24be --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json @@ -0,0 +1,227 @@ +[ + { + "cname": [ + "elasticbeanstalk.com" + ], + "name": "AWS/Elastic Beanstalk", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "s3.amazonaws.com" + ], + "name": "AWS/S3", + "nxdomain": false, + "fingerprint": "The specified bucket does not exist" + }, + { + "cname": [ + "agilecrm.com" + ], + "name": "Agile CRM", + "nxdomain": false, + "fingerprint": "Sorry, this page is no longer available." + }, + { + "cname": [ + "airee.ru" + ], + "name": "Airee.ru", + "nxdomain": false, + "fingerprint": "Ошибка 402. Сервис Айри.рф не оплачен" + }, + { + "cname": [ + "animaapp.io" + ], + "name": "Anima", + "nxdomain": false, + "fingerprint": "The page you were looking for does not exist." + }, + { + "cname": [ + "bitbucket.io" + ], + "name": "Bitbucket", + "nxdomain": false, + "fingerprint": "Repository not found" + }, + { + "cname": [ + "trydiscourse.com" + ], + "name": "Discourse", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "furyns.com" + ], + "name": "Gemfury", + "nxdomain": false, + "fingerprint": "404: This page could not be found." + }, + { + "cname": [ + "ghost.io" + ], + "name": "Ghost", + "nxdomain": false, + "fingerprint": "Site unavailable\\.\u0026#124;Failed to resolve DNS path for this host" + }, + { + "cname": [ + "hatenablog.com" + ], + "name": "HatenaBlog", + "nxdomain": false, + "fingerprint": "404 Blog is not found" + }, + { + "cname": [ + "helpjuice.com" + ], + "name": "Help Juice", + "nxdomain": false, + "fingerprint": "We could not find what you're looking for." + }, + { + "cname": [ + "helpscoutdocs.com" + ], + "name": "Help Scout", + "nxdomain": false, + "fingerprint": "No settings were found for this company:" + }, + { + "cname": [ + "helprace.com" + ], + "name": "Helprace", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=301" + }, + { + "cname": [ + "youtrack.cloud" + ], + "name": "JetBrains", + "nxdomain": false, + "fingerprint": "is not a registered InCloud YouTrack" + }, + { + "cname": [ + "launchrock.com" + ], + "name": "LaunchRock", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=500" + }, + { + "cname": [ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com" + ], + "name": "Microsoft Azure", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "ngrok.io" + ], + "name": "Ngrok", + "nxdomain": false, + "fingerprint": "Tunnel .*.ngrok.io not found" + }, + { + "cname": [ + "readme.io" + ], + "name": "Readme.io", + "nxdomain": false, + "fingerprint": "The creators of this project are still working on making everything perfect!" + }, + { + "cname": [ + "52.16.160.97" + ], + "name": "SmartJobBoard", + "nxdomain": false, + "fingerprint": "This job board website is either expired or its domain name is invalid." + }, + { + "cname": [ + "s.strikinglydns.com" + ], + "name": "Strikingly", + "nxdomain": false, + "fingerprint": "PAGE NOT FOUND." + }, + { + "cname": [ + "na-west1.surge.sh" + ], + "name": "Surge.sh", + "nxdomain": false, + "fingerprint": "project not found" + }, + { + "cname": [ + "surveysparrow.com" + ], + "name": "SurveySparrow", + "nxdomain": false, + "fingerprint": "Account not found." + }, + { + "cname": [ + "read.uberflip.com" + ], + "name": "Uberflip", + "nxdomain": false, + "fingerprint": "The URL you've accessed does not provide a hub." + }, + { + "cname": [ + "stats.uptimerobot.com" + ], + "name": "Uptimerobot", + "nxdomain": false, + "fingerprint": "page not found" + }, + { + "cname": [ + "wordpress.com" + ], + "name": "Wordpress", + "nxdomain": false, + "fingerprint": "Do you want to register .*.wordpress.com?" + }, + { + "cname": [ + "worksites.net", + "69.164.223.206" + ], + "name": "Worksites", + "nxdomain": false, + "fingerprint": "Hello! Sorry, but the website you\u0026rsquo;re looking for doesn\u0026rsquo;t exist." + } +] diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json new file mode 100644 index 0000000000..85fd5262bf --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json @@ -0,0 +1,416 @@ +[ + { + "name": "000Domains", + "provider_url": "https://000domains.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.000domains.com", + "ns2.000domains.com", + "fwns1.000domains.com", + "fwns2.000domains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/19" + }, + { + "name": "AWS Route 53", + "provider_url": "https://aws.amazon.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns-*.awsdns-*.org", + "ns-*.awsdns-*.co.uk", + "ns-*.awsdns-*.com", + "ns-*.awsdns-*.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/1" + }, + { + "name": "Azure (Microsoft)", + "provider_url": "https://azure.microsoft.com/", + "status": "edge_case", + "host_patterns": [ + "ns1-*.azure-dns.com", + "ns2-*.azure-dns.net", + "ns3-*.azure-dns.org", + "ns4-*.azure-dns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/5" + }, + { + "name": "BigCommerce", + "provider_url": "https://bigcommerce.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bigcommerce.com", + "ns2.bigcommerce.com", + "ns3.bigcommerce.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/35" + }, + { + "name": "Bizland", + "provider_url": "https://bizland.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bizland.com", + "ns2.bizland.com", + "clickme.click2site.com", + "clickme2.click2site.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/3" + }, + { + "name": "ClouDNS", + "provider_url": "https://cloudns.net/", + "status": "not_vulnerable", + "host_patterns": [ + "*.cloudns.net" + ] + }, + { + "name": "Cloudflare", + "provider_url": "https://cloudflare.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.ns.cloudflare.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/10" + }, + { + "name": "Digital Ocean", + "provider_url": "https://digitalocean.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/22" + }, + { + "name": "DNSMadeEasy", + "provider_url": "https://dnsmadeeasy.com/", + "status": "vulnerable", + "host_patterns": [ + "ns*.dnsmadeeasy.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/6" + }, + { + "name": "DNSimple", + "provider_url": "https://dnsimple.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.dnsimple.com", + "ns2.dnsimple.com", + "ns3.dnsimple.com", + "ns4.dnsimple.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/16" + }, + { + "name": "Domain.com", + "provider_url": "https://domain.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1.domain.com", + "ns2.domain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/17" + }, + { + "name": "DomainPeople", + "provider_url": "https://domainpeople.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.domainpeople.com", + "ns2.domainpeople.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/14" + }, + { + "name": "Dotster", + "provider_url": "https://dotster.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.dotster.com", + "ns2.dotster.com", + "ns1.nameresolve.com", + "ns2.nameresolve.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/18" + }, + { + "name": "Dreamhost", + "provider_url": "https://dreamhost.com/", + "status": "edge_case", + "host_patterns": [ + "ns1.dreamhost.com", + "ns2.dreamhost.com", + "ns3.dreamhost.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/40" + }, + { + "name": "EasyDNS", + "provider_url": "https://easydns.com/", + "status": "not_vulnerable", + "host_patterns": [ + "dns1.easydns.com", + "dns2.easydns.net", + "dns3.easydns.org", + "dns4.easydns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/9" + }, + { + "name": "Gandi.net", + "provider_url": "https://gandi.net/", + "status": "not_vulnerable", + "host_patterns": [ + "a.dns.gandi.net", + "b.dns.gandi.net", + "c.dns.gandi.net" + ] + }, + { + "name": "Google Cloud", + "provider_url": "https://cloud.google.com/", + "status": "edge_case", + "host_patterns": [ + "ns-cloud-*.googledomains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/2" + }, + { + "name": "Hostinger (old NS)", + "provider_url": "https://hostinger.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hostinger.com", + "ns2.hostinger.com" + ] + }, + { + "name": "Hover", + "provider_url": "https://hover.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hover.com", + "ns2.hover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/21" + }, + { + "name": "Hurricane Electric", + "provider_url": "https://dns.he.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.he.net", + "ns2.he.net", + "ns3.he.net", + "ns4.he.net", + "ns5.he.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/25" + }, + { + "name": "Linode", + "provider_url": "https://linode.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.linode.com", + "ns2.linode.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/26" + }, + { + "name": "MediaTemple (mt)", + "provider_url": "https://mediatemple.net/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mediatemple.net", + "ns2.mediatemple.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/23" + }, + { + "name": "MyDomain", + "provider_url": "https://mydomain.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mydomain.com", + "ns2.mydomain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/4" + }, + { + "name": "Name.com", + "provider_url": "https://name.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1*.name.com", + "ns2*.name.com", + "ns3*.name.com", + "ns4*.name.com" + ], + "instructions_url": "https://github.com/libertalialtd/can-i-take-over-dns/issues/8" + }, + { + "name": "Namecheap", + "provider_url": "https://namecheap.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.namecheaphosting.com", + "*.registrar-servers.com" + ] + }, + { + "name": "Network Solutions", + "provider_url": "https://networksolutions.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.worldnic.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/15" + }, + { + "name": "NS1", + "provider_url": "https://nsone.net/", + "status": "registration_closed", + "host_patterns": [ + "dns1.p*.nsone.net", + "dns2.p*.nsone.net", + "dns3.p*.nsone.net", + "dns4.p*.nsone.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/7" + }, + { + "name": "TierraNet", + "provider_url": "https://tierra.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.domaindiscover.com", + "ns2.domaindiscover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/24" + }, + { + "name": "Reg.ru", + "provider_url": "https://reg.ru/", + "status": "vulnerable", + "host_patterns": [ + "ns1.reg.ru", + "ns2.reg.ru" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/28" + }, + { + "name": "UltraDNS", + "provider_url": "https://www.home.neustar/dns-services/ultra-dns", + "status": "not_vulnerable", + "host_patterns": [ + "pdns*.ultradns.com", + "udns*.ultradns.com", + "sdns*.ultradns.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/29" + }, + { + "name": "Yahoo Small Business", + "provider_url": "https://yahoosmallbusiness.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "yns1.yahoo.com", + "yns2.yahoo.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/20" + }, + { + "name": "Activision", + "provider_url": "https://activision.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.activision.com" + ], + "private_dns": true + }, + { + "name": "Adobe", + "provider_url": "https://adobe.com/", + "status": "not_vulnerable", + "host_patterns": [ + "adobe-dns-0*.adobe.com" + ], + "private_dns": true + }, + { + "name": "Apple", + "provider_url": "https://apple.com/", + "status": "not_vulnerable", + "host_patterns": [ + "a.ns.apple.com", + "b.ns.apple.com", + "c.ns.apple.com", + "d.ns.apple.com" + ], + "private_dns": true + }, + { + "name": "Automattic", + "provider_url": "https://automattic.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.automattic.com" + ], + "private_dns": true + }, + { + "name": "Capital One", + "provider_url": "https://capitalone.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.capitalone.com" + ], + "private_dns": true + }, + { + "name": "Disney", + "provider_url": "https://disney.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.twdcns.com", + "ns*.twdcns.info", + "ns*.twdcns.co.uk" + ], + "private_dns": true + }, + { + "name": "Google", + "provider_url": "https://google.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.google.com" + ], + "private_dns": true + }, + { + "name": "Lowe's", + "provider_url": "https://lowes.com/", + "status": "not_vulnerable", + "host_patterns": [ + "authns*.lowes.com" + ], + "private_dns": true + }, + { + "name": "T-Mobile", + "provider_url": "https://tmobileus.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns10.tmobileus.com", + "ns10.tmobileus.net" + ], + "private_dns": true + } +] diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go new file mode 100644 index 0000000000..694bf833aa --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -0,0 +1,107 @@ +package detect + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" +) + +//go:embed data/*.json +var fingerprintFS embed.FS + +var ( + loadFingerprintsOnce sync.Once + loadFingerprintsErr error +) + +func LoadFingerprints() error { + loadFingerprintsOnce.Do(func() { + var cname []CNAMEProviderFingerprint + var ns []NSProviderFingerprint + + if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateCNAMEFingerprints(cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateNSFingerprints(ns); err != nil { + loadFingerprintsErr = err + return + } + + CNAMEProviderFingerprints = cname + NSProviderFingerprints = ns + }) + + return loadFingerprintsErr +} + +func loadJSON(path string, out any) error { + b, err := fingerprintFS.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + + return nil +} + +func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("cname fingerprint[%d] missing name", i) + } + if len(fp.Cname) == 0 { + return fmt.Errorf("cname fingerprint[%d] has no cname patterns", i) + } + + for j, cname := range fp.Cname { + if strings.TrimSpace(cname) == "" { + return fmt.Errorf("cname fingerprint[%d] has empty cname pattern at index %d", i, j) + } + } + + if strings.TrimSpace(fp.Fingerprint) == "" { + return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) + } + + fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + } + + return nil +} + +func validateNSFingerprints(fingerprints []NSProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("ns fingerprint[%d] missing name", i) + } + + if len(fp.HostPatterns) == 0 { + return fmt.Errorf("ns fingerprint[%d] has no host patterns", i) + } + + for j, pattern := range fp.HostPatterns { + if strings.TrimSpace(pattern) == "" { + return fmt.Errorf("ns fingerprint[%d] has empty host pattern at index %d", i, j) + } + } + } + + return nil +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index b7098341a2..d1491995cd 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -3,12 +3,23 @@ package detect import "strings" type CNAMEProviderFingerprint struct { - Cname []string - Name string - Nxdomain bool - Fingerprint string + Cname []string `json:"cname"` + Name string `json:"name"` + Nxdomain bool `json:"nxdomain"` + Fingerprint string `json:"fingerprint"` + Mode FingerprintMode `json:"mode,omitempty"` } +type FingerprintMode string + +const ( + FingerprintModeLiteral FingerprintMode = "literal" + FingerprintModeRegex FingerprintMode = "regex" +) + +var CNAMEProviderFingerprints []CNAMEProviderFingerprint +var NSProviderFingerprints []NSProviderFingerprint + type NSProviderStatus string const ( @@ -20,12 +31,12 @@ const ( ) type NSProviderFingerprint struct { - Name string - ProviderURL string - Status NSProviderStatus - HostPatterns []string - InstructionsURL string - PrivateDNS bool + Name string `json:"name"` + ProviderURL string `json:"provider_url"` + Status NSProviderStatus `json:"status"` + HostPatterns []string `json:"host_patterns"` + InstructionsURL string `json:"instructions_url,omitempty"` + PrivateDNS bool `json:"private_dns,omitempty"` } func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { @@ -75,285 +86,6 @@ func wildcardHostMatch(pattern, host string) bool { return true } -var NSProviderFingerprints = []NSProviderFingerprint{ - { - Name: "000Domains", - ProviderURL: "https://000domains.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", - }, - { - Name: "AWS Route 53", - ProviderURL: "https://aws.amazon.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", - }, - { - Name: "Azure (Microsoft)", - ProviderURL: "https://azure.microsoft.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", - }, - { - Name: "BigCommerce", - ProviderURL: "https://bigcommerce.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", - }, - { - Name: "Bizland", - ProviderURL: "https://bizland.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", - }, - { - Name: "ClouDNS", - ProviderURL: "https://cloudns.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.cloudns.net"}, - }, - { - Name: "Cloudflare", - ProviderURL: "https://cloudflare.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.ns.cloudflare.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", - }, - { - Name: "Digital Ocean", - ProviderURL: "https://digitalocean.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", - }, - { - Name: "DNSMadeEasy", - ProviderURL: "https://dnsmadeeasy.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns*.dnsmadeeasy.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", - }, - { - Name: "DNSimple", - ProviderURL: "https://dnsimple.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", - }, - { - Name: "Domain.com", - ProviderURL: "https://domain.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", - }, - { - Name: "DomainPeople", - ProviderURL: "https://domainpeople.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", - }, - { - Name: "Dotster", - ProviderURL: "https://dotster.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", - }, - { - Name: "Dreamhost", - ProviderURL: "https://dreamhost.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", - }, - { - Name: "EasyDNS", - ProviderURL: "https://easydns.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", - }, - { - Name: "Gandi.net", - ProviderURL: "https://gandi.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, - }, - { - Name: "Google Cloud", - ProviderURL: "https://cloud.google.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns-cloud-*.googledomains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", - }, - { - Name: "Hostinger (old NS)", - ProviderURL: "https://hostinger.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, - }, - { - Name: "Hover", - ProviderURL: "https://hover.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", - }, - { - Name: "Hurricane Electric", - ProviderURL: "https://dns.he.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", - }, - { - Name: "Linode", - ProviderURL: "https://linode.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", - }, - { - Name: "MediaTemple (mt)", - ProviderURL: "https://mediatemple.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", - }, - { - Name: "MyDomain", - ProviderURL: "https://mydomain.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", - }, - { - Name: "Name.com", - ProviderURL: "https://name.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, - InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", - }, - { - Name: "Namecheap", - ProviderURL: "https://namecheap.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, - }, - { - Name: "Network Solutions", - ProviderURL: "https://networksolutions.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.worldnic.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", - }, - { - Name: "NS1", - ProviderURL: "https://nsone.net/", - Status: NSStatusRegistrationClosed, - HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", - }, - { - Name: "TierraNet", - ProviderURL: "https://tierra.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", - }, - { - Name: "Reg.ru", - ProviderURL: "https://reg.ru/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", - }, - { - Name: "UltraDNS", - ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", - }, - { - Name: "Yahoo Small Business", - ProviderURL: "https://yahoosmallbusiness.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", - }, - { - Name: "Activision", - ProviderURL: "https://activision.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.activision.com"}, - PrivateDNS: true, - }, - { - Name: "Adobe", - ProviderURL: "https://adobe.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"adobe-dns-0*.adobe.com"}, - PrivateDNS: true, - }, - { - Name: "Apple", - ProviderURL: "https://apple.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, - PrivateDNS: true, - }, - { - Name: "Automattic", - ProviderURL: "https://automattic.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.automattic.com"}, - PrivateDNS: true, - }, - { - Name: "Capital One", - ProviderURL: "https://capitalone.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.capitalone.com"}, - PrivateDNS: true, - }, - { - Name: "Disney", - ProviderURL: "https://disney.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, - PrivateDNS: true, - }, - { - Name: "Google", - ProviderURL: "https://google.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.google.com"}, - PrivateDNS: true, - }, - { - Name: "Lowe's", - ProviderURL: "https://lowes.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"authns*.lowes.com"}, - PrivateDNS: true, - }, - { - Name: "T-Mobile", - ProviderURL: "https://tmobileus.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, - PrivateDNS: true, - }, -} - func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { @@ -363,230 +95,14 @@ func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { return false } -var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ - { - Cname: []string{ - "elasticbeanstalk.com", - }, - Nxdomain: true, - Name: "AWS/Elastic Beanstalk", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "s3.amazonaws.com", - }, - Nxdomain: false, - Name: "AWS/S3", - Fingerprint: "The specified bucket does not exist", - }, - { - Cname: []string{ - "agilecrm.com", - }, - Nxdomain: false, - Name: "Agile CRM", - Fingerprint: "Sorry, this page is no longer available.", - }, - { - Cname: []string{ - "airee.ru", - }, - Nxdomain: false, - Name: "Airee.ru", - Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", - }, - { - Cname: []string{ - "animaapp.io", - }, - Nxdomain: false, - Name: "Anima", - Fingerprint: "The page you were looking for does not exist.", - }, - { - Cname: []string{ - "bitbucket.io", - }, - Nxdomain: false, - Name: "Bitbucket", - Fingerprint: "Repository not found", - }, - { - Cname: []string{ - "trydiscourse.com", - }, - Nxdomain: true, - Name: "Discourse", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "furyns.com", - }, - Nxdomain: false, - Name: "Gemfury", - Fingerprint: "404: This page could not be found.", - }, - { - Cname: []string{ - "ghost.io", - }, - Nxdomain: false, - Name: "Ghost", - Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", - }, - { - Cname: []string{ - "hatenablog.com", - }, - Nxdomain: false, - Name: "HatenaBlog", - Fingerprint: "404 Blog is not found", - }, - { - Cname: []string{ - "helpjuice.com", - }, - Nxdomain: false, - Name: "Help Juice", - Fingerprint: "We could not find what you're looking for.", - }, - { - Cname: []string{ - "helpscoutdocs.com", - }, - Nxdomain: false, - Name: "Help Scout", - Fingerprint: "No settings were found for this company:", - }, - { - Cname: []string{ - "helprace.com", - }, - Nxdomain: false, - Name: "Helprace", - Fingerprint: "HTTP_STATUS=301", - }, - { - Cname: []string{ - "youtrack.cloud", - }, - Nxdomain: false, - Name: "JetBrains", - Fingerprint: "is not a registered InCloud YouTrack", - }, - { - Cname: []string{ - "launchrock.com", - }, - Nxdomain: false, - Name: "LaunchRock", - Fingerprint: "HTTP_STATUS=500", - }, - { - Cname: []string{ - "cloudapp.azure.com", - "azurewebsites.net", - "blob.core.windows.net", - "cloudapp.azure.com", - "azure-api.net", - "azurehdinsight.net", - "azureedge.net", - "azurecontainer.io", - "database.windows.net", - "azuredatalakestore.net", - "search.windows.net", - "azurecr.io", - "redis.cache.windows.net", - "azurehdinsight.net", - "servicebus.windows.net", - "trafficmanager.net", - "visualstudio.com", - }, - Nxdomain: true, - Name: "Microsoft Azure", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "ngrok.io", - }, - Nxdomain: false, - Name: "Ngrok", - Fingerprint: "Tunnel .*.ngrok.io not found", - }, - { - Cname: []string{ - "readme.io", - }, - Nxdomain: false, - Name: "Readme.io", - Fingerprint: "The creators of this project are still working on making everything perfect!", - }, - { - Cname: []string{ - "52.16.160.97", - }, - Nxdomain: false, - Name: "SmartJobBoard", - Fingerprint: "This job board website is either expired or its domain name is invalid.", - }, - { - Cname: []string{ - "s.strikinglydns.com", - }, - Nxdomain: false, - Name: "Strikingly", - Fingerprint: "PAGE NOT FOUND.", - }, - { - Cname: []string{ - "na-west1.surge.sh", - }, - Nxdomain: false, - Name: "Surge.sh", - Fingerprint: "project not found", - }, - { - Cname: []string{ - "surveysparrow.com", - }, - Nxdomain: false, - Name: "SurveySparrow", - Fingerprint: "Account not found.", - }, - { - Cname: []string{ - "read.uberflip.com", - }, - Nxdomain: false, - Name: "Uberflip", - Fingerprint: "The URL you've accessed does not provide a hub.", - }, - { - Cname: []string{ - "stats.uptimerobot.com", - }, - Nxdomain: false, - Name: "Uptimerobot", - Fingerprint: "page not found", - }, - { - Cname: []string{ - "wordpress.com", - }, - Nxdomain: false, - Name: "Wordpress", - Fingerprint: "Do you want to register .*.wordpress.com?", - }, - { - Cname: []string{ - "worksites.net", - "69.164.223.206", - }, - Nxdomain: false, - Name: "Worksites", - Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", - }, +func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { + if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { + return mode + } + + if strings.Contains(fingerprint, ".*") || strings.Contains(fingerprint, "\\") { + return FingerprintModeRegex + } + + return FingerprintModeLiteral } From 64e789c7376985bcdebebfcd4b0e21d5a8c5f0b6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:51 -0300 Subject: [PATCH 22/34] remove remediation.go file to serve translated text in api/frontend --- scanners/subdomain-takeover/internal/detect/remediation.go | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go deleted file mode 100644 index 9d577baa7f..0000000000 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ /dev/null @@ -1,5 +0,0 @@ -package detect - -func RemediationForReason(reasonCode string, provider string, lameType string) { - return -} From 2c873aba90bd4c635c5ef6fef467c97388bd27f6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:19:47 -0300 Subject: [PATCH 23/34] add RecordType enums --- scanners/subdomain-takeover/internal/model/finding.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 5d387c0f1d..57dbc83c67 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -1,9 +1,16 @@ package model +type RecordType string + +const ( + RecordTypeCNAME RecordType = "CNAME" + RecordTypeNS RecordType = "NS" +) + type Finding struct { Domain string `json:"domain"` DomainKey string `json:"domain_key"` - RecordType string `json:"record_type"` + RecordType RecordType `json:"record_type"` Target string `json:"target"` Provider string `json:"provider"` LameType string `json:"lame_type"` From f405a61604273509af31c3c86e0c547df5cd0695 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:11 -0300 Subject: [PATCH 24/34] add confidence and reason code enums --- .../internal/detect/confidence.go | 32 +++++++++++++++++++ .../internal/detect/reason_codes.go | 15 +++++++++ 2 files changed, 47 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/reason_codes.go diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index ad6c03d18d..9865a561db 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -1 +1,33 @@ package detect + +const ( + ConfidenceSuspected = "suspected" + ConfidenceProbable = "probable" + ConfidenceConfirmed = "confirmed" +) + +// ConfidenceForReason maps stable reason codes to confidence levels. +// Unknown reason codes default to suspected. +func ConfidenceForReason(reasonCode ReasonCode) string { + switch reasonCode { + case ReasonCNAMEDanglingNXDOMAIN: + return ConfidenceProbable + case ReasonCNAMEProviderFingerprintBodyMatch: + return ConfidenceProbable + case ReasonCNAMETargetMatchMissingNXDOMAIN: + return ConfidenceSuspected + case ReasonCNAMETargetMatchMissingBodyFP: + return ConfidenceSuspected + + case ReasonNSFullLameProviderVulnerable: + return ConfidenceConfirmed + case ReasonNSPartialLameProviderVulnerable: + return ConfidenceProbable + case ReasonNSLameProviderUnknown: + return ConfidenceSuspected + case ReasonNSProviderMatchOnly: + return ConfidenceSuspected + default: + return ConfidenceSuspected + } +} diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go new file mode 100644 index 0000000000..f080b18329 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -0,0 +1,15 @@ +package detect + +type ReasonCode string + +const ( + ReasonCNAMEDanglingNXDOMAIN ReasonCode = "CNAME_DANGLING_NXDOMAIN" + ReasonCNAMEProviderFingerprintBodyMatch ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + ReasonCNAMETargetMatchMissingNXDOMAIN ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" + ReasonCNAMETargetMatchMissingBodyFP ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + + ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" + ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" + ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" +) From d18030718304344f9b109a458eea3bab09c5f0a2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:45 -0300 Subject: [PATCH 25/34] move http interface for NS lookups to own file --- .../internal/detect/body_matcher.go | 65 +++++++++++++++++++ .../internal/detect/rules.go | 48 +++----------- 2 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go new file mode 100644 index 0000000000..622b2f7c51 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -0,0 +1,65 @@ +package detect + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +type BodyFingerprintMatcher interface { + Contains(domain string, fingerprint string, mode FingerprintMode) bool +} + +type HTTPBodyFingerprintMatcher struct { + client *http.Client +} + +func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { + return &HTTPBodyFingerprintMatcher{ + client: &http.Client{Timeout: timeout}, + } +} + +type NoopBodyFingerprintMatcher struct{} + +func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { + return &NoopBodyFingerprintMatcher{} +} + +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + return false +} + +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if m == nil || m.client == nil { + return false + } + + url := fmt.Sprintf("http://%s", domain) + res, err := m.client.Get(url) + if err != nil { + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false + } + + bodyText := string(body) + resolvedMode := normalizeFingerprintMode(mode, fingerprint) + + if resolvedMode == FingerprintModeRegex { + re, err := regexp.Compile(fingerprint) + if err != nil { + return false + } + return re.MatchString(bodyText) + } + + return strings.Contains(bodyText, fingerprint) +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 9cb5ac5738..44626a372b 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,28 +1,21 @@ package detect -import ( - "fmt" - "io" - "net/http" - "strings" -) - type CNAMEHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode NeedsNX bool } type NSHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode ProviderVulnerable bool RegistrarMismatch bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { hit := CNAMEHit{ @@ -32,16 +25,17 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider NeedsNX: fp.Nxdomain, } if hit.NeedsNX { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" - if evidence.QueryAnswer.A == "NXDOMAIN" { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { hit.Matched = true - hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN } } else { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" - if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true - hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } return &hit @@ -67,25 +61,3 @@ func ShouldEmitCNAME(hit *CNAMEHit) bool { func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } - -func urlBodyContainsFingerprint(domain string, fp string) bool { - // curl domain and look for matching fp in body - url := fmt.Sprintf("http://%s", domain) - res, err := http.Get(url) - if err != nil { - // log error - return false - } - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - // log err - return false - } - - if strings.Contains(string(body), fp) { - return true - } - return false -} From ad7c2729c124905f932f7eb211e733b7edfb0a59 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:21:55 -0300 Subject: [PATCH 26/34] switch from using QueryAnswers struct field to NoResolve bool for CNAME evidence --- .../internal/detect/evidence.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index fb40318cfc..ba54926bb8 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -8,9 +8,9 @@ import ( // inputs from evidence.go type CNAMEEvidence struct { - Domain string - Target string - QueryAnswer model.QueryAnswers // or just needed fields + Domain string + Target string + NoResolve bool } type NSEvidence struct { @@ -26,9 +26,9 @@ func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { } return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - QueryAnswer: input.QueryAnswers, + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + NoResolve: input.ResolveChain == nil, } } @@ -46,6 +46,6 @@ func ClassifyLameType(nsChecks []model.NsCheck) { func parseCname(record string) string { trimmed := strings.Trim(record, ".") lower := strings.ToLower(trimmed) - record_tokens := strings.Split(lower, " ") - return record_tokens[len(record_tokens)-1] + recordTokens := strings.Split(lower, " ") + return recordTokens[len(recordTokens)-1] } From 967a0882a87a96e7a8499551667342aa36e65010 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:05 -0300 Subject: [PATCH 27/34] add contructor for Classifier --- .../internal/detect/classifier.go | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 0caea61196..28fb59649b 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,33 +2,48 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" -func Classify(input model.Input) ([]model.Finding, error) { +type Classifier struct { + Matcher BodyFingerprintMatcher +} + +func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { + return &Classifier{Matcher: matcher} +} + +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { + matcher := c.Matcher + if matcher == nil { + matcher = NewNoopBodyFingerprintMatcher() + } + + return Classify(input, matcher) +} + +func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} cnameEvidence := ExtractCNAMEEvidence(input) - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(cnameHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: cnameEvidence.Domain, - RecordType: "CNAME", - Target: cnameEvidence.Target, - Provider: cnameHit.Provider, - ReasonCode: cnameHit.ReasonCode, - Confidence: "", - Remediation: "", - }) + if cnameEvidence != nil { + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + if ShouldEmitCNAME(cnameHit) { + findings = append(findings, model.Finding{ + Domain: cnameEvidence.Domain, + RecordType: model.RecordTypeCNAME, + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: string(cnameHit.ReasonCode), + Confidence: ConfidenceForReason(cnameHit.ReasonCode), + }) + } } - nsEvidence := ExtractNSEvidence(input) - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: input.Domain, - }) - } + // nsEvidence := ExtractNSEvidence(input) + // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + // if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // findings = append(findings, model.Finding{ + // Domain: input.Domain, + // }) + // } + return findings, nil } From 92a05793c311af658b10577d8c418277e8d1ff90 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:15 -0300 Subject: [PATCH 28/34] update README.md --- scanners/subdomain-takeover/README.md | 117 +++++++++----------------- 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0bfcdaf9ba..640b246776 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,10 +5,17 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +2. Evaluates passive CNAME takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` +## Current detection scope + +- Implemented: CNAME-based passive detection +- Not implemented yet: NS-based (Sitting Duck) detection + +The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. + ## Passive-only detection policy This service uses passive checks only. @@ -19,63 +26,36 @@ As a result, confidence is conservative and evidence-driven: - `probable`: strong passive indicators with known exploitable conditions - `confirmed`: reserved for deterministic passive signatures only -## Sitting Duck DNS hijack coverage +## Confidence model (current) -The scanner includes passive detection for Sitting Duck style DNS hijack risk. +Current production behavior for CNAME findings: -A Sitting Duck takeover is considered possible when all of the following are true: - -1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. -2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). -3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. +- `suspected` + - provider target matched, but required passive signal is missing +- `probable` + - dangling target with NXDOMAIN where required, or + - provider fingerprint body match for providers that require body verification +- `confirmed` + - reserved; not currently emitted by CNAME rules -### Variations covered +## Fingerprint data -- Partially lame delegation: some NS records are lame, others still answer. -- Full lame delegation: all authoritative NS are lame. -- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. +Provider fingerprint data is stored in: -If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. +- `internal/detect/data/cname_fingerprints.json` +- `internal/detect/data/ns_fingerprints.json` -## Confidence model +The service embeds and validates these files at startup (`detect.LoadFingerprints`). -This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. +For CNAME body matching: -- `suspected` - - dangling/suspicious DNS pattern only, or - - lame delegation evidence without known exploitable-provider confirmation -- `probable` - - DNS pattern + known provider unclaimed signature, or - - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match -- `confirmed` - - explicit passive takeover indicator with deterministic evidence - - if deterministic passive proof is missing, cap at `probable` - -## Required evidence for findings - -Each finding should include: -- `domain` -- `domain_key` -- `record_type` (`CNAME` or `NS`) -- `target` or `ns_host` -- `provider` -- `lame_type` (`partial` or `full`) for Sitting Duck findings -- per-NS resolver outcomes where applicable -- `confidence` -- `reason_code` -- remediation guidance - -## Remediation guidance - -Typical remediation: -1. Fix delegation at registrar and authoritative DNS provider. -2. Remove stale or dead NS entries. -3. Reclaim or securely rebind abandoned DNS zones or accounts. -4. Validate ownership controls for DNS provider account workflows. +- `mode: literal` performs plain substring search +- `mode: regex` performs regular expression matching +- if `mode` is omitted, the loader applies backward-compatible inference ## Prerequisites -- Go 1.24+ +- Go 1.25+ - NATS with JetStream enabled - `nats` CLI (optional, for local testing) @@ -95,43 +75,24 @@ Environment variables: - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) -Local test flow: - -Publish a test request: -```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' -``` +## Message ack behavior -Watch findings: -```bash -nats sub "scans.findings.upsert" -``` +The worker uses explicit JetStream ack semantics: -## Development milestones +- decode failures -> `Term()` (drop poison message) +- classify/publish failures -> `Nak()` (retryable) +- successful processing -> `Ack()` -1. Build config + NATS consume/publish loop. -2. Add DNS evidence extraction. -3. Add provider fingerprint rule engine. -4. Add confidence scoring and remediation text. -5. Add tests for classification rules and message handling. +## Local test flow ---- +Publish a test request: -## External Go modules to install +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +``` -Start minimal: +Watch findings: ```bash -go get github.com/nats-io/nats.go -go get github.com/miekg/dns -go get github.com/rs/zerolog -go get golang.org/x/sync/errgroup -go get github.com/stretchr/testify +nats sub "scans.findings.upsert" ``` - -What each is for: -- `nats.go`: JetStream consume/publish -- `miekg/dns`: robust DNS lookups and record parsing -- `zerolog`: structured logs without heavy setup -- `errgroup`: clean goroutine lifecycle management -- `testify`: easier assertions in unit tests From 9ba9e1fd9bebf2e4aa7390f46a51849e7ad686e2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:04:05 -0300 Subject: [PATCH 29/34] remove unneeded scan result fixtures --- .../fixtures/cyber-gc-ca_dns_scan.json | 202 ------------------ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 -- 2 files changed, 225 deletions(-) delete mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json delete mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json deleted file mode 100644 index cad720cc93..0000000000 --- a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "domain": "cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - [ - "cyber.gc.ca. 7161 IN A 20.151.96.73" - ] - ], - "resolve_ips": [ - "20.151.96.73" - ], - "cname_record": null, - "mx_records": { - "hosts": [ - { - "preference": 10, - "hostname": "cyber-gc-ca.mail.protection.outlook.com", - "addresses": [ - "52.101.190.1", - "52.101.192.0", - "52.101.192.1", - "52.101.190.0" - ] - } - ], - "warnings": [] - }, - "ns_records": { - "hostnames": [ - "ns1.d-zone.ca", - "ns2.d-zone.ca", - "ns40.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns10.ent.global.gc.ca" - ], - "warnings": [] - }, - "dkim": {}, - "spf": { - "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", - "valid": true, - "dns_void_lookups": 0, - "warnings": [], - "parsed": { - "pass": [ - { - "value": "40.80.253.57", - "mechanism": "ip4" - }, - { - "value": "52.139.3.76", - "mechanism": "a" - }, - { - "value": "205.193.218.114", - "mechanism": "a" - }, - { - "value": "205.193.218.115", - "mechanism": "a" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [ - { - "domain": "spf.protection.outlook.com", - "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", - "dns_void_lookups": 0, - "parsed": { - "pass": [ - { - "value": "40.92.0.0/15", - "mechanism": "ip4" - }, - { - "value": "40.107.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.100.0.0/15", - "mechanism": "ip4" - }, - { - "value": "52.102.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.103.0.0/17", - "mechanism": "ip4" - }, - { - "value": "104.47.0.0/17", - "mechanism": "ip4" - }, - { - "value": "2a01:111:f400::/48", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403::/49", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:8000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:c000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:f000::/52", - "mechanism": "ip6" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "warnings": [], - "spf_default": "fail", - "lookups": 0 - } - ], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "spf_default": "fail", - "lookups": 3 - }, - "dmarc": { - "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", - "valid": true, - "location": "cyber.gc.ca", - "warnings": [], - "tags": { - "v": { - "value": "DMARC1", - "explicit": true - }, - "p": { - "value": "reject", - "explicit": true - }, - "pct": { - "value": 100, - "explicit": true - }, - "rua": { - "value": [ - { - "scheme": "mailto", - "address": "dmarc@cyber.gc.ca", - "size_limit": null, - "accepting": true - } - ], - "explicit": true - }, - "adkim": { - "value": "r", - "explicit": false - }, - "aspf": { - "value": "r", - "explicit": false - }, - "fo": { - "value": "0", - "explicit": false - }, - "rf": { - "value": "afrf", - "explicit": false - }, - "ri": { - "value": 86400, - "explicit": false - }, - "sp": { - "value": "reject", - "explicit": false - } - }, - "effective_policy_source": "p", - "effective_policy": "reject" - }, - "wildcard_sibling": false, - "wildcard_entry": false, - "zone_dnssec_enabled": null -} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json deleted file mode 100644 index 8f546a74a5..0000000000 --- a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "domain": "www.cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "www.cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], - ["cyber.gc.ca. 5550 IN A 20.151.96.73"] - ], - "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", - "ns_records": { - "hostnames": [ - "ns40.ent.global.gc.ca", - "ns10.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns2.d-zone.ca", - "ns1.d-zone.ca" - ], - "warnings": [] - } -} From 16dc260e13ba15f1899b3403ea1799ad898e3cf0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:05:11 -0300 Subject: [PATCH 30/34] update model.input to take in published dns_scanner_results payload --- .../internal/detect/classifier.go | 3 +- .../internal/detect/evidence.go | 17 ++++---- .../internal/messaging/consumer.go | 1 - .../internal/model/finding.go | 17 ++++---- .../internal/model/input.go | 41 +++++++++++-------- 5 files changed, 44 insertions(+), 35 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 28fb59649b..5d3e3ad110 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -22,12 +22,13 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} - cnameEvidence := ExtractCNAMEEvidence(input) + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, + DomainKey: input.DomainKey, RecordType: model.RecordTypeCNAME, Target: cnameEvidence.Target, Provider: cnameHit.Provider, diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index ba54926bb8..f035d01c76 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -20,26 +20,29 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { - if input.CnameRecord == nil { +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil { return nil } + domain := results.Domain + return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - NoResolve: input.ResolveChain == nil, + Domain: *domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, } } func ExtractNSEvidence(input model.Input) *NSEvidence { - if len(input.NsDelegations.Hosts) == 0 { + nsDelegations := input.Results.NsDelegations + if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } return &NSEvidence{} } -func ClassifyLameType(nsChecks []model.NsCheck) { +func ClassifyLameType(nsChecks []any) { return } diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go deleted file mode 100644 index cbb7a43f8b..0000000000 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ /dev/null @@ -1 +0,0 @@ -package messaging diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 57dbc83c67..66e98a2f35 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -8,13 +8,12 @@ const ( ) type Finding struct { - Domain string `json:"domain"` - DomainKey string `json:"domain_key"` - RecordType RecordType `json:"record_type"` - Target string `json:"target"` - Provider string `json:"provider"` - LameType string `json:"lame_type"` - Confidence string `json:"confidence"` - ReasonCode string `json:"reason_code"` - Remediation string `json:"remediation"` + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType RecordType `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 7bdd179f73..263e74928d 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,34 +1,30 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` - QueryAnswers QueryAnswers `json:"query_res"` - NsDelegations NsDelegations `json:"ns_delegations"` + DomainKey string `json:"domain_key"` + Results ScanResults `json:"results"` +} + +type ScanResults struct { + Domain *string `json:"domain"` + NsDelegations *NsDelegations `json:"ns_delegations"` + RegistrarContext *RegistrarContext `json:"registrar_context"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords *NsRecords `json:"ns_records"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` -} - -type QueryAnswers struct { - A string `json:"A"` - SOA string `json:"SOA"` - NS string `json:"NS"` - CNAME string `json:"CNAME"` + Error string `json:"error"` } type NsDelegations struct { Hosts []string `json:"ns_hosts"` Checks []NsCheck `json:"ns_checks"` Delegation Delegation `json:"ns_delegation"` + Error string `json:"error"` } type NsCheck struct { @@ -46,3 +42,14 @@ type Delegation struct { LameCount int `json:"lame_count"` LameType string `json:"lame_type"` } + +type RegistrarContext struct { + BaseDomain string `json:"base_domain"` + LookupSuccess bool `json:"lookup_success"` + RDAPURL string `json:"rdap_url"` + RegistrarName string `json:"registrar_name"` + RegistrarID string `json:"registrar_id"` + RDAPNameservers []string `json:"rdap_nameservers"` + DelegationMatchesRDAP *bool `json:"delegation_matches_rdap"` + Error string `json:"error"` +} From 50d653a6728c11ab11f96a22d282b9fbd443ef52 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:15:22 -0300 Subject: [PATCH 31/34] add logic to emit nameserver hijacking exploitability --- .../internal/detect/classifier.go | 26 +++-- .../internal/detect/evidence.go | 42 +++++--- .../internal/detect/rules.go | 95 ++++++++++++++++--- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5d3e3ad110..c3883063a0 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1,6 +1,8 @@ package detect -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) type Classifier struct { Matcher BodyFingerprintMatcher @@ -38,13 +40,21 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin } } - // nsEvidence := ExtractNSEvidence(input) - // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - // if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // findings = append(findings, model.Finding{ - // Domain: input.Domain, - // }) - // } + nsEvidence := ExtractNSEvidence(input.Results) + if nsEvidence != nil { + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(nsHit) { + findings = append(findings, model.Finding{ + Domain: nsEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeNS, + Target: nsHit.Host, + Provider: nsHit.Provider, + ReasonCode: string(nsHit.ReasonCode), + Confidence: ConfidenceForReason(nsHit.ReasonCode), + }) + } + } return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f035d01c76..7ea527de08 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,6 +1,7 @@ package detect import ( + "slices" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" @@ -21,34 +22,47 @@ type NSEvidence struct { } func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil { + if results.CnameRecord == nil || results.Domain == nil { return nil } - domain := results.Domain - return &CNAMEEvidence{ - Domain: *domain, + Domain: *results.Domain, Target: parseCname(*results.CnameRecord), NoResolve: len(results.ResolveChain) == 0, } } -func ExtractNSEvidence(input model.Input) *NSEvidence { - nsDelegations := input.Results.NsDelegations - if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { +func ExtractNSEvidence(input model.ScanResults) *NSEvidence { + nsDelegations := input.NsDelegations + if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } - return &NSEvidence{} -} -func ClassifyLameType(nsChecks []any) { - return + return &NSEvidence{ + Domain: *input.Domain, + NSHosts: parseHostnames(input.NsDelegations.Hosts), + NSDelegations: *input.NsDelegations, + } + } func parseCname(record string) string { - trimmed := strings.Trim(record, ".") - lower := strings.ToLower(trimmed) - recordTokens := strings.Split(lower, " ") + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") return recordTokens[len(recordTokens)-1] } + +func parseHostnames(hosts []string) []string { + var normalizedHosts = []string{} + for _, host := range hosts { + normalHost := normalizeHost(host) + normalizedHosts = append(normalizedHosts, normalHost) + } + slices.Sort(normalizedHosts) + return slices.Compact(normalizedHosts) +} + +func normalizeHost(host string) string { + return strings.Trim(strings.ToLower(host), ".") +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 44626a372b..fe08aabc33 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,5 +1,7 @@ package detect +import "strings" + type CNAMEHit struct { Matched bool Provider string @@ -8,11 +10,11 @@ type CNAMEHit struct { } type NSHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - ProviderVulnerable bool - RegistrarMismatch bool + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool } func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { @@ -24,6 +26,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider ReasonCode: "", NeedsNX: fp.Nxdomain, } + if hit.NeedsNX { hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN if evidence.NoResolve { @@ -38,6 +41,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } + return &hit } } @@ -45,19 +49,88 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return &NSHit{} + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best } func ShouldEmitCNAME(hit *CNAMEHit) bool { if hit == nil { return false } - if hit.Matched { - return true + return hit.Matched +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly } - return false } -func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { - return false +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } } From e20f1ea935bc89a3ad1a26e202b0c7b35badbd3c Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:20:24 -0300 Subject: [PATCH 32/34] refactor rules.go into separate cname and ns detection logic files --- .../internal/detect/classifier.go | 27 +++- .../internal/detect/cname_evidence.go | 31 ++++ .../internal/detect/cname_rules.go | 46 ++++++ .../internal/detect/fingerprint_source.go | 16 +++ .../detect/{evidence.go => ns_evidence.go} | 32 +---- .../internal/detect/ns_reasoning.go | 47 ++++++ .../internal/detect/ns_rules.go | 45 ++++++ .../internal/detect/rules.go | 136 ------------------ 8 files changed, 210 insertions(+), 170 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/cname_evidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source.go rename scanners/subdomain-takeover/internal/detect/{evidence.go => ns_evidence.go} (55%) create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules.go delete mode 100644 scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index c3883063a0..d3853749fd 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -6,10 +6,19 @@ import ( type Classifier struct { Matcher BodyFingerprintMatcher + Source FingerprintSource } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} +} + +func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { + if source == nil { + source = GlobalFingerprintSource{} + } + + return &Classifier{Matcher: matcher, Source: source} } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -18,15 +27,23 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { matcher = NewNoopBodyFingerprintMatcher() } - return Classify(input, matcher) + source := c.Source + if source == nil { + source = GlobalFingerprintSource{} + } + + return Classify(input, matcher, source) } -func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { findings := []model.Finding{} + cnameFingerprints := source.CNAME() + nsFingerprints := source.NS() + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, @@ -42,7 +59,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, diff --git a/scanners/subdomain-takeover/internal/detect/cname_evidence.go b/scanners/subdomain-takeover/internal/detect/cname_evidence.go new file mode 100644 index 0000000000..4466dcd9fc --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type CNAMEEvidence struct { + Domain string + Target string + NoResolve bool +} + +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil || results.Domain == nil { + return nil + } + + return &CNAMEEvidence{ + Domain: *results.Domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, + } +} + +func parseCname(record string) string { + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") + return recordTokens[len(recordTokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go new file mode 100644 index 0000000000..e69c5cddb0 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -0,0 +1,46 @@ +package detect + +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode ReasonCode + NeedsNX bool +} + +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + + if hit.NeedsNX { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN + } + } else { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + } + } + + return &hit + } + } + return nil +} + +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go new file mode 100644 index 0000000000..2ecc9981f8 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -0,0 +1,16 @@ +package detect + +type FingerprintSource interface { + CNAME() []CNAMEProviderFingerprint + NS() []NSProviderFingerprint +} + +type GlobalFingerprintSource struct{} + +func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { + return CNAMEProviderFingerprints +} + +func (GlobalFingerprintSource) NS() []NSProviderFingerprint { + return NSProviderFingerprints +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go similarity index 55% rename from scanners/subdomain-takeover/internal/detect/evidence.go rename to scanners/subdomain-takeover/internal/detect/ns_evidence.go index 7ea527de08..78ddb2f428 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -7,13 +7,6 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -// inputs from evidence.go -type CNAMEEvidence struct { - Domain string - Target string - NoResolve bool -} - type NSEvidence struct { Domain string NSHosts []string @@ -21,18 +14,6 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil || results.Domain == nil { - return nil - } - - return &CNAMEEvidence{ - Domain: *results.Domain, - Target: parseCname(*results.CnameRecord), - NoResolve: len(results.ResolveChain) == 0, - } -} - func ExtractNSEvidence(input model.ScanResults) *NSEvidence { nsDelegations := input.NsDelegations if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { @@ -44,21 +25,14 @@ func ExtractNSEvidence(input model.ScanResults) *NSEvidence { NSHosts: parseHostnames(input.NsDelegations.Hosts), NSDelegations: *input.NsDelegations, } - -} - -func parseCname(record string) string { - normalRecord := normalizeHost(record) - recordTokens := strings.Split(normalRecord, " ") - return recordTokens[len(recordTokens)-1] } func parseHostnames(hosts []string) []string { - var normalizedHosts = []string{} + normalizedHosts := make([]string, 0, len(hosts)) for _, host := range hosts { - normalHost := normalizeHost(host) - normalizedHosts = append(normalizedHosts, normalHost) + normalizedHosts = append(normalizedHosts, normalizeHost(host)) } + slices.Sort(normalizedHosts) return slices.Compact(normalizedHosts) } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go new file mode 100644 index 0000000000..953cee5580 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -0,0 +1,47 @@ +package detect + +import "strings" + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly + } +} + +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go new file mode 100644 index 0000000000..ff993ef28b --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -0,0 +1,45 @@ +package detect + +type NSHit struct { + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool +} + +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go deleted file mode 100644 index fe08aabc33..0000000000 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ /dev/null @@ -1,136 +0,0 @@ -package detect - -import "strings" - -type CNAMEHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - NeedsNX bool -} - -type NSHit struct { - Matched bool - Host string - Provider string - ReasonCode ReasonCode - RegistrarMismatch bool -} - -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { - for _, fp := range fingerprints { - if fp.ContainsTarget(evidence.Target) { - hit := CNAMEHit{ - Matched: false, - Provider: fp.Name, - ReasonCode: "", - NeedsNX: fp.Nxdomain, - } - - if hit.NeedsNX { - hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN - if evidence.NoResolve { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN - } - } else { - hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP - mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) - if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch - } - } - - return &hit - } - } - return nil -} - -func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { - return nil - } - - lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) - var best *NSHit - - for _, host := range evidence.NSHosts { - for _, fp := range fingerprints { - if fp.ContainsNSHost(host) { - reasonCode := getNSHijackReasonCode(lameType, fp.Status) - hit := &NSHit{ - Matched: isNSMatch(reasonCode), - Host: host, - Provider: fp.Name, - ReasonCode: reasonCode, - } - - if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { - best = hit - } - } - } - } - - return best -} - -func ShouldEmitCNAME(hit *CNAMEHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func ShouldEmitNSHijack(hit *NSHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { - switch normalizeLameType(lameType) { - case "full": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSFullLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - case "partial": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSPartialLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - default: - return ReasonNSProviderMatchOnly - } -} - -func isNSMatch(reasonCode ReasonCode) bool { - return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable -} - -func isExploitableProviderStatus(status NSProviderStatus) bool { - return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase -} - -func normalizeLameType(lameType string) string { - return strings.ToLower(strings.TrimSpace(lameType)) -} - -func nsReasonRank(reasonCode ReasonCode) int { - switch reasonCode { - case ReasonNSFullLameProviderVulnerable: - return 4 - case ReasonNSPartialLameProviderVulnerable: - return 3 - case ReasonNSLameProviderUnknown: - return 2 - case ReasonNSProviderMatchOnly: - return 1 - default: - return 0 - } -} From dc360ef065a24ce47a03e86d1e42b7845f26ccd0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:06 -0300 Subject: [PATCH 33/34] add debug logging --- .../subdomain-takeover/cmd/service/main.go | 17 ++++-- .../subdomain-takeover/internal/app/runner.go | 14 ++++- .../subdomain-takeover/internal/app/worker.go | 14 ++++- .../internal/bootstrap/nats.go | 8 ++- .../internal/detect/classifier.go | 60 +++++++++++++++++-- .../internal/detect/cname_rules.go | 31 ++++++++++ .../internal/detect/fingerprint_loader.go | 12 +++- .../internal/detect/logger.go | 9 +++ .../internal/detect/ns_rules.go | 28 ++++++++- .../internal/messaging/publisher.go | 28 ++++++++- 10 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/logger.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 0f588ec30d..78a2500bff 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -24,8 +24,17 @@ func main() { } logger = bootstrap.NewLogger(cfg.LogLevel) - - if err := detect.LoadFingerprints(); err != nil { + logger.Info(). + Str("nats_url", cfg.NATSURL). + Str("stream", cfg.NATSStream). + Str("subject_in", cfg.SubjectIn). + Str("subject_out", cfg.SubjectOut). + Str("durable", cfg.DurableName). + Int("worker_count", cfg.WorkerCount). + Str("log_level", cfg.LogLevel.String()). + Msg("service configuration loaded") + + if err := detect.LoadFingerprints(logger); err != nil { logger.Fatal().Err(err).Msg("failed to load fingerprints") } @@ -36,7 +45,7 @@ func main() { signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sig) - runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg, logger) if err != nil { logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") } @@ -45,7 +54,7 @@ func main() { pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) - classifier := detect.NewClassifier(matcher) + classifier := detect.NewClassifier(matcher).WithLogger(logger) worker := app.NewWorker(logger, pub, classifier) go func() { diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 6d7ee52e8d..b8c0121f87 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -26,11 +26,14 @@ type MessageHandler interface { func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter + nextErrCount := 0 if deps.WorkerCount < 1 { deps.WorkerCount = 1 } + logger.Info().Int("worker_count", deps.WorkerCount).Msg("runner started") + var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) @@ -50,11 +53,18 @@ Loop: msg, err := iter.Next() if err != nil { if ctx.Err() != nil { + logger.Info().Msg("runner stopping: context canceled") break Loop } - logger.Debug().Err(err).Msg("next returned, continuing") + nextErrCount++ + if nextErrCount%10 == 0 { + logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") + } else { + logger.Debug().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("next returned, continuing") + } continue } + nextErrCount = 0 select { case sem <- struct{}{}: @@ -80,6 +90,6 @@ Loop: case <-waitDone: logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") + logger.Warn().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index e282fab952..eca59f91bc 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -30,7 +30,11 @@ func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier Sca } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { - log := w.logger.With().Str("component", "worker").Logger() + log := w.logger.With(). + Str("component", "worker"). + Str("subject", msg.Subject()). + Int("msg_size", len(msg.Data())). + Logger() scan, err := decodeScan(msg.Data()) if err != nil { @@ -39,6 +43,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log = log.With().Str("domain_key", scan.DomainKey).Logger() + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") @@ -46,6 +52,10 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + if len(findings) == 0 { + log.Debug().Msg("classification produced no findings") + } + for _, finding := range findings { err = w.publisher.Publish(ctx, finding) if err != nil { @@ -60,6 +70,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log.Debug().Int("findings_count", len(findings)).Msg("message acknowledged") + return nil } diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go index 51edb6b060..ea2fc49e21 100644 --- a/scanners/subdomain-takeover/internal/bootstrap/nats.go +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -7,6 +7,7 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) type RuntimeDeps struct { @@ -15,18 +16,21 @@ type RuntimeDeps struct { JS jetstream.JetStream } -func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { +func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logger) (*RuntimeDeps, error) { + logger.Debug().Str("nats_url", cfg.NATSURL).Msg("connecting to nats") nc, err := nats.Connect(cfg.NATSURL) if err != nil { return nil, err } + logger.Debug().Msg("creating jetstream client") js, err := jetstream.New(nc) if err != nil { nc.Close() return nil, err } + logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NATSStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, @@ -36,6 +40,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, @@ -45,6 +50,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Msg("creating pull message iterator") iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) if err != nil { nc.Close() diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index d3853749fd..ffd1d527c6 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,15 +2,17 @@ package detect import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" ) type Classifier struct { Matcher BodyFingerprintMatcher Source FingerprintSource + logger zerolog.Logger } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} } func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { @@ -18,7 +20,13 @@ func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintS source = GlobalFingerprintSource{} } - return &Classifier{Matcher: matcher, Source: source} + return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} +} + +func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { + c.logger = logger.With().Str("component", "classifier").Logger() + SetLogger(logger) + return c } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -32,10 +40,15 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { source = GlobalFingerprintSource{} } - return Classify(input, matcher, source) + logger := c.logger + if logger.GetLevel() == zerolog.NoLevel { + logger = zerolog.Nop() + } + + return Classify(input, matcher, source, logger) } -func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} cnameFingerprints := source.CNAME() @@ -43,8 +56,15 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { + logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("emitting cname finding") findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, DomainKey: input.DomainKey, @@ -55,12 +75,34 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr Confidence: ConfidenceForReason(cnameHit.ReasonCode), }) } + if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("cname finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") } nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Int("ns_hosts", len(nsEvidence.NSHosts)). + Msg("ns evidence extracted") nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("emitting ns finding") findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, DomainKey: input.DomainKey, @@ -70,7 +112,17 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr ReasonCode: string(nsHit.ReasonCode), Confidence: ConfidenceForReason(nsHit.ReasonCode), }) + } else if nsHit != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("ns finding suppressed") } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") } return findings, nil diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index e69c5cddb0..b693a4cd70 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -10,6 +10,13 @@ type CNAMEHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("target", evidence.Target). + Str("provider", fp.Name). + Bool("needs_nx", fp.Nxdomain). + Msg("cname provider target matched") + hit := CNAMEHit{ Matched: false, Provider: fp.Name, @@ -26,12 +33,36 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("fingerprint_mode", string(mode)). + Msg("checking cname body fingerprint") if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname body fingerprint matched") + } else { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: body fingerprint missing") } } + if hit.NeedsNX && !hit.Matched { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: required nxdomain signal missing") + } + return &hit } } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go index 694bf833aa..da2101e51a 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" "sync" + + "github.com/rs/zerolog" ) //go:embed data/*.json @@ -16,33 +18,41 @@ var ( loadFingerprintsErr error ) -func LoadFingerprints() error { +func LoadFingerprints(logger zerolog.Logger) error { + log := logger.With().Str("component", "fingerprint_loader").Logger() + loadFingerprintsOnce.Do(func() { var cname []CNAMEProviderFingerprint var ns []NSProviderFingerprint if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := validateCNAMEFingerprints(cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } if err := validateNSFingerprints(ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } CNAMEProviderFingerprints = cname NSProviderFingerprints = ns + + log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") }) return loadFingerprintsErr diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go new file mode 100644 index 0000000000..c3d4a30293 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger.go @@ -0,0 +1,9 @@ +package detect + +import "github.com/rs/zerolog" + +var detectLogger = zerolog.Nop() + +func SetLogger(logger zerolog.Logger) { + detectLogger = logger.With().Str("component", "detect").Logger() +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index ff993ef28b..5485e9cc95 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -5,11 +5,11 @@ type NSHit struct { Host string Provider string ReasonCode ReasonCode - RegistrarMismatch bool } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -20,6 +20,7 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr for _, fp := range fingerprints { if fp.ContainsNSHost(host) { reasonCode := getNSHijackReasonCode(lameType, fp.Status) + rank := nsReasonRank(reasonCode) hit := &NSHit{ Matched: isNSMatch(reasonCode), Host: host, @@ -27,6 +28,17 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr ReasonCode: reasonCode, } + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", host). + Str("provider", fp.Name). + Str("provider_status", string(fp.Status)). + Str("lame_type", lameType). + Str("reason_code", string(reasonCode)). + Int("rank", rank). + Bool("emittable", hit.Matched). + Msg("ns candidate evaluated") + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { best = hit } @@ -34,6 +46,20 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr } } + if best == nil { + detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + return nil + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", best.Host). + Str("provider", best.Provider). + Str("reason_code", string(best.ReasonCode)). + Int("rank", nsReasonRank(best.ReasonCode)). + Bool("emittable", best.Matched). + Msg("ns best candidate selected") + return best } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 5b5585a75f..5626f45983 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -22,15 +22,37 @@ func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { payload, err := json.Marshal(finding) if err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Msg("marshal finding failed") return err } if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("publish failed") return err } - p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + p.logger.Debug(). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("finding published") return nil } From 92c33569e8990ce92c682a405d4caca5c16b8393 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:11 -0300 Subject: [PATCH 34/34] update README.md --- scanners/subdomain-takeover/README.md | 54 ++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 640b246776..7e28388320 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,16 +5,27 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates passive CNAME takeover risk using provider fingerprints +2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` ## Current detection scope -- Implemented: CNAME-based passive detection -- Not implemented yet: NS-based (Sitting Duck) detection +Implemented today: -The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. +- CNAME passive detection +- NS passive detection (vulnerable-only emission policy) +- Deterministic NS candidate selection when multiple providers/hosts match +- Nil-safe evidence extraction for CNAME and NS +- Classifier refactor to support injected fingerprint sources (`FingerprintSource`) + +Current NS behavior: + +- NS provider/host matches are evaluated and ranked +- Findings are emitted only for exploitable NS outcomes: + - `NS_FULL_LAME_PROVIDER_VULNERABLE` + - `NS_PARTIAL_LAME_PROVIDER_VULNERABLE` +- Non-exploitable NS matches (`NS_LAME_PROVIDER_UNKNOWN`, `NS_PROVIDER_MATCH_ONLY`) are classified but not emitted ## Passive-only detection policy @@ -28,7 +39,7 @@ As a result, confidence is conservative and evidence-driven: ## Confidence model (current) -Current production behavior for CNAME findings: +### CNAME - `suspected` - provider target matched, but required passive signal is missing @@ -38,6 +49,28 @@ Current production behavior for CNAME findings: - `confirmed` - reserved; not currently emitted by CNAME rules +### NS + +- `confirmed` + - full lame delegation + exploitable provider status +- `probable` + - partial lame delegation + exploitable provider status +- `suspected` + - reason codes exist for weaker NS states, but these are currently not emitted + +## Internal package layout + +Detection logic was split into focused files to reduce coupling: + +- `internal/detect/cname_rules.go` +- `internal/detect/ns_rules.go` +- `internal/detect/ns_reasoning.go` +- `internal/detect/cname_evidence.go` +- `internal/detect/ns_evidence.go` +- `internal/detect/fingerprint_source.go` + +`Classifier` now supports dependency injection via `NewClassifierWithSource(...)`, while `NewClassifier(...)` defaults to global loaded fingerprints. + ## Fingerprint data Provider fingerprint data is stored in: @@ -88,7 +121,7 @@ The worker uses explicit JetStream ack semantics: Publish a test request: ```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"example.gc.ca","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_delegations":{"ns_hosts":["ns1.example-dns-provider.net"],"ns_checks":[],"ns_delegation":{"total_ns":1,"authoritative_ok":0,"lame_count":1,"lame_type":"full"},"error":""}}}' ``` Watch findings: @@ -96,3 +129,12 @@ Watch findings: ```bash nats sub "scans.findings.upsert" ``` + +## Next steps / nice-to-haves + +1. Add table-driven tests for NS reason mapping, ranking, and emission gating. +2. Add classifier integration tests using injected `FingerprintSource`. +3. Document and implement explicit policy for `edge_case` and `registration_closed` NS provider statuses. +4. Add structured debug logs for NS matching decisions (host, provider, status, reason). +5. Add registrar-context mismatch handling (`RegistrarMismatch`) and confidence policy. +6. Add metrics (counts by reason code, emitted vs suppressed) for production observability.