From ebcf5f925e423bf64fda37533d15e55732de57b9 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Fri, 28 Jun 2024 13:55:12 -0300 Subject: [PATCH 01/13] fix: funcs don't need to be public --- botblocker.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/botblocker.go b/botblocker.go index d27d021..bb16823 100644 --- a/botblocker.go +++ b/botblocker.go @@ -37,13 +37,13 @@ type BotBlocker struct { Config } -func (b *BotBlocker) Update() error { +func (b *BotBlocker) update() error { startTime := time.Now() - err := b.UpdateIps() + err := b.updateIps() if err != nil { return fmt.Errorf("failed to update IP blocklists: %w", err) } - err = b.UpdateUserAgents() + err = b.updateUserAgents() if err != nil { return fmt.Errorf("failed to update IP blocklists: %w", err) } @@ -54,7 +54,7 @@ func (b *BotBlocker) Update() error { return nil } -func (b *BotBlocker) UpdateIps() error { +func (b *BotBlocker) updateIps() error { ipBlockList := make([]netip.Addr, 0) log.Info("Updating IP blocklist") @@ -84,7 +84,7 @@ func (b *BotBlocker) UpdateIps() error { return nil } -func (b *BotBlocker) UpdateUserAgents() error { +func (b *BotBlocker) updateUserAgents() error { userAgentBlockList := make([]string, 0) log.Info("Updating user agent blocklist") @@ -122,7 +122,7 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h next: next, Config: *config, } - err = blocker.Update() + err = blocker.update() if err != nil { return nil, fmt.Errorf("failed to update blocklists: %s", err) } @@ -131,7 +131,7 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if time.Now().Sub(b.lastUpdated) > time.Duration(time.Hour) { - err := b.Update() + err := b.update() if err != nil { log.Errorf("failed to update blocklist: %v", err) } From b8c42948813f04eea01d35be4c02b37409dd2065 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Fri, 28 Jun 2024 15:47:21 -0300 Subject: [PATCH 02/13] refactor for testing --- botblocker.go | 97 ++++++++++++++++-------- botblocker_test.go | 118 +++++++++++++++++++++++++++++ fixtures/lists/ip-blocklist | 1 + fixtures/lists/useragent-blocklist | 1 + 4 files changed, 187 insertions(+), 30 deletions(-) create mode 100644 botblocker_test.go diff --git a/botblocker.go b/botblocker.go index bb16823..113fb9b 100644 --- a/botblocker.go +++ b/botblocker.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "io" "net/http" "net/netip" @@ -67,16 +68,11 @@ func (b *BotBlocker) updateIps() error { return fmt.Errorf("failed fetch IP list: received a %v from %v", resp.Status, url) } - defer resp.Body.Close() - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - addrStr := scanner.Text() - addr, err := netip.ParseAddr(addrStr) - if err != nil { - return fmt.Errorf("failed to parse IP address: %w", err) - } - ipBlockList = append(ipBlockList, addr) + ips, err := readIps(resp.Body) + if err != nil { + return fmt.Errorf("failed to update IPs: %e", err) } + ipBlockList = append(ipBlockList, ips...) } b.ipBlocklist = ipBlockList @@ -84,6 +80,35 @@ func (b *BotBlocker) updateIps() error { return nil } +func readIps(ipReader io.ReadCloser) ([]netip.Addr, error) { + ips := make([]netip.Addr, 0) + defer ipReader.Close() + scanner := bufio.NewScanner(ipReader) + for scanner.Scan() { + addrStr := strings.TrimSpace(scanner.Text()) + addr, err := netip.ParseAddr(addrStr) + if err != nil { + return []netip.Addr{}, err + } + ips = append(ips, addr) + } + + return ips, nil +} + +func readUserAgents(userAgentReader io.ReadCloser) ([]string, error) { + userAgents := make([]string, 0) + + defer userAgentReader.Close() + scanner := bufio.NewScanner(userAgentReader) + for scanner.Scan() { + agent := strings.ToLower(strings.TrimSpace(scanner.Text())) + userAgents = append(userAgents, agent) + } + + return userAgents, nil +} + func (b *BotBlocker) updateUserAgents() error { userAgentBlockList := make([]string, 0) @@ -97,12 +122,11 @@ func (b *BotBlocker) updateUserAgents() error { return fmt.Errorf("failed fetch useragent list: received a %v from %v", resp.Status, url) } - defer resp.Body.Close() - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - agent := strings.ToLower(strings.TrimSpace(scanner.Text())) - userAgentBlockList = append(userAgentBlockList, agent) + agents, err := readUserAgents(resp.Body) + if err != nil { + return err } + userAgentBlockList = append(userAgentBlockList, agents...) } b.userAgentBlockList = userAgentBlockList @@ -144,27 +168,40 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { http.Error(rw, "internal error", http.StatusInternalServerError) return } - remoteAddr := remoteAddrPort.Addr() - - for _, badIP := range b.ipBlocklist { - if remoteAddr == badIP { - log.Infof("blocked request with from IP %v", remoteAddrPort.Addr()) - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) - http.Error(rw, "blocked", http.StatusForbidden) - return - } + if b.shouldBlockIp(remoteAddrPort.Addr()) { + log.Infof("blocked request with from IP %v", remoteAddrPort.Addr()) + log.Debugf("Checked request in %v", time.Now().Sub(startTime)) + http.Error(rw, "blocked", http.StatusForbidden) + return } agent := strings.ToLower(req.UserAgent()) - for _, badAgent := range b.userAgentBlockList { - if strings.Contains(agent, badAgent) { - log.Infof("blocked request with user agent %v because it contained %v", agent, badAgent) - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) - http.Error(rw, "blocked", http.StatusForbidden) - return - } + if b.shouldBlockAgent(agent) { + log.Infof("blocked request with user agent %v because it contained %v", agent, agent) + log.Debugf("Checked request in %v", time.Now().Sub(startTime)) + http.Error(rw, "blocked", http.StatusForbidden) + return } log.Debugf("Checked request in %v", time.Now().Sub(startTime)) b.next.ServeHTTP(rw, req) } + +func (b *BotBlocker) shouldBlockIp(addr netip.Addr) bool { + for _, badIp := range b.ipBlocklist { + if addr == badIp { + return true + } + } + return false +} + +func (b *BotBlocker) shouldBlockAgent(userAgent string) bool { + userAgent = strings.ToLower(strings.TrimSpace(userAgent)) + for _, badAgent := range b.userAgentBlockList { + if strings.Contains(userAgent, badAgent) { + return true + } + } + return false +} diff --git a/botblocker_test.go b/botblocker_test.go new file mode 100644 index 0000000..3dedb41 --- /dev/null +++ b/botblocker_test.go @@ -0,0 +1,118 @@ +package traefik_ultimate_bad_bot_blocker + +import ( + "net/netip" + "os" + "testing" +) + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func equalIps(a, b []netip.Addr) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestReadIps(t *testing.T) { + f, err := os.Open("fixtures/lists/ip-blocklist") + if err != nil { + t.Fatal("Failed to open testfile") + } + + expected := []netip.Addr{ + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + } + ips, err := readIps(f) + if !equalIps(ips, expected) || err != nil { + t.Fatalf("readIps(f) = %v, %e; want %v, ", ips, err, expected) + } +} + +func TestReadUserAgents(t *testing.T) { + f, err := os.Open("fixtures/lists/useragent-blocklist") + if err != nil { + t.Fatal("Failed to open testfile") + } + + expected := []string{"nintendobrowser", "claudebot"} + userAgents, err := readUserAgents(f) + if !equalStrings(userAgents, expected) || err != nil { + t.Fatalf("readUserAgents(f) = %v, %e; want %v, ", userAgents, err, expected) + } +} + +func TestShouldBlockIp(t *testing.T) { + botBlocker := BotBlocker{ + ipBlocklist: []netip.Addr{ + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + }, + } + badIp := netip.AddrFrom4([4]byte{10, 10, 10, 2}) + + blocked := botBlocker.shouldBlockIp(badIp) + if !blocked { + t.Fatalf("botBlocker.shouldBlockIp(%v) = %t; want true", badIp, blocked) + } +} + +func TestShouldAllowIp(t *testing.T) { + botBlocker := BotBlocker{ + ipBlocklist: []netip.Addr{ + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + }, + } + ip := netip.AddrFrom4([4]byte{10, 10, 10, 2}) + + blocked := botBlocker.shouldBlockIp(ip) + if !blocked { + t.Fatalf("botBlocker.shouldBlockIp(%v) = %t; want false", ip, blocked) + } +} + +func TestShouldBlockUserAgent(t *testing.T) { + botBlocker := BotBlocker{ + userAgentBlockList: []string{ + "nintendobrowser", + }, + } + badUserAgent := "Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US" + + blocked := botBlocker.shouldBlockAgent(badUserAgent) + if !blocked { + t.Fatalf("botBlocker.shouldBlockAgent(%s) = %t; want true", badUserAgent, blocked) + } +} + +func TestShouldAlowUserAgent(t *testing.T) { + botBlocker := BotBlocker{ + userAgentBlockList: []string{ + "nintendobrowser", + }, + } + userAgent := "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + + blocked := botBlocker.shouldBlockAgent(userAgent) + if blocked { + t.Fatalf("botBlocker.shouldBlockAgent(%s) = %t; want false", userAgent, blocked) + } +} diff --git a/fixtures/lists/ip-blocklist b/fixtures/lists/ip-blocklist index b5a79ba..ff4fe54 100644 --- a/fixtures/lists/ip-blocklist +++ b/fixtures/lists/ip-blocklist @@ -1 +1,2 @@ 10.10.10.2 + 192.168.1.1 diff --git a/fixtures/lists/useragent-blocklist b/fixtures/lists/useragent-blocklist index 21bb7c3..9fc402a 100644 --- a/fixtures/lists/useragent-blocklist +++ b/fixtures/lists/useragent-blocklist @@ -1 +1,2 @@ nintendobrowser +claudebot From 084726edbcc018f8d80642b100006889c49576f6 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Tue, 2 Jul 2024 16:22:20 -0300 Subject: [PATCH 03/13] feat: block cidrs --- README.md | 4 ++ botblocker.go | 60 ++++++++++++++++----------- botblocker_test.go | 82 +++++++++++++++++++++++++++++++------ docker-compose.yml | 15 +++++++ fixtures/lists/ip-blocklist | 3 +- 5 files changed, 125 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index b3329b3..759105a 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,7 @@ spec: ## Blocklist The blocklists should be acccessible via http/s and be a plain text list of IP address or useragents. + +## Testing + +Running `got test` will run a set of unit tests. Running `docker compose up` will start an end to end testing environment where `allowed-*` containers should be able to make requests, while `blocked-*` containers should fail. diff --git a/botblocker.go b/botblocker.go index 113fb9b..5c40205 100644 --- a/botblocker.go +++ b/botblocker.go @@ -32,7 +32,7 @@ func CreateConfig() *Config { type BotBlocker struct { next http.Handler name string - ipBlocklist []netip.Addr + prefixBlocklist []netip.Prefix userAgentBlockList []string lastUpdated time.Time Config @@ -42,58 +42,68 @@ func (b *BotBlocker) update() error { startTime := time.Now() err := b.updateIps() if err != nil { - return fmt.Errorf("failed to update IP blocklists: %w", err) + return fmt.Errorf("failed to update CIDR blocklists: %w", err) } err = b.updateUserAgents() if err != nil { - return fmt.Errorf("failed to update IP blocklists: %w", err) + return fmt.Errorf("failed to update user agent blocklists: %w", err) } b.lastUpdated = time.Now() duration := time.Now().Sub(startTime) - log.Info("Updated block lists. Blocked IPs: ", len(b.ipBlocklist), " Duration: ", duration) + log.Info("Updated block lists. Blocked CIDRs: ", len(b.prefixBlocklist), " Duration: ", duration) return nil } func (b *BotBlocker) updateIps() error { - ipBlockList := make([]netip.Addr, 0) + prefixBlockList := make([]netip.Prefix, 0) - log.Info("Updating IP blocklist") + log.Info("Updating CIDR blocklist") for _, url := range b.IpBlocklistUrls { resp, err := http.Get(url) if err != nil { - return fmt.Errorf("failed fetch IP list: %w", err) + return fmt.Errorf("failed fetch CIDR list: %w", err) } if resp.StatusCode > 299 { - return fmt.Errorf("failed fetch IP list: received a %v from %v", resp.Status, url) + return fmt.Errorf("failed to fetch CIDR list: received a %v from %v", resp.Status, url) } - ips, err := readIps(resp.Body) + prefixes, err := readPrefixes(resp.Body) if err != nil { - return fmt.Errorf("failed to update IPs: %e", err) + return fmt.Errorf("failed to update CIDRs: %e", err) } - ipBlockList = append(ipBlockList, ips...) + prefixBlockList = append(prefixBlockList, prefixes...) } - b.ipBlocklist = ipBlockList + b.prefixBlocklist = prefixBlockList return nil } -func readIps(ipReader io.ReadCloser) ([]netip.Addr, error) { - ips := make([]netip.Addr, 0) - defer ipReader.Close() - scanner := bufio.NewScanner(ipReader) +func readPrefixes(prefixReader io.ReadCloser) ([]netip.Prefix, error) { + prefixes := make([]netip.Prefix, 0) + defer prefixReader.Close() + scanner := bufio.NewScanner(prefixReader) for scanner.Scan() { - addrStr := strings.TrimSpace(scanner.Text()) - addr, err := netip.ParseAddr(addrStr) - if err != nil { - return []netip.Addr{}, err + entry := strings.TrimSpace(scanner.Text()) + var prefix netip.Prefix + if strings.Contains(entry, "/") { + var err error + prefix, err = netip.ParsePrefix(entry) + if err != nil { + return []netip.Prefix{}, err + } + } else { + addr, err := netip.ParseAddr(entry) + if err != nil { + return []netip.Prefix{}, err + } + prefix = netip.PrefixFrom(addr, 32) } - ips = append(ips, addr) + prefixes = append(prefixes, prefix) } - return ips, nil + return prefixes, nil } func readUserAgents(userAgentReader io.ReadCloser) ([]string, error) { @@ -161,7 +171,7 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } } startTime := time.Now() - log.Debugf("Checking request: IP: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent()) + log.Debugf("Checking request: CIDR: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent()) remoteAddrPort, err := netip.ParseAddrPort(req.RemoteAddr) if err != nil { @@ -188,8 +198,8 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func (b *BotBlocker) shouldBlockIp(addr netip.Addr) bool { - for _, badIp := range b.ipBlocklist { - if addr == badIp { + for _, badPrefix := range b.prefixBlocklist { + if badPrefix.Contains(addr) { return true } } diff --git a/botblocker_test.go b/botblocker_test.go index 3dedb41..1840d17 100644 --- a/botblocker_test.go +++ b/botblocker_test.go @@ -18,7 +18,7 @@ func equalStrings(a, b []string) bool { return true } -func equalIps(a, b []netip.Addr) bool { +func equalPrefixes(a, b []netip.Prefix) bool { if len(a) != len(b) { return false } @@ -36,13 +36,23 @@ func TestReadIps(t *testing.T) { t.Fatal("Failed to open testfile") } - expected := []netip.Addr{ - netip.AddrFrom4([4]byte{10, 10, 10, 2}), - netip.AddrFrom4([4]byte{192, 168, 1, 1}), + expected := []netip.Prefix{ + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + 32, + ), + netip.PrefixFrom( + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + 32, + ), + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 20, 0}), + 24, + ), } - ips, err := readIps(f) - if !equalIps(ips, expected) || err != nil { - t.Fatalf("readIps(f) = %v, %e; want %v, ", ips, err, expected) + prefixes, err := readPrefixes(f) + if !equalPrefixes(prefixes, expected) || err != nil { + t.Fatalf("readPrefixes(f) = %v, %e; want %v, ", prefixes, err, expected) } } @@ -61,9 +71,15 @@ func TestReadUserAgents(t *testing.T) { func TestShouldBlockIp(t *testing.T) { botBlocker := BotBlocker{ - ipBlocklist: []netip.Addr{ - netip.AddrFrom4([4]byte{10, 10, 10, 2}), - netip.AddrFrom4([4]byte{192, 168, 1, 1}), + prefixBlocklist: []netip.Prefix{ + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + 32, + ), + netip.PrefixFrom( + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + 32, + ), }, } badIp := netip.AddrFrom4([4]byte{10, 10, 10, 2}) @@ -76,9 +92,15 @@ func TestShouldBlockIp(t *testing.T) { func TestShouldAllowIp(t *testing.T) { botBlocker := BotBlocker{ - ipBlocklist: []netip.Addr{ - netip.AddrFrom4([4]byte{10, 10, 10, 2}), - netip.AddrFrom4([4]byte{192, 168, 1, 1}), + prefixBlocklist: []netip.Prefix{ + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 10, 2}), + 32, + ), + netip.PrefixFrom( + netip.AddrFrom4([4]byte{192, 168, 1, 1}), + 32, + ), }, } ip := netip.AddrFrom4([4]byte{10, 10, 10, 2}) @@ -89,6 +111,40 @@ func TestShouldAllowIp(t *testing.T) { } } +func TestShouldBlockIpCidr(t *testing.T) { + botBlocker := BotBlocker{ + prefixBlocklist: []netip.Prefix{ + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 10, 0}), + 24, + ), + }, + } + badIp := netip.AddrFrom4([4]byte{10, 10, 10, 2}) + + blocked := botBlocker.shouldBlockIp(badIp) + if !blocked { + t.Fatalf("botBlocker.shouldBlockIp(%v) = %t; want true", badIp, blocked) + } +} + +func TestShouldAllowIpCidr(t *testing.T) { + botBlocker := BotBlocker{ + prefixBlocklist: []netip.Prefix{ + netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 10, 10, 0}), + 24, + ), + }, + } + goodIp := netip.AddrFrom4([4]byte{10, 10, 20, 2}) + + blocked := botBlocker.shouldBlockIp(goodIp) + if blocked { + t.Fatalf("botBlocker.shouldBlockIp(%v) = %t; want false", goodIp, blocked) + } +} + func TestShouldBlockUserAgent(t *testing.T) { botBlocker := BotBlocker{ userAgentBlockList: []string{ diff --git a/docker-compose.yml b/docker-compose.yml index 058cec6..63fecda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,21 @@ services: networks: test: ipv4_address: 10.10.10.3 + blocked-cidr: + image: quay.io/curl/curl + external_links: + - traefik:whoami.example.com + entrypoint: + - /bin/sh + - -c + - | + while true; do + curl whoami.example.com 2>/dev/null + sleep 5 + done + networks: + test: + ipv4_address: 10.10.20.2 lists: image: nginx volumes: diff --git a/fixtures/lists/ip-blocklist b/fixtures/lists/ip-blocklist index ff4fe54..02c0502 100644 --- a/fixtures/lists/ip-blocklist +++ b/fixtures/lists/ip-blocklist @@ -1,2 +1,3 @@ 10.10.10.2 - 192.168.1.1 + 192.168.1.1/32 +10.10.20.0/24 From e8a37cd96476afe4c6e7294b51e8780f7eafd61f Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Tue, 2 Jul 2024 16:41:18 -0300 Subject: [PATCH 04/13] fix: user correct bitmask for ipv6 addresses --- botblocker.go | 11 ++++++++++- botblocker_test.go | 4 ++++ fixtures/lists/ip-blocklist | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/botblocker.go b/botblocker.go index 5c40205..66de24c 100644 --- a/botblocker.go +++ b/botblocker.go @@ -98,7 +98,16 @@ func readPrefixes(prefixReader io.ReadCloser) ([]netip.Prefix, error) { if err != nil { return []netip.Prefix{}, err } - prefix = netip.PrefixFrom(addr, 32) + var bits int + if addr.Is4() { + bits = 32 + } else { + bits = 128 + } + prefix, err = addr.Prefix(bits) + if err != nil { + return []netip.Prefix{}, err + } } prefixes = append(prefixes, prefix) } diff --git a/botblocker_test.go b/botblocker_test.go index 1840d17..fd3f23a 100644 --- a/botblocker_test.go +++ b/botblocker_test.go @@ -49,6 +49,10 @@ func TestReadIps(t *testing.T) { netip.AddrFrom4([4]byte{10, 10, 20, 0}), 24, ), + netip.PrefixFrom( + netip.AddrFrom16([16]byte{0x20, 0x01, 0xd, 0xb8, 0x33, 0x33, 0x44, 0x44, 0x55, 0x55, 0x66, 0x66, 0x77, 0x77, 0x88, 0x88}), + 128, + ), } prefixes, err := readPrefixes(f) if !equalPrefixes(prefixes, expected) || err != nil { diff --git a/fixtures/lists/ip-blocklist b/fixtures/lists/ip-blocklist index 02c0502..8387ecd 100644 --- a/fixtures/lists/ip-blocklist +++ b/fixtures/lists/ip-blocklist @@ -1,3 +1,4 @@ 10.10.10.2 192.168.1.1/32 10.10.20.0/24 +2001:db8:3333:4444:5555:6666:7777:8888 From a928a1d0d4d3a52e594ebd6ac9ea4796bd66cb30 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Tue, 2 Jul 2024 16:46:04 -0300 Subject: [PATCH 05/13] refactor: dry up timing logging --- botblocker.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/botblocker.go b/botblocker.go index 66de24c..e083985 100644 --- a/botblocker.go +++ b/botblocker.go @@ -181,6 +181,10 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } startTime := time.Now() log.Debugf("Checking request: CIDR: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent()) + timer := func() { + log.Debugf("Checked request in %v", time.Now().Sub(startTime)) + } + defer timer() remoteAddrPort, err := netip.ParseAddrPort(req.RemoteAddr) if err != nil { @@ -189,7 +193,6 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } if b.shouldBlockIp(remoteAddrPort.Addr()) { log.Infof("blocked request with from IP %v", remoteAddrPort.Addr()) - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) http.Error(rw, "blocked", http.StatusForbidden) return } @@ -197,12 +200,10 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { agent := strings.ToLower(req.UserAgent()) if b.shouldBlockAgent(agent) { log.Infof("blocked request with user agent %v because it contained %v", agent, agent) - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) http.Error(rw, "blocked", http.StatusForbidden) return } - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) b.next.ServeHTTP(rw, req) } From 09f2239116a24fc44e64063b6d6f9978bc4f1b37 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Tue, 2 Jul 2024 16:54:08 -0300 Subject: [PATCH 06/13] feat: add ci --- .github/workflows/ci.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..02d02b8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,18 @@ +name: Test +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: [ '1.19', '1.20', '1.21.x' ] + + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - name: Run tests + run: go test -v . From 2df7d701f2b0c4c40e7f6bfd798f3f092d6f2103 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Tue, 2 Jul 2024 16:57:56 -0300 Subject: [PATCH 07/13] fix: use older version of go --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9ef6d42..921b207 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/discoverygarden/traefik-ultimate-bad-bot-blocker -go 1.21.4 +go 1.19 From 44b54166652686fbba463f2bca4f786531aafe6b Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Wed, 3 Jul 2024 14:31:16 -0300 Subject: [PATCH 08/13] feat: add yamllint and fix yaml --- .github/workflows/ci.yaml | 12 ++++++-- .yamllint.yaml | 6 ++++ docker-compose.yml | 64 +++++++++++++++++++-------------------- 3 files changed, 48 insertions(+), 34 deletions(-) create mode 100644 .yamllint.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02d02b8..7e0124a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,10 @@ -name: Test +name: CI +# yamllint thinks the `on` key is being turned into `true` +# yamllint disable-line rule:truthy on: [push] jobs: - build: + test: runs-on: ubuntu-latest strategy: matrix: @@ -16,3 +18,9 @@ jobs: go-version: ${{ matrix.go-version }} - name: Run tests run: go test -v . + lint-yaml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Lint YAML + run: yamllint . diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..3576647 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,6 @@ +extends: default +rules: + document-start: disable + line-length: disable + brackets: + max-spaces-inside: 1 diff --git a/docker-compose.yml b/docker-compose.yml index 63fecda..7793fc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ name: traefik-bad-bot-blocker services: traefik: image: traefik:3.0 - command: + command: - "--providers.docker=true" - "--providers.docker.exposedbydefault=false" - "--entryPoints.web.address=:80" @@ -33,14 +33,14 @@ services: image: quay.io/curl/curl external_links: - traefik:whoami.example.com - entrypoint: - - /bin/sh - - -c - - | - while true; do - curl whoami.example.com 2>/dev/null | grep X-Real-Ip - sleep 5 - done + entrypoint: + - /bin/sh + - -c + - | + while true; do + curl whoami.example.com 2>/dev/null | grep X-Real-Ip + sleep 5 + done networks: test: ipv4_address: 10.10.10.1 @@ -48,14 +48,14 @@ services: image: quay.io/curl/curl external_links: - traefik:whoami.example.com - entrypoint: - - /bin/sh - - -c - - | - while true; do - curl whoami.example.com 2>/dev/null - sleep 5 - done + entrypoint: + - /bin/sh + - -c + - | + while true; do + curl whoami.example.com 2>/dev/null + sleep 5 + done networks: test: ipv4_address: 10.10.10.2 @@ -63,14 +63,14 @@ services: image: quay.io/curl/curl external_links: - traefik:whoami.example.com - entrypoint: - - /bin/sh - - -c - - | - while true; do - curl -A 'Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US' whoami.example.com 2>/dev/null - sleep 5 - done + entrypoint: + - /bin/sh + - -c + - | + while true; do + curl -A 'Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US' whoami.example.com 2>/dev/null + sleep 5 + done networks: test: ipv4_address: 10.10.10.3 @@ -79,13 +79,13 @@ services: external_links: - traefik:whoami.example.com entrypoint: - - /bin/sh - - -c - - | - while true; do - curl whoami.example.com 2>/dev/null - sleep 5 - done + - /bin/sh + - -c + - | + while true; do + curl whoami.example.com 2>/dev/null + sleep 5 + done networks: test: ipv4_address: 10.10.20.2 From 91ef6e53455c1c7590389cd5bc116b9a53d941ad Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Wed, 3 Jul 2024 14:34:24 -0300 Subject: [PATCH 09/13] feat: add semver job --- .github/workflows/semver.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/semver.yaml diff --git a/.github/workflows/semver.yaml b/.github/workflows/semver.yaml new file mode 100644 index 0000000..c37bd3d --- /dev/null +++ b/.github/workflows/semver.yaml @@ -0,0 +1,18 @@ +name: Auto Semver +on: + pull_request: + types: + - closed + branches: + - main +jobs: + update: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Run Auto Semver + uses: discoverygarden/auto-semver@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} From 03b76563090b7e441ffcb78227b99326e1dfcd92 Mon Sep 17 00:00:00 2001 From: Alexander Cairns Date: Wed, 3 Jul 2024 14:36:09 -0300 Subject: [PATCH 10/13] fix: satisfy yamllint --- .github/workflows/semver.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semver.yaml b/.github/workflows/semver.yaml index c37bd3d..f2af463 100644 --- a/.github/workflows/semver.yaml +++ b/.github/workflows/semver.yaml @@ -1,7 +1,9 @@ name: Auto Semver +# yamllint thinks the `on` key is being turned into `true` +# yamllint disable-line rule:truthy on: pull_request: - types: + types: - closed branches: - main From ec1e1eb29d82c1c1a853d5b3408dd2885cbd6af6 Mon Sep 17 00:00:00 2001 From: Adam Bowman <50798875+adamcbowman@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:04:31 -0300 Subject: [PATCH 11/13] update checkout action version --- .github/workflows/semver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semver.yaml b/.github/workflows/semver.yaml index f2af463..909b4a5 100644 --- a/.github/workflows/semver.yaml +++ b/.github/workflows/semver.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Auto Semver uses: discoverygarden/auto-semver@v1 with: From 07b8661a03fe658debd091855bf4966aefeba49c Mon Sep 17 00:00:00 2001 From: Adam Bowman Date: Thu, 4 Jul 2024 11:17:45 -0300 Subject: [PATCH 12/13] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 759105a..84ef85d 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,4 @@ The blocklists should be acccessible via http/s and be a plain text list of IP a ## Testing -Running `got test` will run a set of unit tests. Running `docker compose up` will start an end to end testing environment where `allowed-*` containers should be able to make requests, while `blocked-*` containers should fail. +Running `go test` will run a set of unit tests. Running `docker compose up` will start an end to end testing environment where `allowed-*` containers should be able to make requests, while `blocked-*` containers should fail. From 78c997f8a10fbd6e2911b8a1e011ab98f33fd278 Mon Sep 17 00:00:00 2001 From: Adam Bowman Date: Thu, 4 Jul 2024 11:41:21 -0300 Subject: [PATCH 13/13] use time.Since function --- botblocker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/botblocker.go b/botblocker.go index e083985..1d695a4 100644 --- a/botblocker.go +++ b/botblocker.go @@ -50,7 +50,7 @@ func (b *BotBlocker) update() error { } b.lastUpdated = time.Now() - duration := time.Now().Sub(startTime) + duration := time.Since(startTime) log.Info("Updated block lists. Blocked CIDRs: ", len(b.prefixBlocklist), " Duration: ", duration) return nil } @@ -173,7 +173,7 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h } func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if time.Now().Sub(b.lastUpdated) > time.Duration(time.Hour) { + if time.Since(b.lastUpdated) > time.Hour { err := b.update() if err != nil { log.Errorf("failed to update blocklist: %v", err) @@ -182,7 +182,7 @@ func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { startTime := time.Now() log.Debugf("Checking request: CIDR: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent()) timer := func() { - log.Debugf("Checked request in %v", time.Now().Sub(startTime)) + log.Debugf("Checked request in %v", time.Since(startTime)) } defer timer()