From 5b31974b3485ada63965578ba5930c7c0c694b35 Mon Sep 17 00:00:00 2001 From: jbergner Date: Fri, 13 Jun 2025 06:52:44 +0200 Subject: [PATCH] RC1 --- main.go | 359 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 328 insertions(+), 31 deletions(-) diff --git a/main.go b/main.go index 50c8c38..ffdb42e 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "encoding/json" "expvar" "fmt" + "io" "log" "math/big" "math/bits" @@ -72,46 +73,342 @@ func startMetricUpdater() { }() } +// +// +// + +type Source struct { + Category string + URL []string +} + +type Config struct { + RedisAddr string + Sources []Source + TTLHours int + IsWorker bool // true ⇒ lädt Blocklisten & schreibt sie nach Redis +} + +func loadConfig() Config { + // default Blocklist source + srcs := []Source{{ + Category: "generic", + URL: []string{ + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", + "https://raw.githubusercontent.com/bitwire-it/ipblocklist/refs/heads/main/ip-list.txt", + "", + }, + }, + } + + if env := os.Getenv("BLOCKLIST_SOURCES"); env != "" { + srcs = nil + for _, spec := range strings.Split(env, ",") { + spec = strings.TrimSpace(spec) + if spec == "" { + continue + } + parts := strings.SplitN(spec, ":", 2) + if len(parts) != 2 { + continue + } + cat := strings.TrimSpace(parts[0]) + raw := strings.FieldsFunc(parts[1], func(r rune) bool { return r == '|' || r == ';' }) + var urls []string + for _, u := range raw { + if u = strings.TrimSpace(u); u != "" { + urls = append(urls, u) + } + } + if len(urls) > 0 { + srcs = append(srcs, Source{Category: cat, URL: urls}) + } + } + } + + ttl := 24 + if env := os.Getenv("TTL_HOURS"); env != "" { + fmt.Sscanf(env, "%d", &ttl) + } + + isWorker := strings.ToLower(os.Getenv("ROLE")) == "worker" + + return Config{ + //RedisAddr: getenv("REDIS_ADDR", "redis:6379"), + RedisAddr: getenv("REDIS_ADDR", "10.10.5.249:6379"), + Sources: srcs, + TTLHours: ttl, + IsWorker: isWorker, + } +} + +// Alle gültigen ISO 3166-1 Alpha-2 Ländercodes (abgekürzt, reale Liste ist länger) +var allCountryCodes = []string{ + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", + "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", + "BT", "BW", "BY", "BZ", "CA", "CD", "CF", "CG", "CH", "CI", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", + "EE", "EG", "ER", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", + "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", + "ID", "IE", "IL", "IN", "IQ", "IR", "IS", "IT", "JM", "JO", "JP", "KE", "KG", + "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", + "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", + "ML", "MM", "MN", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", + "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", + "PL", "PT", "PW", "PY", "QA", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", + "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", "SY", "SZ", + "TD", "TG", "TH", "TJ", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TZ", "UA", + "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "YE", "ZA", "ZM", "ZW", +} + +// Hauptfunktion: gibt alle IPv4-Ranges eines Landes (CIDR) aus allen RIRs zurück +func GetIPRangesByCountry(countryCode string) ([]string, error) { + var allCIDRs []string + upperCode := strings.ToUpper(countryCode) + + for _, url := range rirFiles { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("fehler beim abrufen von %s: %w", url, err) + } + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "2") || strings.HasPrefix(line, "#") { + continue // Kommentar oder Header + } + if strings.Contains(line, "|"+upperCode+"|ipv4|") { + fields := strings.Split(line, "|") + if len(fields) < 5 { + continue + } + ipStart := fields[3] + count, _ := strconv.Atoi(fields[4]) + cidrs := summarizeCIDR(ipStart, count) + allCIDRs = append(allCIDRs, cidrs...) + } + } + } + return allCIDRs, nil +} + +// Hilfsfunktion: Start-IP + Anzahl → []CIDR +func summarizeCIDR(start string, count int) []string { + var cidrs []string + ip := net.ParseIP(start).To4() + startInt := ipToInt(ip) + + for count > 0 { + maxSize := 32 + for maxSize > 0 { + mask := 1 << uint(32-maxSize) + if startInt%uint32(mask) == 0 && mask <= count { + break + } + maxSize-- + } + cidr := fmt.Sprintf("%s/%d", intToIP(startInt), maxSize) + cidrs = append(cidrs, cidr) + count -= 1 << uint(32-maxSize) + startInt += uint32(1 << uint(32-maxSize)) + } + return cidrs +} + +func ipToInt(ip net.IP) uint32 { + return uint32(ip[0])<<24 + uint32(ip[1])<<16 + uint32(ip[2])<<8 + uint32(ip[3]) +} + +func intToIP(i uint32) net.IP { + return net.IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)) +} + +func keyBlock(cat string, p netip.Prefix) string { return "bl:" + cat + ":" + p.String() } + +func LoadAllCountryPrefixesIntoRedisAndRanger( + rdb *redis.Client, + ttlHours int, +) error { + for _, countryCode := range allCountryCodes { + + expiry := time.Duration(ttlHours) * time.Hour + results := make(map[string][]netip.Prefix) + + fmt.Printf("💡 Loading %s...\n", countryCode) + cidrs, err := GetIPRangesByCountry(countryCode) + if err != nil { + log.Printf("Error at %s: %v", countryCode, err) + } + fmt.Println("✅ Got " + strconv.Itoa(len(cidrs)) + " Ranges for Country " + countryCode) + var validPrefixes []netip.Prefix + for _, c := range cidrs { + prefix, err := netip.ParsePrefix(c) + if err != nil { + log.Printf("CIDR invalid [%s]: %v", c, err) + continue + } + validPrefixes = append(validPrefixes, prefix) + } + fmt.Println("✅ Got " + strconv.Itoa(len(validPrefixes)) + " valid Prefixes for Country " + countryCode) + + if len(validPrefixes) > 0 { + results[countryCode] = validPrefixes + } + + // Nach Verarbeitung: alles in Ranger + Redis eintragen + for code, prefixes := range results { + for _, p := range prefixes { + key := keyBlock(code, p) + if err := rdb.Set(ctx, key, "1", expiry).Err(); err != nil { + log.Printf("Redis-Error at %s: %v", key, err) + } + } + fmt.Println("✅ Import Subset " + strconv.Itoa(len(prefixes)) + " Entries") + } + fmt.Println("✅ Import done!") + fmt.Println("--------------------------------------------------") + } + + return nil +} + +func syncLoop(ctx context.Context, cfg Config, rdb *redis.Client) { + + fmt.Println("💡 Loading Lists...") + if err := syncOnce(ctx, cfg, rdb); err != nil { + log.Println("initial sync:", err) + } + fmt.Println("✅ Loading Lists Done.") + ticker := time.NewTicker(30 * time.Minute) + for { + select { + case <-ticker.C: + fmt.Println("💡 Loading Lists Timer...") + if err := syncOnce(ctx, cfg, rdb); err != nil { + log.Println("sync loop:", err) + } + fmt.Println("✅ Loading Lists Timer Done.") + case <-ctx.Done(): + ticker.Stop() + return + } + } +} + +func syncOnce(ctx context.Context, cfg Config, rdb *redis.Client) error { + expiry := time.Duration(cfg.TTLHours) * time.Hour + newBlocks := make(map[string]map[netip.Prefix]struct{}) + + for _, src := range cfg.Sources { + for _, url := range src.URL { + fmt.Println("💡 Loading List " + src.Category + " : " + url) + if err := fetchList(ctx, url, func(p netip.Prefix) { + if _, ok := newBlocks[src.Category]; !ok { + newBlocks[src.Category] = map[netip.Prefix]struct{}{} + } + newBlocks[src.Category][p] = struct{}{} + _ = rdb.Set(ctx, keyBlock(src.Category, p), "1", expiry).Err() + + }); err != nil { + fmt.Println("❌ Fail.") + return err + } + fmt.Println("✅ Done.") + } + } + return nil +} + +func fetchList(ctx context.Context, url string, cb func(netip.Prefix)) error { + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s -> %s", url, resp.Status) + } + return parseStream(resp.Body, cb) +} + +func parseStream(r io.Reader, cb func(netip.Prefix)) error { + s := bufio.NewScanner(r) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if p, err := netip.ParsePrefix(line); err == nil { + cb(p) + continue + } + if addr, err := netip.ParseAddr(line); err == nil { + plen := 32 + if addr.Is6() { + plen = 128 + } + cb(netip.PrefixFrom(addr, plen)) + } + } + return s.Err() +} + // -------------------------------------------- // INIT + MAIN // -------------------------------------------- func main() { - var err error - // Redis client - rdb = redis.NewClient(&redis.Options{Addr: redisAddr}) - if err := rdb.Ping(ctx).Err(); err != nil { - log.Fatalf("redis: %v", err) - } + if getenv("IMPORTER", "1") == "1" { + //Hier alles doof. selbe funktion wie unten. muss durch individuallisten ersetzt werden... + cfg := loadConfig() + rdb = redis.NewClient(&redis.Options{Addr: redisAddr}) + /*if err := LoadAllCountryPrefixesIntoRedisAndRanger(rdb, cfg.TTLHours); err != nil { + log.Fatalf("Fehler beim Laden aller Länderranges: %v", err) + }*/ + syncLoop(ctx, cfg, rdb) + log.Println("🚀 Import erfolgreich!") + } else { + var err error - // LRU cache - ipCache, err = lru.New[string, []string](cacheSize) - if err != nil { - log.Fatalf("cache init: %v", err) - } - - startMetricUpdater() - - // Admin load all blocklists (on demand or scheduled) - go func() { - if getenv("IMPORT_RIRS", "0") == "1" { - log.Println("Lade IP-Ranges aus RIRs...") - if err := importRIRDataToRedis(); err != nil { - log.Fatalf("import error: %v", err) - } - log.Println("✅ Import abgeschlossen.") + // Redis client + rdb = redis.NewClient(&redis.Options{Addr: redisAddr}) + if err := rdb.Ping(ctx).Err(); err != nil { + log.Fatalf("redis: %v", err) } - }() - // Routes - http.HandleFunc("/check/", handleCheck) - http.HandleFunc("/whitelist", handleWhitelist) - http.HandleFunc("/info", handleInfo) - http.Handle("/debug/vars", http.DefaultServeMux) + // LRU cache + ipCache, err = lru.New[string, []string](cacheSize) + if err != nil { + log.Fatalf("cache init: %v", err) + } + + startMetricUpdater() + + // Admin load all blocklists (on demand or scheduled) + go func() { + if getenv("IMPORT_RIRS", "0") == "1" { + log.Println("Lade IP-Ranges aus RIRs...") + if err := importRIRDataToRedis(); err != nil { + log.Fatalf("import error: %v", err) + } + log.Println("✅ Import abgeschlossen.") + } + }() + + // Routes + http.HandleFunc("/check/", handleCheck) + http.HandleFunc("/whitelist", handleWhitelist) + http.HandleFunc("/info", handleInfo) + http.Handle("/debug/vars", http.DefaultServeMux) + + log.Println("🚀 Server läuft auf :8080") + log.Fatal(http.ListenAndServe(":8080", nil)) + } - log.Println("🚀 Server läuft auf :8080") - log.Fatal(http.ListenAndServe(":8080", nil)) } func getenv(k, fallback string) string { @@ -238,7 +535,7 @@ func checkIP(ip netip.Addr, cats []string) ([]string, error) { func handleWhitelist(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { - http.Error(w, "method not allowed", 405) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } var body struct {