diff --git a/main.go b/main.go index c2ae719..aafcd13 100644 --- a/main.go +++ b/main.go @@ -8,10 +8,12 @@ import ( "fmt" "io" "log" + "net" "net/http" "net/netip" "os" "sort" + "strconv" "strings" "sync" "time" @@ -39,7 +41,11 @@ func loadConfig() Config { // default Blocklist source srcs := []Source{{ Category: "generic", - URL: []string{"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"}, + URL: []string{ + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", + "https://raw.githubusercontent.com/bitwire-it/ipblocklist/refs/heads/main/ip-list.txt", + "", + }, }} if env := os.Getenv("BLOCKLIST_SOURCES"); env != "" { @@ -75,7 +81,8 @@ func loadConfig() Config { isWorker := strings.ToLower(os.Getenv("ROLE")) == "worker" return Config{ - RedisAddr: getenv("REDIS_ADDR", "redis:6379"), + //RedisAddr: getenv("REDIS_ADDR", "redis:6379"), + RedisAddr: getenv("REDIS_ADDR", "10.10.5.249:6379"), Sources: srcs, TTLHours: ttl, IsWorker: isWorker, @@ -440,6 +447,235 @@ func writeJSON(w http.ResponseWriter, v any) { _ = json.NewEncoder(w).Encode(v) } +// ----------------------------------------------------------------------------- +// Class-Download from RIRs +// ----------------------------------------------------------------------------- + +// Liste der Delegated Files aller 5 RIRs +var rirFiles = []string{ + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-latest", + "https://ftp.apnic.net/stats/apnic/delegated-apnic-latest", + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest", + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest", +} + +// Hauptfunktion: gibt alle IPv4-Ranges eines Landes (CIDR) aus allen RIRs zurück +func GetIPRangesByCountry(countryCode string) ([]string, error) { + var allCIDRs []string + upperCode := strings.ToUpper(countryCode) + + for _, url := range rirFiles { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("fehler beim abrufen von %s: %w", url, err) + } + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "2") || strings.HasPrefix(line, "#") { + continue // Kommentar oder Header + } + if strings.Contains(line, "|"+upperCode+"|ipv4|") { + fields := strings.Split(line, "|") + if len(fields) < 5 { + continue + } + ipStart := fields[3] + count, _ := strconv.Atoi(fields[4]) + cidrs := summarizeCIDR(ipStart, count) + allCIDRs = append(allCIDRs, cidrs...) + } + } + } + return allCIDRs, nil +} + +// Hilfsfunktion: Start-IP + Anzahl → []CIDR +func summarizeCIDR(start string, count int) []string { + var cidrs []string + ip := net.ParseIP(start).To4() + startInt := ipToInt(ip) + + for count > 0 { + maxSize := 32 + for maxSize > 0 { + mask := 1 << uint(32-maxSize) + if startInt%uint32(mask) == 0 && mask <= count { + break + } + maxSize-- + } + cidr := fmt.Sprintf("%s/%d", intToIP(startInt), maxSize) + cidrs = append(cidrs, cidr) + count -= 1 << uint(32-maxSize) + startInt += uint32(1 << uint(32-maxSize)) + } + return cidrs +} + +func ipToInt(ip net.IP) uint32 { + return uint32(ip[0])<<24 + uint32(ip[1])<<16 + uint32(ip[2])<<8 + uint32(ip[3]) +} + +func intToIP(i uint32) net.IP { + return net.IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)) +} + +// Alle gültigen ISO 3166-1 Alpha-2 Ländercodes (abgekürzt, reale Liste ist länger) +var allCountryCodes = []string{ + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", + "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", + "BT", "BW", "BY", "BZ", "CA", "CD", "CF", "CG", "CH", "CI", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", + "EE", "EG", "ER", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", + "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", + "ID", "IE", "IL", "IN", "IQ", "IR", "IS", "IT", "JM", "JO", "JP", "KE", "KG", + "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", + "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", + "ML", "MM", "MN", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", + "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", + "PL", "PT", "PW", "PY", "QA", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", + "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", "SY", "SZ", + "TD", "TG", "TH", "TJ", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TZ", "UA", + "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "YE", "ZA", "ZM", "ZW", +} + +// Aufruffunktion für alle Ländercodes +func GetAllCountryIPRanges() (map[string][]string, error) { + allResults := make(map[string][]string) + + for _, code := range allCountryCodes { + fmt.Printf("Verarbeite %s...\n", code) + cidrs, err := GetIPRangesByCountry(code) + if err != nil { + fmt.Printf("Fehler bei %s: %v\n", code, err) + continue + } + if len(cidrs) > 0 { + allResults[code] = cidrs + } + } + + return allResults, nil +} + +// Bessere und optimierte Routine + +func GetAllCountryPrefixes() (map[string][]netip.Prefix, error) { + var ( + resultMu sync.Mutex + results = make(map[string][]netip.Prefix) + wg sync.WaitGroup + sem = make(chan struct{}, 10) // max. 10 gleichzeitige Länder + ) + + for _, code := range allCountryCodes { + wg.Add(1) + sem <- struct{}{} // blockiert wenn mehr als 10 laufen + + go func(countryCode string) { + defer wg.Done() + defer func() { <-sem }() // Slot freigeben + + fmt.Printf("Verarbeite %s...\n", countryCode) + cidrs, err := GetIPRangesByCountry(countryCode) + if err != nil { + log.Printf("Fehler bei %s: %v", countryCode, err) + return + } + + var validPrefixes []netip.Prefix + for _, c := range cidrs { + prefix, err := netip.ParsePrefix(c) + if err != nil { + log.Printf("CIDR-Fehler [%s]: %v", c, err) + continue + } + validPrefixes = append(validPrefixes, prefix) + } + + if len(validPrefixes) > 0 { + resultMu.Lock() + results[countryCode] = validPrefixes + resultMu.Unlock() + } + }(code) + } + + wg.Wait() + return results, nil +} + +// Option 2 + +func LoadAllCountryPrefixesIntoRedisAndRanger( + ctx context.Context, + rdb *redis.Client, + ranger *Ranger, + ttlHours int, +) error { + var ( + resultMu sync.Mutex + wg sync.WaitGroup + sem = make(chan struct{}, 10) // max. 10 gleichzeitige Downloads + ) + + expiry := time.Duration(ttlHours) * time.Hour + results := make(map[string][]netip.Prefix) + + for _, code := range allCountryCodes { + wg.Add(1) + sem <- struct{}{} // Slot reservieren + + go func(countryCode string) { + defer wg.Done() + defer func() { <-sem }() // Slot freigeben + + fmt.Printf("Lade %s...\n", countryCode) + cidrs, err := GetIPRangesByCountry(countryCode) + if err != nil { + log.Printf("Fehler bei %s: %v", countryCode, err) + return + } + + var validPrefixes []netip.Prefix + for _, c := range cidrs { + prefix, err := netip.ParsePrefix(c) + if err != nil { + log.Printf("CIDR ungültig [%s]: %v", c, err) + continue + } + validPrefixes = append(validPrefixes, prefix) + } + + if len(validPrefixes) > 0 { + resultMu.Lock() + results[countryCode] = validPrefixes + resultMu.Unlock() + } + }(code) + } + + wg.Wait() + + // Nach Verarbeitung: alles in Ranger + Redis eintragen + for code, prefixes := range results { + for _, p := range prefixes { + ranger.addBlock(code, p) + + key := keyBlock(code, p) + if err := rdb.Set(ctx, key, "1", expiry).Err(); err != nil { + log.Printf("Redis-Fehler bei %s: %v", key, err) + } + } + } + + return nil +} + // ----------------------------------------------------------------------------- // MAIN // ----------------------------------------------------------------------------- @@ -463,6 +699,35 @@ func main() { subscribeKeyspace(ctx, rdb, ranger) + /*a, _ := GetAllCountryIPRanges() + + for _, code := range allCountryCodes { + for _, b := range a[code] { + prefix, err := netip.ParsePrefix(b) + if err != nil { + log.Printf("ungültiger Prefix '%s': %v", b, err) + continue + } + ranger.addBlock(code, prefix) + } + }*/ + + /*allPrefixes, err := GetAllCountryPrefixes() + if err != nil { + log.Fatalf("Fehler beim Laden: %v", err) + } + + // In den Ranger einfügen + for code, prefixes := range allPrefixes { + for _, p := range prefixes { + ranger.addBlock(code, p) + } + }*/ + + if err := LoadAllCountryPrefixesIntoRedisAndRanger(ctx, rdb, ranger, cfg.TTLHours); err != nil { + log.Fatalf("Fehler beim Laden aller Länderranges: %v", err) + } + if cfg.IsWorker { go syncLoop(ctx, cfg, rdb, ranger) }