Das klappt, lässt aber den RAM explodieren.

main.go

@@ -8,10 +8,12 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/netip"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -39,7 +41,11 @@ func loadConfig() Config {
 	// default Blocklist source
 	srcs := []Source{{
 		Category: "generic",
-		URL:      []string{"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"},
+		URL: []string{
+			"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset",
+			"https://raw.githubusercontent.com/bitwire-it/ipblocklist/refs/heads/main/ip-list.txt",
+			"",
+		},
 	}}
 
 	if env := os.Getenv("BLOCKLIST_SOURCES"); env != "" {
@@ -75,7 +81,8 @@ func loadConfig() Config {
 	isWorker := strings.ToLower(os.Getenv("ROLE")) == "worker"
 
 	return Config{
-		RedisAddr: getenv("REDIS_ADDR", "redis:6379"),
+		//RedisAddr: getenv("REDIS_ADDR", "redis:6379"),
+		RedisAddr: getenv("REDIS_ADDR", "10.10.5.249:6379"),
 		Sources:   srcs,
 		TTLHours:  ttl,
 		IsWorker:  isWorker,
@@ -440,6 +447,235 @@ func writeJSON(w http.ResponseWriter, v any) {
 	_ = json.NewEncoder(w).Encode(v)
 }
 
+// -----------------------------------------------------------------------------
+// Class-Download from RIRs
+// -----------------------------------------------------------------------------
+
+// Liste der Delegated Files aller 5 RIRs
+var rirFiles = []string{
+	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-latest",
+	"https://ftp.apnic.net/stats/apnic/delegated-apnic-latest",
+	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+	"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest",
+	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest",
+}
+
+// Hauptfunktion: gibt alle IPv4-Ranges eines Landes (CIDR) aus allen RIRs zurück
+func GetIPRangesByCountry(countryCode string) ([]string, error) {
+	var allCIDRs []string
+	upperCode := strings.ToUpper(countryCode)
+
+	for _, url := range rirFiles {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("fehler beim abrufen von %s: %w", url, err)
+		}
+		defer resp.Body.Close()
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.HasPrefix(line, "2") || strings.HasPrefix(line, "#") {
+				continue // Kommentar oder Header
+			}
+			if strings.Contains(line, "|"+upperCode+"|ipv4|") {
+				fields := strings.Split(line, "|")
+				if len(fields) < 5 {
+					continue
+				}
+				ipStart := fields[3]
+				count, _ := strconv.Atoi(fields[4])
+				cidrs := summarizeCIDR(ipStart, count)
+				allCIDRs = append(allCIDRs, cidrs...)
+			}
+		}
+	}
+	return allCIDRs, nil
+}
+
+// Hilfsfunktion: Start-IP + Anzahl → []CIDR
+func summarizeCIDR(start string, count int) []string {
+	var cidrs []string
+	ip := net.ParseIP(start).To4()
+	startInt := ipToInt(ip)
+
+	for count > 0 {
+		maxSize := 32
+		for maxSize > 0 {
+			mask := 1 << uint(32-maxSize)
+			if startInt%uint32(mask) == 0 && mask <= count {
+				break
+			}
+			maxSize--
+		}
+		cidr := fmt.Sprintf("%s/%d", intToIP(startInt), maxSize)
+		cidrs = append(cidrs, cidr)
+		count -= 1 << uint(32-maxSize)
+		startInt += uint32(1 << uint(32-maxSize))
+	}
+	return cidrs
+}
+
+func ipToInt(ip net.IP) uint32 {
+	return uint32(ip[0])<<24 + uint32(ip[1])<<16 + uint32(ip[2])<<8 + uint32(ip[3])
+}
+
+func intToIP(i uint32) net.IP {
+	return net.IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
+}
+
+// Alle gültigen ISO 3166-1 Alpha-2 Ländercodes (abgekürzt, reale Liste ist länger)
+var allCountryCodes = []string{
+	"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AR", "AT", "AU", "AZ",
+	"BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS",
+	"BT", "BW", "BY", "BZ", "CA", "CD", "CF", "CG", "CH", "CI", "CL", "CM", "CN",
+	"CO", "CR", "CU", "CV", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC",
+	"EE", "EG", "ER", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE",
+	"GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU",
+	"ID", "IE", "IL", "IN", "IQ", "IR", "IS", "IT", "JM", "JO", "JP", "KE", "KG",
+	"KH", "KI", "KM", "KN", "KP", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK",
+	"LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK",
+	"ML", "MM", "MN", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE",
+	"NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK",
+	"PL", "PT", "PW", "PY", "QA", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD",
+	"SE", "SG", "SI", "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", "SY", "SZ",
+	"TD", "TG", "TH", "TJ", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TZ", "UA",
+	"UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "YE", "ZA", "ZM", "ZW",
+}
+
+// Aufruffunktion für alle Ländercodes
+func GetAllCountryIPRanges() (map[string][]string, error) {
+	allResults := make(map[string][]string)
+
+	for _, code := range allCountryCodes {
+		fmt.Printf("Verarbeite %s...\n", code)
+		cidrs, err := GetIPRangesByCountry(code)
+		if err != nil {
+			fmt.Printf("Fehler bei %s: %v\n", code, err)
+			continue
+		}
+		if len(cidrs) > 0 {
+			allResults[code] = cidrs
+		}
+	}
+
+	return allResults, nil
+}
+
+// Bessere und optimierte Routine
+
+func GetAllCountryPrefixes() (map[string][]netip.Prefix, error) {
+	var (
+		resultMu sync.Mutex
+		results  = make(map[string][]netip.Prefix)
+		wg       sync.WaitGroup
+		sem      = make(chan struct{}, 10) // max. 10 gleichzeitige Länder
+	)
+
+	for _, code := range allCountryCodes {
+		wg.Add(1)
+		sem <- struct{}{} // blockiert wenn mehr als 10 laufen
+
+		go func(countryCode string) {
+			defer wg.Done()
+			defer func() { <-sem }() // Slot freigeben
+
+			fmt.Printf("Verarbeite %s...\n", countryCode)
+			cidrs, err := GetIPRangesByCountry(countryCode)
+			if err != nil {
+				log.Printf("Fehler bei %s: %v", countryCode, err)
+				return
+			}
+
+			var validPrefixes []netip.Prefix
+			for _, c := range cidrs {
+				prefix, err := netip.ParsePrefix(c)
+				if err != nil {
+					log.Printf("CIDR-Fehler [%s]: %v", c, err)
+					continue
+				}
+				validPrefixes = append(validPrefixes, prefix)
+			}
+
+			if len(validPrefixes) > 0 {
+				resultMu.Lock()
+				results[countryCode] = validPrefixes
+				resultMu.Unlock()
+			}
+		}(code)
+	}
+
+	wg.Wait()
+	return results, nil
+}
+
+// Option 2
+
+func LoadAllCountryPrefixesIntoRedisAndRanger(
+	ctx context.Context,
+	rdb *redis.Client,
+	ranger *Ranger,
+	ttlHours int,
+) error {
+	var (
+		resultMu sync.Mutex
+		wg       sync.WaitGroup
+		sem      = make(chan struct{}, 10) // max. 10 gleichzeitige Downloads
+	)
+
+	expiry := time.Duration(ttlHours) * time.Hour
+	results := make(map[string][]netip.Prefix)
+
+	for _, code := range allCountryCodes {
+		wg.Add(1)
+		sem <- struct{}{} // Slot reservieren
+
+		go func(countryCode string) {
+			defer wg.Done()
+			defer func() { <-sem }() // Slot freigeben
+
+			fmt.Printf("Lade %s...\n", countryCode)
+			cidrs, err := GetIPRangesByCountry(countryCode)
+			if err != nil {
+				log.Printf("Fehler bei %s: %v", countryCode, err)
+				return
+			}
+
+			var validPrefixes []netip.Prefix
+			for _, c := range cidrs {
+				prefix, err := netip.ParsePrefix(c)
+				if err != nil {
+					log.Printf("CIDR ungültig [%s]: %v", c, err)
+					continue
+				}
+				validPrefixes = append(validPrefixes, prefix)
+			}
+
+			if len(validPrefixes) > 0 {
+				resultMu.Lock()
+				results[countryCode] = validPrefixes
+				resultMu.Unlock()
+			}
+		}(code)
+	}
+
+	wg.Wait()
+
+	// Nach Verarbeitung: alles in Ranger + Redis eintragen
+	for code, prefixes := range results {
+		for _, p := range prefixes {
+			ranger.addBlock(code, p)
+
+			key := keyBlock(code, p)
+			if err := rdb.Set(ctx, key, "1", expiry).Err(); err != nil {
+				log.Printf("Redis-Fehler bei %s: %v", key, err)
+			}
+		}
+	}
+
+	return nil
+}
+
 // -----------------------------------------------------------------------------
 // MAIN
 // -----------------------------------------------------------------------------
@@ -463,6 +699,35 @@ func main() {
 
 	subscribeKeyspace(ctx, rdb, ranger)
 
+	/*a, _ := GetAllCountryIPRanges()
+
+	for _, code := range allCountryCodes {
+		for _, b := range a[code] {
+			prefix, err := netip.ParsePrefix(b)
+			if err != nil {
+				log.Printf("ungültiger Prefix '%s': %v", b, err)
+				continue
+			}
+			ranger.addBlock(code, prefix)
+		}
+	}*/
+
+	/*allPrefixes, err := GetAllCountryPrefixes()
+	if err != nil {
+		log.Fatalf("Fehler beim Laden: %v", err)
+	}
+
+	// In den Ranger einfügen
+	for code, prefixes := range allPrefixes {
+		for _, p := range prefixes {
+			ranger.addBlock(code, p)
+		}
+	}*/
+
+	if err := LoadAllCountryPrefixesIntoRedisAndRanger(ctx, rdb, ranger, cfg.TTLHours); err != nil {
+		log.Fatalf("Fehler beim Laden aller Länderranges: %v", err)
+	}
+
 	if cfg.IsWorker {
 		go syncLoop(ctx, cfg, rdb, ranger)
 	}