RC1

2025-06-13 06:52:44 +02:00
parent bbccee0754
commit 5b31974b34
1 changed files with 328 additions and 31 deletions
--- a/main.go
+++ b/main.go
@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"expvar"
 	"fmt"
+	"io"
 	"log"
 	"math/big"
 	"math/bits"
@@ -72,46 +73,342 @@ func startMetricUpdater() {
 	}()
 }

+//
+//
+//
+
+type Source struct {
+	Category string
+	URL      []string
+}
+
+type Config struct {
+	RedisAddr string
+	Sources   []Source
+	TTLHours  int
+	IsWorker  bool // true ⇒ lädt Blocklisten & schreibt sie nach Redis
+}
+
+func loadConfig() Config {
+	// default Blocklist source
+	srcs := []Source{{
+		Category: "generic",
+		URL: []string{
+			"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset",
+			"https://raw.githubusercontent.com/bitwire-it/ipblocklist/refs/heads/main/ip-list.txt",
+			"",
+		},
+	},
+	}
+
+	if env := os.Getenv("BLOCKLIST_SOURCES"); env != "" {
+		srcs = nil
+		for _, spec := range strings.Split(env, ",") {
+			spec = strings.TrimSpace(spec)
+			if spec == "" {
+				continue
+			}
+			parts := strings.SplitN(spec, ":", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			cat := strings.TrimSpace(parts[0])
+			raw := strings.FieldsFunc(parts[1], func(r rune) bool { return r == '|' || r == ';' })
+			var urls []string
+			for _, u := range raw {
+				if u = strings.TrimSpace(u); u != "" {
+					urls = append(urls, u)
+				}
+			}
+			if len(urls) > 0 {
+				srcs = append(srcs, Source{Category: cat, URL: urls})
+			}
+		}
+	}
+
+	ttl := 24
+	if env := os.Getenv("TTL_HOURS"); env != "" {
+		fmt.Sscanf(env, "%d", &ttl)
+	}
+
+	isWorker := strings.ToLower(os.Getenv("ROLE")) == "worker"
+
+	return Config{
+		//RedisAddr: getenv("REDIS_ADDR", "redis:6379"),
+		RedisAddr: getenv("REDIS_ADDR", "10.10.5.249:6379"),
+		Sources:   srcs,
+		TTLHours:  ttl,
+		IsWorker:  isWorker,
+	}
+}
+
+// Alle gültigen ISO 3166-1 Alpha-2 Ländercodes (abgekürzt, reale Liste ist länger)
+var allCountryCodes = []string{
+	"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AR", "AT", "AU", "AZ",
+	"BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS",
+	"BT", "BW", "BY", "BZ", "CA", "CD", "CF", "CG", "CH", "CI", "CL", "CM", "CN",
+	"CO", "CR", "CU", "CV", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC",
+	"EE", "EG", "ER", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE",
+	"GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU",
+	"ID", "IE", "IL", "IN", "IQ", "IR", "IS", "IT", "JM", "JO", "JP", "KE", "KG",
+	"KH", "KI", "KM", "KN", "KP", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK",
+	"LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK",
+	"ML", "MM", "MN", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE",
+	"NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK",
+	"PL", "PT", "PW", "PY", "QA", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD",
+	"SE", "SG", "SI", "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", "SY", "SZ",
+	"TD", "TG", "TH", "TJ", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TZ", "UA",
+	"UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "YE", "ZA", "ZM", "ZW",
+}
+
+// Hauptfunktion: gibt alle IPv4-Ranges eines Landes (CIDR) aus allen RIRs zurück
+func GetIPRangesByCountry(countryCode string) ([]string, error) {
+	var allCIDRs []string
+	upperCode := strings.ToUpper(countryCode)
+
+	for _, url := range rirFiles {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("fehler beim abrufen von %s: %w", url, err)
+		}
+		defer resp.Body.Close()
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.HasPrefix(line, "2") || strings.HasPrefix(line, "#") {
+				continue // Kommentar oder Header
+			}
+			if strings.Contains(line, "|"+upperCode+"|ipv4|") {
+				fields := strings.Split(line, "|")
+				if len(fields) < 5 {
+					continue
+				}
+				ipStart := fields[3]
+				count, _ := strconv.Atoi(fields[4])
+				cidrs := summarizeCIDR(ipStart, count)
+				allCIDRs = append(allCIDRs, cidrs...)
+			}
+		}
+	}
+	return allCIDRs, nil
+}
+
+// Hilfsfunktion: Start-IP + Anzahl → []CIDR
+func summarizeCIDR(start string, count int) []string {
+	var cidrs []string
+	ip := net.ParseIP(start).To4()
+	startInt := ipToInt(ip)
+
+	for count > 0 {
+		maxSize := 32
+		for maxSize > 0 {
+			mask := 1 << uint(32-maxSize)
+			if startInt%uint32(mask) == 0 && mask <= count {
+				break
+			}
+			maxSize--
+		}
+		cidr := fmt.Sprintf("%s/%d", intToIP(startInt), maxSize)
+		cidrs = append(cidrs, cidr)
+		count -= 1 << uint(32-maxSize)
+		startInt += uint32(1 << uint(32-maxSize))
+	}
+	return cidrs
+}
+
+func ipToInt(ip net.IP) uint32 {
+	return uint32(ip[0])<<24 + uint32(ip[1])<<16 + uint32(ip[2])<<8 + uint32(ip[3])
+}
+
+func intToIP(i uint32) net.IP {
+	return net.IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
+}
+
+func keyBlock(cat string, p netip.Prefix) string { return "bl:" + cat + ":" + p.String() }
+
+func LoadAllCountryPrefixesIntoRedisAndRanger(
+	rdb *redis.Client,
+	ttlHours int,
+) error {
+	for _, countryCode := range allCountryCodes {
+
+		expiry := time.Duration(ttlHours) * time.Hour
+		results := make(map[string][]netip.Prefix)
+
+		fmt.Printf("💡 Loading %s...\n", countryCode)
+		cidrs, err := GetIPRangesByCountry(countryCode)
+		if err != nil {
+			log.Printf("Error at %s: %v", countryCode, err)
+		}
+		fmt.Println("✅ Got " + strconv.Itoa(len(cidrs)) + " Ranges for Country " + countryCode)
+		var validPrefixes []netip.Prefix
+		for _, c := range cidrs {
+			prefix, err := netip.ParsePrefix(c)
+			if err != nil {
+				log.Printf("CIDR invalid [%s]: %v", c, err)
+				continue
+			}
+			validPrefixes = append(validPrefixes, prefix)
+		}
+		fmt.Println("✅ Got " + strconv.Itoa(len(validPrefixes)) + " valid Prefixes for Country " + countryCode)
+
+		if len(validPrefixes) > 0 {
+			results[countryCode] = validPrefixes
+		}
+
+		// Nach Verarbeitung: alles in Ranger + Redis eintragen
+		for code, prefixes := range results {
+			for _, p := range prefixes {
+				key := keyBlock(code, p)
+				if err := rdb.Set(ctx, key, "1", expiry).Err(); err != nil {
+					log.Printf("Redis-Error at %s: %v", key, err)
+				}
+			}
+			fmt.Println("✅ Import Subset " + strconv.Itoa(len(prefixes)) + " Entries")
+		}
+		fmt.Println("✅ Import done!")
+		fmt.Println("--------------------------------------------------")
+	}
+
+	return nil
+}
+
+func syncLoop(ctx context.Context, cfg Config, rdb *redis.Client) {
+
+	fmt.Println("💡 Loading Lists...")
+	if err := syncOnce(ctx, cfg, rdb); err != nil {
+		log.Println("initial sync:", err)
+	}
+	fmt.Println("✅ Loading Lists Done.")
+	ticker := time.NewTicker(30 * time.Minute)
+	for {
+		select {
+		case <-ticker.C:
+			fmt.Println("💡 Loading Lists Timer...")
+			if err := syncOnce(ctx, cfg, rdb); err != nil {
+				log.Println("sync loop:", err)
+			}
+			fmt.Println("✅ Loading Lists Timer Done.")
+		case <-ctx.Done():
+			ticker.Stop()
+			return
+		}
+	}
+}
+
+func syncOnce(ctx context.Context, cfg Config, rdb *redis.Client) error {
+	expiry := time.Duration(cfg.TTLHours) * time.Hour
+	newBlocks := make(map[string]map[netip.Prefix]struct{})
+
+	for _, src := range cfg.Sources {
+		for _, url := range src.URL {
+			fmt.Println("💡 Loading List " + src.Category + " : " + url)
+			if err := fetchList(ctx, url, func(p netip.Prefix) {
+				if _, ok := newBlocks[src.Category]; !ok {
+					newBlocks[src.Category] = map[netip.Prefix]struct{}{}
+				}
+				newBlocks[src.Category][p] = struct{}{}
+				_ = rdb.Set(ctx, keyBlock(src.Category, p), "1", expiry).Err()
+
+			}); err != nil {
+				fmt.Println("❌ Fail.")
+				return err
+			}
+			fmt.Println("✅ Done.")
+		}
+	}
+	return nil
+}
+
+func fetchList(ctx context.Context, url string, cb func(netip.Prefix)) error {
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s -> %s", url, resp.Status)
+	}
+	return parseStream(resp.Body, cb)
+}
+
+func parseStream(r io.Reader, cb func(netip.Prefix)) error {
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		line := strings.TrimSpace(s.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		if p, err := netip.ParsePrefix(line); err == nil {
+			cb(p)
+			continue
+		}
+		if addr, err := netip.ParseAddr(line); err == nil {
+			plen := 32
+			if addr.Is6() {
+				plen = 128
+			}
+			cb(netip.PrefixFrom(addr, plen))
+		}
+	}
+	return s.Err()
+}
+
 // --------------------------------------------
 // INIT + MAIN
 // --------------------------------------------

 func main() {
-	var err error

-	// Redis client
-	rdb = redis.NewClient(&redis.Options{Addr: redisAddr})
-	if err := rdb.Ping(ctx).Err(); err != nil {
-		log.Fatalf("redis: %v", err)
-	}
+	if getenv("IMPORTER", "1") == "1" {
+		//Hier alles doof. selbe funktion wie unten. muss durch individuallisten ersetzt werden...
+		cfg := loadConfig()
+		rdb = redis.NewClient(&redis.Options{Addr: redisAddr})
+		/*if err := LoadAllCountryPrefixesIntoRedisAndRanger(rdb, cfg.TTLHours); err != nil {
+			log.Fatalf("Fehler beim Laden aller Länderranges: %v", err)
+		}*/
+		syncLoop(ctx, cfg, rdb)
+		log.Println("🚀 Import erfolgreich!")
+	} else {
+		var err error

-	// LRU cache
-	ipCache, err = lru.New[string, []string](cacheSize)
-	if err != nil {
-		log.Fatalf("cache init: %v", err)
-	}
-
-	startMetricUpdater()
-
-	// Admin load all blocklists (on demand or scheduled)
-	go func() {
-		if getenv("IMPORT_RIRS", "0") == "1" {
-			log.Println("Lade IP-Ranges aus RIRs...")
-			if err := importRIRDataToRedis(); err != nil {
-				log.Fatalf("import error: %v", err)
-			}
-			log.Println("✅ Import abgeschlossen.")
+		// Redis client
+		rdb = redis.NewClient(&redis.Options{Addr: redisAddr})
+		if err := rdb.Ping(ctx).Err(); err != nil {
+			log.Fatalf("redis: %v", err)
 		}
-	}()

-	// Routes
-	http.HandleFunc("/check/", handleCheck)
-	http.HandleFunc("/whitelist", handleWhitelist)
-	http.HandleFunc("/info", handleInfo)
-	http.Handle("/debug/vars", http.DefaultServeMux)
+		// LRU cache
+		ipCache, err = lru.New[string, []string](cacheSize)
+		if err != nil {
+			log.Fatalf("cache init: %v", err)
+		}
+
+		startMetricUpdater()
+
+		// Admin load all blocklists (on demand or scheduled)
+		go func() {
+			if getenv("IMPORT_RIRS", "0") == "1" {
+				log.Println("Lade IP-Ranges aus RIRs...")
+				if err := importRIRDataToRedis(); err != nil {
+					log.Fatalf("import error: %v", err)
+				}
+				log.Println("✅ Import abgeschlossen.")
+			}
+		}()
+
+		// Routes
+		http.HandleFunc("/check/", handleCheck)
+		http.HandleFunc("/whitelist", handleWhitelist)
+		http.HandleFunc("/info", handleInfo)
+		http.Handle("/debug/vars", http.DefaultServeMux)
+
+		log.Println("🚀 Server läuft auf :8080")
+		log.Fatal(http.ListenAndServe(":8080", nil))
+	}

-	log.Println("🚀 Server läuft auf :8080")
-	log.Fatal(http.ListenAndServe(":8080", nil))
 }

 func getenv(k, fallback string) string {
@@ -238,7 +535,7 @@ func checkIP(ip netip.Addr, cats []string) ([]string, error) {

 func handleWhitelist(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodPost {
-		http.Error(w, "method not allowed", 405)
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
 	var body struct {