diff --git a/compose.yml b/compose.yml index 4850f4b..0e4bb10 100644 --- a/compose.yml +++ b/compose.yml @@ -22,6 +22,7 @@ services: - flod_nw depends_on: - redis + - importer environment: # Redis-Adresse schon per Docker-Netzwerk korrekt: REDIS_ADDR: redis:6379 @@ -31,6 +32,13 @@ services: #- "8080:8080" # : restart: unless-stopped + importer: + image: git.send.nrw/sendnrw/flod-ipv64-parser:latest + container_name: ipblock-importer + networks: + - flod_nw + restart: unless-stopped + redis: image: redis:7-alpine container_name: ipblock-redis diff --git a/main.go b/main.go index 0a4b0a2..00b308c 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,13 @@ import ( "context" "encoding/json" "fmt" + "io" + "log" "net" "net/http" "net/netip" + "os" + "path/filepath" "strings" "sync" "time" @@ -23,6 +27,35 @@ var rdb = redis.NewClient(&redis.Options{ Addr: "redis:6379", }) +// ────────────────────────────────────────────────────────────────────────────── +// Helpers +// ────────────────────────────────────────────────────────────────────────────── +// ExportListJSON schreibt die Map als prettified JSON‑Datei. +func ExportListJSON(path string, m map[string]string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + return enc.Encode(m) +} + +// ImportListJSON liest eine JSON‑Datei und gibt map[string]string zurück. +func ImportListJSON(path string) (map[string]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var m map[string]string + if err := json.NewDecoder(f).Decode(&m); err != nil { + return nil, err + } + return m, nil +} + // URLs der Blocklisten var blocklistURLs = map[string]string{ "bitwire": "https://raw.githubusercontent.com/bitwire-it/ipblocklist/refs/heads/main/ip-list.txt", @@ -634,8 +667,51 @@ func updateBlocklistMetrics() { } } +type target struct { + Name, URL string +} + +func fetchAndSave(client *http.Client, t target, outDir string) error { + fileName := filepath.Base(t.URL) + if fileName == "" { + fileName = strings.ReplaceAll(strings.ToLower(strings.ReplaceAll(t.Name, " ", "_")), "..", "") + } + dst := filepath.Join(outDir, fileName) + + log.Printf("Downloading %-40s → %s", t.Name, dst) + resp, err := client.Get(t.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad HTTP status: %s", resp.Status) + } + + tmp := dst + ".part" + f, err := os.Create(tmp) + if err != nil { + return err + } + if _, err := io.Copy(f, resp.Body); err != nil { + f.Close() + os.Remove(tmp) + return err + } + f.Close() + return os.Rename(tmp, dst) +} + // Import-Logik func importBlocklists() error { + client := &http.Client{Timeout: 60 * time.Second} + t := target{Name: "Catalog", URL: "http://importer:8080/lists.json"} + if err := fetchAndSave(client, t, "./lists/"); err != nil { + log.Printf("ERROR %s → %v", t.URL, err) + } + blocklistURLs, _ = ImportListJSON("./lists/") + var wg sync.WaitGroup errCh := make(chan error, len(blocklistURLs))