delay on import

main.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -33,11 +34,14 @@ func getenv(key, def string) string {
 	return def
 }
 
+const defaultDelay = 2 * time.Second // default gap between two downloads
+
 var (
 	listenAddr     = getenv("LISTEN_ADDR", ":8080")
 	storeKind      = getenv("STORE", "memory") // "memory" or "redis"
 	importBaseURL  = getenv("FLOD_IMPORT_URL", "https://git.send.nrw/sendnrw/flod-lists/raw/branch/main/")
 	importInterval = mustParseDuration(getenv("IMPORT_INTERVAL", "30m"))
+	delayStr       = os.Getenv("DELAY")
 	enableHoneypot = strings.EqualFold(getenv("HONEYPOT", "off"), "on")
 	honeyTCP       = splitCSV(getenv("HONEY_TCP", "135,139,445,389,636,3268,3269,88,3389"))
 	honeyUDP       = splitCSV(getenv("HONEY_UDP", "135,137,138,389,3389,88"))
@@ -729,6 +733,19 @@ func importCategory(st Store, cat, url string) (int, error) {
 }
 
 func startImporter(st Store, cats map[string]string, srv *server) {
+
+	var delay time.Duration
+	if delayStr == "" {
+		delay = defaultDelay
+	} else {
+		secs, err := strconv.Atoi(delayStr)
+		if err != nil || secs <= 0 {
+			log.Printf("invalid DELAY=%q, using default (%v)", delayStr, defaultDelay)
+		} else {
+			delay = time.Duration(secs) * time.Second
+		}
+	}
+
 	go func() {
 		for {
 			start := time.Now()
@@ -750,6 +767,7 @@ func startImporter(st Store, cats map[string]string, srv *server) {
 					}
 					log.Printf("✅ [%s] %d entries", cat, n)
 				}()
+				time.Sleep(delay)
 			}
 			wg.Wait()
 			srv.rebuildIndex()