From 176fe99ac0c1a48ead2a7ee5e5c499aa3c91982c Mon Sep 17 00:00:00 2001 From: jbergner Date: Fri, 29 Aug 2025 23:27:53 +0200 Subject: [PATCH] update mit logging --- compose.yml | 42 ++++++++---------- main.go | 122 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 118 insertions(+), 46 deletions(-) diff --git a/compose.yml b/compose.yml index a78e9dd..7a290ff 100644 --- a/compose.yml +++ b/compose.yml @@ -1,20 +1,21 @@ +version: "3.9" services: updater: - image: git.send.nrw/sendnrw/go-ubuntu-mirror:latest # unverändert vom vorherigen Setup container_name: ubuntu-mirror-updater restart: unless-stopped volumes: - - mirror-data:/var/spool/apt-mirror - + - /docker/ubuntu-apt-mirror/data:/var/spool/apt-mirror + networks: + - traefik-net + image: compose-updater web: - build: - context: . - dockerfile: Dockerfile # <- die Alpine-Datei oben + image: git.send.nrw/sendnrw/go-ubuntu-mirror:latest container_name: ubuntu-mirror-web restart: unless-stopped - depends_on: [updater] + depends_on: + - updater volumes: - - mirror-data:/data:ro + - /docker/ubuntu-apt-mirror/data:/data command: > -archive=/data/mirror/archive.ubuntu.com/ubuntu -security=/data/mirror/security.ubuntu.com/ubuntu @@ -22,31 +23,24 @@ services: -autoindex=true -cache=600 -addr=:8080 + -trust-proxy=true + -log-json=false # auf true setzen für JSON-Lines labels: - traefik.enable=true - # HTTPS Router - traefik.http.routers.ubuntu_mirror.rule=Host(`ubuntu-24-04.send.nrw`) - traefik.http.routers.ubuntu_mirror.entrypoints=websecure - traefik.http.routers.ubuntu_mirror.tls=true - traefik.http.routers.ubuntu_mirror.tls.certresolver=letsencrypt - traefik.http.routers.ubuntu_mirror.service=ubuntu_mirror_svc - # HTTP -> HTTPS Redirect + - traefik.http.services.ubuntu_mirror_svc.loadbalancer.server.port=8080 + - traefik.http.services.ubuntu_mirror_svc.loadbalancer.server.scheme=http - traefik.http.routers.ubuntu_mirror_http.rule=Host(`ubuntu-24-04.send.nrw`) - traefik.http.routers.ubuntu_mirror_http.entrypoints=web - - traefik.http.routers.ubuntu_mirror_http.middlewares=to-https - traefik.http.routers.ubuntu_mirror_http.service=ubuntu_mirror_svc - traefik.http.middlewares.to-https.redirectscheme.scheme=https - traefik.http.middlewares.to-https.redirectscheme.permanent=true - # Service-Ziel (Go-Server auf 8080) - - traefik.http.services.ubuntu_mirror_svc.loadbalancer.server.port=8080 - - traefik.http.services.ubuntu_mirror_svc.loadbalancer.server.scheme=http - # Optional: externes Traefik-Netz - # - traefik.docker.network=traefik_proxy - # networks: - # - traefik_proxy - -volumes: - mirror-data: -# networks: -# traefik_proxy: -# external: true + networks: + - traefik-net +networks: + traefik-net: + external: true \ No newline at end of file diff --git a/main.go b/main.go index 1b604d0..d58c53b 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,13 @@ package main import ( + "encoding/json" "errors" "flag" "fmt" "log" "mime" + "net" "net/http" "os" "path" @@ -14,7 +16,8 @@ import ( "time" ) -// unionFS tries multiple http.FileSystem roots in order. +/* ---------- Union-FS wie zuvor ---------- */ + type unionFS struct{ roots []http.FileSystem } func (u unionFS) Open(name string) (http.File, error) { @@ -27,7 +30,8 @@ func (u unionFS) Open(name string) (http.File, error) { return nil, os.ErrNotExist } -// fileHandler serves files from an http.FileSystem with nice directory listing +/* ---------- File-Handler wie zuvor ---------- */ + type fileHandler struct { fs http.FileSystem autoIndex bool @@ -35,14 +39,12 @@ type fileHandler struct { } func (h fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Normalize path upath := r.URL.Path if !strings.HasPrefix(upath, "/") { upath = "/" + upath } - upath = path.Clean(upath) // prevents path traversal + upath = path.Clean(upath) - // Open f, err := h.fs.Open(upath) if err != nil { http.NotFound(w, r) @@ -56,14 +58,11 @@ func (h fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Directories if fi.IsDir() { - // redirect to slash-terminated path (as net/http does) if !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently) return } - // If index.html exists, serve it indexPath := path.Join(upath, "index.html") if ff, err := h.fs.Open(indexPath); err == nil { defer ff.Close() @@ -71,7 +70,6 @@ func (h fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.serveFile(w, r, indexPath, ff, info) return } - // Otherwise: simple autoindex (for debugging in browser) if h.autoIndex { h.serveDirList(w, r, f) return @@ -80,25 +78,18 @@ func (h fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Files h.serveFile(w, r, upath, f, fi) } func (h fileHandler) serveFile(w http.ResponseWriter, r *http.Request, name string, f http.File, fi os.FileInfo) { - // Content-Type by extension (fallback: octet-stream) ctype := mime.TypeByExtension(strings.ToLower(filepath.Ext(name))) if ctype == "" { ctype = "application/octet-stream" } w.Header().Set("Content-Type", ctype) - - // Conservative cache (APT macht eigene Validierungen über InRelease/Release) if h.cacheMaxAge > 0 { w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(h.cacheMaxAge.Seconds()))) } - - // ServeContent enables Range requests + sets Last-Modified - // Use name without leading slash to avoid special-case in ServeContent http.ServeContent(w, r, strings.TrimPrefix(name, "/"), fi.ModTime(), f) } @@ -128,15 +119,102 @@ func (h fileHandler) serveDirList(w http.ResponseWriter, r *http.Request, d http fmt.Fprint(w, "") } +/* ---------- Logging-Middleware ---------- */ + +type logResponseWriter struct { + http.ResponseWriter + status int + bytes int64 +} + +func (lw *logResponseWriter) WriteHeader(code int) { + lw.status = code + lw.ResponseWriter.WriteHeader(code) +} +func (lw *logResponseWriter) Write(b []byte) (int, error) { + n, err := lw.ResponseWriter.Write(b) + lw.bytes += int64(n) + return n, err +} + +func clientIP(r *http.Request, trustProxy bool) string { + if trustProxy { + // X-Forwarded-For: "client, proxy1, proxy2" + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + parts := strings.Split(xff, ",") + return strings.TrimSpace(parts[0]) + } + if xr := r.Header.Get("X-Real-IP"); xr != "" { + return xr + } + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +type accessRecord struct { + TS string `json:"ts"` + Method string `json:"method"` + Path string `json:"path"` + Status int `json:"status"` + Bytes int64 `json:"bytes"` + Duration string `json:"duration"` + IP string `json:"ip"` + UA string `json:"ua"` +} + +func loggingMiddleware(next http.Handler, trustProxy bool, logJSON bool) http.Handler { + // Für Text-Logs Datum+Uhrzeit (UTC) im Prefix + if logJSON { + log.SetFlags(0) + } else { + log.SetFlags(log.Ldate | log.Ltime) + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + lw := &logResponseWriter{ResponseWriter: w, status: 200} + + next.ServeHTTP(lw, r) + + rec := accessRecord{ + TS: time.Now().UTC().Format(time.RFC3339Nano), + Method: r.Method, + Path: r.URL.RequestURI(), + Status: lw.status, + Bytes: lw.bytes, + Duration: time.Since(start).String(), + IP: clientIP(r, trustProxy), + UA: r.UserAgent(), + } + + if logJSON { + if b, err := json.Marshal(rec); err == nil { + log.Print(string(b)) + } else { + log.Printf("log marshal error: %v", err) + } + } else { + log.Printf(`%s %s status=%d bytes=%d dur=%s ip=%s ua="%s"`, + rec.Method, rec.Path, rec.Status, rec.Bytes, rec.Duration, rec.IP, rec.UA) + } + }) +} + +/* ---------- main() mit Flags ---------- */ + func main() { var ( - addr = flag.String("addr", ":8080", "listen address") - // Point these to the *ubuntu* directories inside your mirror volume + addr = flag.String("addr", ":8080", "listen address") archiveRoot = flag.String("archive", "/data/mirror/archive.ubuntu.com/ubuntu", "archive ubuntu root") securityRoot = flag.String("security", "/data/mirror/security.ubuntu.com/ubuntu", "security ubuntu root") oldReleases = flag.String("old", "/data/mirror/old-releases.ubuntu.com/ubuntu", "old-releases ubuntu root") autoIndex = flag.Bool("autoindex", true, "enable directory listings") cacheSeconds = flag.Int("cache", 600, "Cache-Control max-age seconds (0 to disable)") + trustProxy = flag.Bool("trust-proxy", true, "trust X-Forwarded-For / X-Real-IP") + logJSON = flag.Bool("log-json", false, "log in JSON lines") ) flag.Parse() @@ -168,19 +246,19 @@ func main() { } mux := http.NewServeMux() - // Serve under /ubuntu/... to match client sources.list mux.Handle("/ubuntu/", http.StripPrefix("/ubuntu", handler)) - // health endpoint mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }) + wrapped := loggingMiddleware(mux, *trustProxy, *logJSON) + srv := &http.Server{ Addr: *addr, - Handler: mux, + Handler: wrapped, ReadTimeout: 30 * time.Second, - WriteTimeout: 10 * time.Minute, // big files + WriteTimeout: 10 * time.Minute, IdleTimeout: 120 * time.Second, ReadHeaderTimeout: 10 * time.Second, MaxHeaderBytes: 1 << 20,