280 lines
8.4 KiB
Go
280 lines
8.4 KiB
Go
package bulk
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"html/template"
|
|
"io"
|
|
"io/fs"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
bulkweb "github.com/b1tsblog/ai-disclosure-standard/bulkweb"
|
|
)
|
|
|
|
const Version = "1.0.2"
|
|
|
|
type Server struct {
|
|
cfg Config
|
|
logger *slog.Logger
|
|
templates *template.Template
|
|
client *http.Client
|
|
mux *http.ServeMux
|
|
}
|
|
|
|
type pageData struct {
|
|
Name string
|
|
Version string
|
|
DisclosureBaseURL string
|
|
GeneratorURL string
|
|
MaxURLs int
|
|
}
|
|
|
|
type publicConfig struct {
|
|
DisclosureBaseURL string `json:"disclosureBaseURL"`
|
|
GeneratorURL string `json:"generatorURL"`
|
|
MaxURLs int `json:"maxURLs"`
|
|
Version string `json:"version"`
|
|
}
|
|
|
|
type batchRequest struct {
|
|
Template string `json:"template"`
|
|
Subjects []string `json:"subjects"`
|
|
}
|
|
|
|
type batchResult struct {
|
|
Subject string `json:"subject"`
|
|
OK bool `json:"ok"`
|
|
Data json.RawMessage `json:"data,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type batchResponse struct {
|
|
Results []batchResult `json:"results"`
|
|
}
|
|
|
|
func New(cfg Config, logger *slog.Logger) (http.Handler, error) {
|
|
if err := ValidateConfig(cfg); err != nil {
|
|
return nil, err
|
|
}
|
|
tmpl, err := template.New("root").ParseFS(bulkweb.Files, "templates/*.html")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse bulk templates: %w", err)
|
|
}
|
|
s := &Server{
|
|
cfg: cfg,
|
|
logger: logger,
|
|
templates: tmpl,
|
|
client: &http.Client{
|
|
Timeout: cfg.RequestTimeout,
|
|
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
},
|
|
},
|
|
mux: http.NewServeMux(),
|
|
}
|
|
s.routes()
|
|
return s.securityHeaders(s.mux), nil
|
|
}
|
|
|
|
func (s *Server) routes() {
|
|
staticFS, _ := fs.Sub(bulkweb.Files, "static")
|
|
s.mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS))))
|
|
s.mux.HandleFunc("GET /", s.handleIndex)
|
|
s.mux.HandleFunc("GET /api/config", s.handleConfig)
|
|
s.mux.HandleFunc("POST /api/render-batch", s.handleRenderBatch)
|
|
s.mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = io.WriteString(w, "ok\n")
|
|
})
|
|
}
|
|
|
|
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
_ = s.templates.ExecuteTemplate(w, "index.html", pageData{
|
|
Name: s.cfg.PublicName, Version: Version, DisclosureBaseURL: s.cfg.DisclosureBaseURL,
|
|
GeneratorURL: s.cfg.GeneratorURL, MaxURLs: s.cfg.MaxURLs,
|
|
})
|
|
}
|
|
|
|
func (s *Server) handleConfig(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
_ = json.NewEncoder(w).Encode(publicConfig{
|
|
DisclosureBaseURL: s.cfg.DisclosureBaseURL,
|
|
GeneratorURL: s.cfg.GeneratorURL,
|
|
MaxURLs: s.cfg.MaxURLs,
|
|
Version: Version,
|
|
})
|
|
}
|
|
|
|
func (s *Server) handleRenderBatch(w http.ResponseWriter, r *http.Request) {
|
|
body := http.MaxBytesReader(w, r.Body, 1<<20)
|
|
defer body.Close()
|
|
dec := json.NewDecoder(body)
|
|
dec.DisallowUnknownFields()
|
|
var request batchRequest
|
|
if err := dec.Decode(&request); err != nil {
|
|
s.problem(w, http.StatusBadRequest, "invalid_json", "Request body must be valid JSON.")
|
|
return
|
|
}
|
|
if err := ensureEOF(dec); err != nil {
|
|
s.problem(w, http.StatusBadRequest, "invalid_json", "Request body must contain exactly one JSON object.")
|
|
return
|
|
}
|
|
if len(request.Subjects) == 0 {
|
|
s.problem(w, http.StatusBadRequest, "missing_subjects", "At least one subject URL is required.")
|
|
return
|
|
}
|
|
if len(request.Subjects) > s.cfg.MaxURLs {
|
|
s.problem(w, http.StatusRequestEntityTooLarge, "too_many_subjects", fmt.Sprintf("At most %d subject URLs are allowed per batch.", s.cfg.MaxURLs))
|
|
return
|
|
}
|
|
if len(request.Template) > 24<<10 {
|
|
s.problem(w, http.StatusRequestEntityTooLarge, "template_too_large", "Template query is too large.")
|
|
return
|
|
}
|
|
|
|
templateValues, err := url.ParseQuery(strings.TrimPrefix(strings.TrimSpace(request.Template), "?"))
|
|
if err != nil {
|
|
s.problem(w, http.StatusBadRequest, "invalid_template", "Template must be a valid URL query string.")
|
|
return
|
|
}
|
|
templateValues.Del("subject")
|
|
|
|
results := make([]batchResult, len(request.Subjects))
|
|
jobs := make(chan int)
|
|
var wg sync.WaitGroup
|
|
workers := min(s.cfg.Workers, len(request.Subjects))
|
|
for range workers {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for idx := range jobs {
|
|
subject := strings.TrimSpace(request.Subjects[idx])
|
|
results[idx] = s.renderOne(r.Context(), templateValues, subject)
|
|
}
|
|
}()
|
|
}
|
|
for idx := range request.Subjects {
|
|
jobs <- idx
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
_ = json.NewEncoder(w).Encode(batchResponse{Results: results})
|
|
}
|
|
|
|
func (s *Server) renderOne(ctx context.Context, templateValues url.Values, subject string) batchResult {
|
|
if err := validateSubject(subject); err != nil {
|
|
return batchResult{Subject: subject, Error: err.Error()}
|
|
}
|
|
values := cloneValues(templateValues)
|
|
values.Set("subject", subject)
|
|
endpoint := s.cfg.CoreInternalURL + "/v1/render?" + values.Encode()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
if err != nil {
|
|
return batchResult{Subject: subject, Error: "could not build core request"}
|
|
}
|
|
req.Header.Set("Accept", "application/json")
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
s.logger.Warn("core render failed", "error", err)
|
|
return batchResult{Subject: subject, Error: "disclosure core is unavailable"}
|
|
}
|
|
defer resp.Body.Close()
|
|
payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
|
if err != nil {
|
|
return batchResult{Subject: subject, Error: "could not read disclosure core response"}
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
message := coreErrorMessage(payload)
|
|
if message == "" {
|
|
message = fmt.Sprintf("disclosure core returned HTTP %d", resp.StatusCode)
|
|
}
|
|
return batchResult{Subject: subject, Error: message}
|
|
}
|
|
if !json.Valid(payload) {
|
|
return batchResult{Subject: subject, Error: "disclosure core returned invalid JSON"}
|
|
}
|
|
return batchResult{Subject: subject, OK: true, Data: json.RawMessage(payload)}
|
|
}
|
|
|
|
func validateSubject(raw string) error {
|
|
u, err := url.ParseRequestURI(raw)
|
|
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil {
|
|
return errors.New("subject must be an absolute http(s) URL without credentials")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func coreErrorMessage(payload []byte) string {
|
|
var problem struct {
|
|
Detail string `json:"detail"`
|
|
Title string `json:"title"`
|
|
}
|
|
if json.Unmarshal(payload, &problem) != nil {
|
|
return ""
|
|
}
|
|
if strings.TrimSpace(problem.Detail) != "" {
|
|
return problem.Detail
|
|
}
|
|
return strings.TrimSpace(problem.Title)
|
|
}
|
|
|
|
func ensureEOF(dec *json.Decoder) error {
|
|
var extra any
|
|
err := dec.Decode(&extra)
|
|
if errors.Is(err, io.EOF) {
|
|
return nil
|
|
}
|
|
if err == nil {
|
|
return errors.New("unexpected second JSON value")
|
|
}
|
|
return err
|
|
}
|
|
|
|
func cloneValues(in url.Values) url.Values {
|
|
out := make(url.Values, len(in))
|
|
for key, values := range in {
|
|
out[key] = append([]string(nil), values...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Server) problem(w http.ResponseWriter, status int, code, detail string) {
|
|
w.Header().Set("Content-Type", "application/problem+json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"type": "about:blank", "title": code, "status": status, "detail": detail,
|
|
})
|
|
}
|
|
|
|
func (s *Server) securityHeaders(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.Header().Set("Referrer-Policy", "no-referrer")
|
|
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=(), payment=()")
|
|
w.Header().Set("Cross-Origin-Opener-Policy", "same-origin")
|
|
w.Header().Set("Content-Security-Policy", "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'")
|
|
started := time.Now()
|
|
next.ServeHTTP(w, r)
|
|
s.logger.Debug("request", "method", r.Method, "path", r.URL.Path, "duration", time.Since(started))
|
|
})
|
|
}
|