add api for access log events

This commit is contained in:
pascal
2026-01-29 14:27:57 +01:00
parent f204da0d68
commit 8e0b7b6c25
23 changed files with 745 additions and 189 deletions

View File

@@ -0,0 +1,105 @@
package accesslogs
import (
"net"
"net/netip"
"time"
"github.com/netbirdio/netbird/management/server/peer"
"github.com/netbirdio/netbird/shared/management/http/api"
"github.com/netbirdio/netbird/shared/management/proto"
)
type AccessLogEntry struct {
ID string `gorm:"primaryKey"`
AccountID string `gorm:"index"`
ProxyID string `gorm:"index"`
Timestamp time.Time `gorm:"index"`
GeoLocation peer.Location `gorm:"embedded;embeddedPrefix:location_"`
Method string
Host string
Path string
Duration time.Duration
StatusCode int
Reason string
UserId string
AuthMethodUsed string
}
// FromProto creates an AccessLogEntry from a proto.AccessLog
func (a *AccessLogEntry) FromProto(proxyLog *proto.AccessLog) {
a.ID = proxyLog.GetLogId()
a.ProxyID = proxyLog.GetServiceId()
a.Timestamp = proxyLog.GetTimestamp().AsTime()
a.Method = proxyLog.GetMethod()
a.Host = proxyLog.GetHost()
a.Path = proxyLog.GetPath()
a.Duration = time.Duration(proxyLog.GetDurationMs()) * time.Millisecond
a.StatusCode = int(proxyLog.GetResponseCode())
a.UserId = proxyLog.GetUserId()
a.AuthMethodUsed = proxyLog.GetAuthMechanism()
a.AccountID = proxyLog.GetAccountId()
if sourceIP := proxyLog.GetSourceIp(); sourceIP != "" {
if ip, err := netip.ParseAddr(sourceIP); err == nil {
a.GeoLocation.ConnectionIP = net.IP(ip.AsSlice())
}
}
if !proxyLog.GetAuthSuccess() {
a.Reason = "Authentication failed"
} else if proxyLog.GetResponseCode() >= 400 {
a.Reason = "Request failed"
}
}
// ToAPIResponse converts an AccessLogEntry to the API ProxyAccessLog type
func (a *AccessLogEntry) ToAPIResponse() *api.ProxyAccessLog {
var sourceIP *string
if a.GeoLocation.ConnectionIP != nil {
ip := a.GeoLocation.ConnectionIP.String()
sourceIP = &ip
}
var reason *string
if a.Reason != "" {
reason = &a.Reason
}
var userID *string
if a.UserId != "" {
userID = &a.UserId
}
var authMethod *string
if a.AuthMethodUsed != "" {
authMethod = &a.AuthMethodUsed
}
var countryCode *string
if a.GeoLocation.CountryCode != "" {
countryCode = &a.GeoLocation.CountryCode
}
var cityName *string
if a.GeoLocation.CityName != "" {
cityName = &a.GeoLocation.CityName
}
return &api.ProxyAccessLog{
Id: a.ID,
ProxyId: a.ProxyID,
Timestamp: a.Timestamp,
Method: a.Method,
Host: a.Host,
Path: a.Path,
DurationMs: int(a.Duration.Milliseconds()),
StatusCode: a.StatusCode,
SourceIp: sourceIP,
Reason: reason,
UserId: userID,
AuthMethodUsed: authMethod,
CountryCode: countryCode,
CityName: cityName,
}
}

View File

@@ -0,0 +1,10 @@
package accesslogs
import (
"context"
)
type Manager interface {
SaveAccessLog(ctx context.Context, proxyLog *AccessLogEntry) error
GetAllAccessLogs(ctx context.Context, accountID, userID string) ([]*AccessLogEntry, error)
}

View File

@@ -0,0 +1,45 @@
package manager
import (
"net/http"
"github.com/gorilla/mux"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
nbcontext "github.com/netbirdio/netbird/management/server/context"
"github.com/netbirdio/netbird/shared/management/http/api"
"github.com/netbirdio/netbird/shared/management/http/util"
)
type handler struct {
manager accesslogs.Manager
}
func RegisterEndpoints(router *mux.Router, manager accesslogs.Manager) {
h := &handler{
manager: manager,
}
router.HandleFunc("/events/proxy", h.getAccessLogs).Methods("GET", "OPTIONS")
}
func (h *handler) getAccessLogs(w http.ResponseWriter, r *http.Request) {
userAuth, err := nbcontext.GetUserAuthFromContext(r.Context())
if err != nil {
util.WriteError(r.Context(), err, w)
return
}
logs, err := h.manager.GetAllAccessLogs(r.Context(), userAuth.AccountId, userAuth.UserId)
if err != nil {
util.WriteError(r.Context(), err, w)
return
}
apiLogs := make([]api.ProxyAccessLog, 0, len(logs))
for _, log := range logs {
apiLogs = append(apiLogs, *log.ToAPIResponse())
}
util.WriteJSONObject(r.Context(), w, apiLogs)
}

View File

@@ -0,0 +1,74 @@
package manager
import (
"context"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/server/geolocation"
"github.com/netbirdio/netbird/management/server/permissions"
"github.com/netbirdio/netbird/management/server/permissions/modules"
"github.com/netbirdio/netbird/management/server/permissions/operations"
"github.com/netbirdio/netbird/management/server/store"
"github.com/netbirdio/netbird/shared/management/status"
)
type managerImpl struct {
store store.Store
permissionsManager permissions.Manager
geo geolocation.Geolocation
}
func NewManager(store store.Store, permissionsManager permissions.Manager, geo geolocation.Geolocation) accesslogs.Manager {
return &managerImpl{
store: store,
permissionsManager: permissionsManager,
geo: geo,
}
}
// SaveAccessLog saves an access log entry to the database after enriching it
func (m *managerImpl) SaveAccessLog(ctx context.Context, logEntry *accesslogs.AccessLogEntry) error {
if m.geo != nil && logEntry.GeoLocation.ConnectionIP != nil {
location, err := m.geo.Lookup(logEntry.GeoLocation.ConnectionIP)
if err != nil {
log.WithContext(ctx).Warnf("failed to get location for access log source IP [%s]: %v", logEntry.GeoLocation.ConnectionIP.String(), err)
} else {
logEntry.GeoLocation.CountryCode = location.Country.ISOCode
logEntry.GeoLocation.CityName = location.City.Names.En
logEntry.GeoLocation.GeoNameID = location.City.GeonameID
}
}
if err := m.store.CreateAccessLog(ctx, logEntry); err != nil {
log.WithContext(ctx).WithFields(log.Fields{
"proxy_id": logEntry.ProxyID,
"method": logEntry.Method,
"host": logEntry.Host,
"path": logEntry.Path,
"status": logEntry.StatusCode,
}).Errorf("failed to save access log: %v", err)
return err
}
return nil
}
// GetAllAccessLogs retrieves all access logs for an account
func (m *managerImpl) GetAllAccessLogs(ctx context.Context, accountID, userID string) ([]*accesslogs.AccessLogEntry, error) {
ok, err := m.permissionsManager.ValidateUserPermissions(ctx, accountID, userID, modules.Services, operations.Read)
if err != nil {
return nil, status.NewPermissionValidationError(err)
}
if !ok {
return nil, status.NewPermissionDeniedError()
}
logs, err := m.store.GetAccountAccessLogs(ctx, store.LockingStrengthNone, accountID)
if err != nil {
return nil, err
}
return logs, nil
}

View File

@@ -76,7 +76,7 @@ func (h *handler) createCustomDomain(w http.ResponseWriter, r *http.Request) {
return
}
var req api.PostApiReverseProxyDomainsJSONRequestBody
var req api.PostApiReverseProxiesDomainsJSONRequestBody
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w)
return

View File

@@ -7,6 +7,8 @@ import (
"github.com/gorilla/mux"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
accesslogsmanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs/manager"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
nbcontext "github.com/netbirdio/netbird/management/server/context"
"github.com/netbirdio/netbird/shared/management/http/api"
@@ -18,7 +20,7 @@ type handler struct {
manager reverseproxy.Manager
}
func RegisterEndpoints(manager reverseproxy.Manager, domainManager domain.Manager, router *mux.Router) {
func RegisterEndpoints(manager reverseproxy.Manager, domainManager domain.Manager, accessLogsManager accesslogs.Manager, router *mux.Router) {
h := &handler{
manager: manager,
}
@@ -27,6 +29,8 @@ func RegisterEndpoints(manager reverseproxy.Manager, domainManager domain.Manage
domainRouter := router.PathPrefix("/reverse-proxies").Subrouter()
domain.RegisterEndpoints(domainRouter, domainManager)
accesslogsmanager.RegisterEndpoints(router, accessLogsManager)
router.HandleFunc("/reverse-proxies", h.getAllReverseProxies).Methods("GET", "OPTIONS")
router.HandleFunc("/reverse-proxies", h.createReverseProxy).Methods("POST", "OPTIONS")
router.HandleFunc("/reverse-proxies/{proxyId}", h.getReverseProxy).Methods("GET", "OPTIONS")
@@ -62,7 +66,7 @@ func (h *handler) createReverseProxy(w http.ResponseWriter, r *http.Request) {
return
}
var req api.PostApiReverseProxyJSONRequestBody
var req api.ReverseProxyRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w)
return
@@ -120,7 +124,7 @@ func (h *handler) updateReverseProxy(w http.ResponseWriter, r *http.Request) {
return
}
var req api.PutApiReverseProxyProxyIdJSONRequestBody
var req api.ReverseProxyRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w)
return

View File

@@ -182,12 +182,13 @@ func (r *ReverseProxy) ToProtoMapping(operation Operation, setupKey string) *pro
}
return &proto.ProxyMapping{
Type: operationToProtoType(operation),
Id: r.ID,
Domain: r.Domain,
Path: pathMappings,
SetupKey: setupKey,
Auth: auth,
Type: operationToProtoType(operation),
Id: r.ID,
Domain: r.Domain,
Path: pathMappings,
SetupKey: setupKey,
Auth: auth,
AccountId: r.AccountID,
}
}

View File

@@ -21,6 +21,8 @@ import (
"github.com/netbirdio/management-integrations/integrations"
"github.com/netbirdio/netbird/encryption"
"github.com/netbirdio/netbird/formatter/hook"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
accesslogsmanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs/manager"
nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
"github.com/netbirdio/netbird/management/server/activity"
nbContext "github.com/netbirdio/netbird/management/server/context"
@@ -92,7 +94,7 @@ func (s *BaseServer) EventStore() activity.Store {
func (s *BaseServer) APIHandler() http.Handler {
return Create(s, func() http.Handler {
httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.ZonesManager(), s.RecordsManager(), s.NetworkMapController(), s.IdpManager(), s.ReverseProxyManager(), s.ReverseProxyDomainManager())
httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.ZonesManager(), s.RecordsManager(), s.NetworkMapController(), s.IdpManager(), s.ReverseProxyManager(), s.ReverseProxyDomainManager(), s.AccessLogsManager())
if err != nil {
log.Fatalf("failed to create API handler: %v", err)
}
@@ -159,11 +161,18 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
func (s *BaseServer) ReverseProxyGRPCServer() *nbgrpc.ProxyServiceServer {
return Create(s, func() *nbgrpc.ProxyServiceServer {
proxyService := nbgrpc.NewProxyServiceServer(s.Store(), s.AccountManager())
proxyService := nbgrpc.NewProxyServiceServer(s.Store(), s.AccountManager(), s.AccessLogsManager())
return proxyService
})
}
func (s *BaseServer) AccessLogsManager() accesslogs.Manager {
return Create(s, func() accesslogs.Manager {
accessLogManager := accesslogsmanager.NewManager(s.Store(), s.PermissionsManager(), s.GeoLocationManager())
return accessLogManager
})
}
func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
// Load server's certificate and private key
serverCert, err := tls.LoadX509KeyPair(certFile, certKey)

View File

@@ -6,13 +6,15 @@ import (
"sync"
"time"
"github.com/netbirdio/netbird/management/server/activity"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"github.com/netbirdio/netbird/management/server/activity"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/server/store"
"github.com/netbirdio/netbird/management/server/types"
"github.com/netbirdio/netbird/shared/management/proto"
@@ -21,6 +23,7 @@ import (
type reverseProxyStore interface {
GetReverseProxies(ctx context.Context, lockStrength store.LockingStrength) ([]*reverseproxy.ReverseProxy, error)
GetAccountReverseProxies(ctx context.Context, lockStrength store.LockingStrength, accountID string) ([]*reverseproxy.ReverseProxy, error)
GetReverseProxyByID(ctx context.Context, lockStrength store.LockingStrength, accountID string, serviceID string) (*reverseproxy.ReverseProxy, error)
}
type keyStore interface {
@@ -43,6 +46,9 @@ type ProxyServiceServer struct {
// Store for client setup keys
keyStore keyStore
// Manager for access logs
accessLogManager accesslogs.Manager
}
// proxyConnection represents a connected proxy
@@ -56,11 +62,12 @@ type proxyConnection struct {
}
// NewProxyServiceServer creates a new proxy service server
func NewProxyServiceServer(store reverseProxyStore, keys keyStore) *ProxyServiceServer {
func NewProxyServiceServer(store reverseProxyStore, keys keyStore, accessLogMgr accesslogs.Manager) *ProxyServiceServer {
return &ProxyServiceServer{
updatesChan: make(chan *proto.ProxyMapping, 100),
reverseProxyStore: store,
keyStore: keys,
accessLogManager: accessLogMgr,
}
}
@@ -186,21 +193,30 @@ func (s *ProxyServiceServer) sender(conn *proxyConnection, errChan chan<- error)
// SendAccessLog processes access log from proxy
func (s *ProxyServiceServer) SendAccessLog(ctx context.Context, req *proto.SendAccessLogRequest) (*proto.SendAccessLogResponse, error) {
log.WithFields(log.Fields{
"proxy_id": "", // TODO: get proxy id, probably from context or maybe from request message.
"reverse_proxy_id": req.GetLog().GetServiceId(),
"host": req.GetLog().GetHost(),
"path": req.GetLog().GetPath(),
"method": req.GetLog().GetMethod(),
"response_code": req.GetLog().GetResponseCode(),
"duration_ms": req.GetLog().GetDurationMs(),
"source_ip": req.GetLog().GetSourceIp(),
"auth_mechanism": req.GetLog().GetAuthMechanism(),
"user_id": req.GetLog().GetUserId(),
"auth_success": req.GetLog().GetAuthSuccess(),
}).Info("Access log from proxy")
accessLog := req.GetLog()
log.WithFields(log.Fields{
"reverse_proxy_id": accessLog.GetServiceId(),
"account_id": accessLog.GetAccountId(),
"host": accessLog.GetHost(),
"path": accessLog.GetPath(),
"method": accessLog.GetMethod(),
"response_code": accessLog.GetResponseCode(),
"duration_ms": accessLog.GetDurationMs(),
"source_ip": accessLog.GetSourceIp(),
"auth_mechanism": accessLog.GetAuthMechanism(),
"user_id": accessLog.GetUserId(),
"auth_success": accessLog.GetAuthSuccess(),
}).Debug("Access log from proxy")
logEntry := &accesslogs.AccessLogEntry{}
logEntry.FromProto(accessLog)
if err := s.accessLogManager.SaveAccessLog(ctx, logEntry); err != nil {
log.WithContext(ctx).Errorf("failed to save access log: %v", err)
return nil, status.Errorf(codes.Internal, "failed to save access log: %v", err)
}
// TODO: Store access log in database/metrics system
return &proto.SendAccessLogResponse{}, nil
}

View File

@@ -13,6 +13,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
reverseproxymanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/manager"
idpmanager "github.com/netbirdio/netbird/management/server/idp"
@@ -63,7 +64,7 @@ const (
)
// NewAPIHandler creates the Management service HTTP API handler registering all the available endpoints.
func NewAPIHandler(ctx context.Context, accountManager account.Manager, networksManager nbnetworks.Manager, resourceManager resources.Manager, routerManager routers.Manager, groupsManager nbgroups.Manager, LocationManager geolocation.Geolocation, authManager auth.Manager, appMetrics telemetry.AppMetrics, integratedValidator integrated_validator.IntegratedValidator, proxyController port_forwarding.Controller, permissionsManager permissions.Manager, peersManager nbpeers.Manager, settingsManager settings.Manager, zManager zones.Manager, rManager records.Manager, networkMapController network_map.Controller, idpManager idpmanager.Manager, reverseProxyManager reverseproxy.Manager, reverseProxyDomainManager domain.Manager) (http.Handler, error) {
func NewAPIHandler(ctx context.Context, accountManager account.Manager, networksManager nbnetworks.Manager, resourceManager resources.Manager, routerManager routers.Manager, groupsManager nbgroups.Manager, LocationManager geolocation.Geolocation, authManager auth.Manager, appMetrics telemetry.AppMetrics, integratedValidator integrated_validator.IntegratedValidator, proxyController port_forwarding.Controller, permissionsManager permissions.Manager, peersManager nbpeers.Manager, settingsManager settings.Manager, zManager zones.Manager, rManager records.Manager, networkMapController network_map.Controller, idpManager idpmanager.Manager, reverseProxyManager reverseproxy.Manager, reverseProxyDomainManager domain.Manager, reverseProxyAccessLogsManager accesslogs.Manager) (http.Handler, error) {
// Register bypass paths for unauthenticated endpoints
if err := bypass.AddBypassPath("/api/instance"); err != nil {
@@ -159,7 +160,7 @@ func NewAPIHandler(ctx context.Context, accountManager account.Manager, networks
idp.AddEndpoints(accountManager, router)
instance.AddEndpoints(instanceManager, router)
instance.AddVersionEndpoint(instanceManager, router)
reverseproxymanager.RegisterEndpoints(reverseProxyManager, reverseProxyDomainManager, router)
reverseproxymanager.RegisterEndpoints(reverseProxyManager, reverseProxyDomainManager, reverseProxyAccessLogsManager, router)
// Mount embedded IdP handler at /oauth2 path if configured
if embeddedIdpEnabled {

View File

@@ -29,6 +29,7 @@ import (
nbdns "github.com/netbirdio/netbird/dns"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
"github.com/netbirdio/netbird/management/internals/modules/zones"
"github.com/netbirdio/netbird/management/internals/modules/zones/records"
@@ -130,6 +131,7 @@ func NewSqlStore(ctx context.Context, db *gorm.DB, storeEngine types.Engine, met
&installation{}, &types.ExtraSettings{}, &posture.Checks{}, &nbpeer.NetworkAddress{},
&networkTypes.Network{}, &routerTypes.NetworkRouter{}, &resourceTypes.NetworkResource{}, &types.AccountOnboarding{},
&types.Job{}, &zones.Zone{}, &records.Record{}, &types.UserInviteRecord{}, &reverseproxy.ReverseProxy{}, &domain.Domain{},
&accesslogs.AccessLogEntry{},
)
if err != nil {
return nil, fmt.Errorf("auto migratePreAuto: %w", err)
@@ -4782,3 +4784,40 @@ func (s *SqlStore) DeleteCustomDomain(ctx context.Context, accountID string, dom
return nil
}
// CreateAccessLog creates a new access log entry in the database
func (s *SqlStore) CreateAccessLog(ctx context.Context, logEntry *accesslogs.AccessLogEntry) error {
result := s.db.Create(logEntry)
if result.Error != nil {
log.WithContext(ctx).WithFields(log.Fields{
"proxy_id": logEntry.ProxyID,
"method": logEntry.Method,
"host": logEntry.Host,
"path": logEntry.Path,
}).Errorf("failed to create access log entry in store: %v", result.Error)
return status.Errorf(status.Internal, "failed to create access log entry in store")
}
return nil
}
// GetAccountAccessLogs retrieves all access logs for a given account
func (s *SqlStore) GetAccountAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*accesslogs.AccessLogEntry, error) {
var logs []*accesslogs.AccessLogEntry
query := s.db.WithContext(ctx).
Where(accountIDCondition, accountID).
Order("timestamp DESC").
Limit(1000)
if lockStrength != LockingStrengthNone {
query = query.Clauses(clause.Locking{Strength: string(lockStrength)})
}
result := query.Find(&logs)
if result.Error != nil {
log.WithContext(ctx).Errorf("failed to get access logs from store: %v", result.Error)
return nil, status.Errorf(status.Internal, "failed to get access logs from store")
}
return logs, nil
}

View File

@@ -24,6 +24,7 @@ import (
"github.com/netbirdio/netbird/dns"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
"github.com/netbirdio/netbird/management/internals/modules/zones"
"github.com/netbirdio/netbird/management/internals/modules/zones/records"
@@ -257,6 +258,9 @@ type Store interface {
CreateCustomDomain(ctx context.Context, accountID string, domainName string, validated bool) (*domain.Domain, error)
UpdateCustomDomain(ctx context.Context, accountID string, d *domain.Domain) (*domain.Domain, error)
DeleteCustomDomain(ctx context.Context, accountID string, domainID string) error
CreateAccessLog(ctx context.Context, log *accesslogs.AccessLogEntry) error
GetAccountAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*accesslogs.AccessLogEntry, error)
}
const (