pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-02-17 10:26:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete bash migration

Browse Source
This commit is contained in:
Owen
2025-02-12 21:56:13 -05:00
parent a7b8ffaf9f
commit 81c4199e87
9 changed files with 301 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
install/crowdsec/acquis.yaml Normal file
View File

@@ -0,0 +1,18 @@
filenames:
- /var/log/auth.log
- /var/log/syslog
labels:
type: syslog
---
poll_without_inotify: false
filenames:
- /var/log/traefik/*.log
labels:
type: traefik
---
listen_addr: 0.0.0.0:7422
appsec_config: crowdsecurity/appsec-default
name: myAppSecComponent
source: appsec
labels:
type: appsec

12
install/crowdsec/config.yaml Normal file
View File

@@ -0,0 +1,12 @@
api:
client:
insecure_skip_verify: false
credentials_path: /etc/crowdsec/local_api_credentials.yaml
server:
log_level: info
listen_uri: 0.0.0.0:9090
profiles_path: /etc/crowdsec/profiles.yaml
trusted_ips:
- 0.0.0.0/0
- 127.0.0.1
- ::1

108
install/crowdsec/dynamic_config.yml Normal file
View File

@@ -0,0 +1,108 @@
http:
middlewares:
redirect-to-https:
redirectScheme:
scheme: https
default-whitelist: # Whitelist middleware for internal IPs
ipWhiteList: # Internal IP addresses
sourceRange: # Internal IP addresses
- "10.0.0.0/8" # Internal IP addresses
- "192.168.0.0/16" # Internal IP addresses
- "172.16.0.0/12" # Internal IP addresses
# Basic security headers
security-headers:
headers:
customResponseHeaders: # Custom response headers
Server: "" # Remove server header
X-Powered-By: "" # Remove powered by header
X-Forwarded-Proto: "https" # Set forwarded proto to https
sslProxyHeaders: # SSL proxy headers
X-Forwarded-Proto: "https" # Set forwarded proto to https
hostsProxyHeaders: # Hosts proxy headers
- "X-Forwarded-Host" # Set forwarded host
contentTypeNosniff: true # Prevent MIME sniffing
customFrameOptionsValue: "SAMEORIGIN" # Set frame options
referrerPolicy: "strict-origin-when-cross-origin" # Set referrer policy
forceSTSHeader: true # Force STS header
stsIncludeSubdomains: true # Include subdomains
stsSeconds: 63072000 # STS seconds
stsPreload: true # Preload STS
# CrowdSec configuration with proper IP forwarding
crowdsec:
plugin:
crowdsec:
enabled: true # Enable CrowdSec plugin
logLevel: INFO # Log level
updateIntervalSeconds: 15 # Update interval
updateMaxFailure: 0 # Update max failure
defaultDecisionSeconds: 15 # Default decision seconds
httpTimeoutSeconds: 10 # HTTP timeout
crowdsecMode: live # CrowdSec mode
crowdsecAppsecEnabled: true # Enable AppSec
crowdsecAppsecHost: crowdsec:7422 # CrowdSec IP address which you noted down later
crowdsecAppsecFailureBlock: true # Block on failure
crowdsecAppsecUnreachableBlock: true # Block on unreachable
crowdsecLapiKey: "{{.TraefikBouncerKey}}" # CrowdSec API key which you noted down later
crowdsecLapiHost: crowdsec:9090 # CrowdSec
crowdsecLapiScheme: http # CrowdSec API scheme
forwardedHeadersTrustedIPs: # Forwarded headers trusted IPs
- "0.0.0.0/0" # All IP addresses are trusted for forwarded headers (CHANGE MADE HERE)
clientTrustedIPs: # Client trusted IPs (CHANGE MADE HERE)
- "10.0.0.0/8" # Internal LAN IP addresses
- "172.16.0.0/12" # Internal LAN IP addresses
- "192.168.0.0/16" # Internal LAN IP addresses
- "100.89.137.0/20" # Internal LAN IP addresses
routers:
# HTTP to HTTPS redirect router
main-app-router-redirect:
rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name
service: next-service
entryPoints:
- web
middlewares:
- redirect-to-https
# Next.js router (handles everything except API and WebSocket paths)
next-router:
rule: "Host(`{{.DomainName}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name
service: next-service
entryPoints:
- websecure
middlewares:
- security-headers # Add security headers middleware
tls:
certResolver: letsencrypt
# API router (handles /api/v1 paths)
api-router:
rule: "Host(`{{.DomainName}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name
service: api-service
entryPoints:
- websecure
middlewares:
- security-headers # Add security headers middleware
tls:
certResolver: letsencrypt
# WebSocket router
ws-router:
rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name
service: api-service
entryPoints:
- websecure
middlewares:
- security-headers # Add security headers middleware
tls:
certResolver: letsencrypt
services:
next-service:
loadBalancer:
servers:
- url: "http://pangolin:3002" # Next.js server
api-service:
loadBalancer:
servers:
- url: "http://pangolin:3000" # API/WebSocket server

376
install/crowdsec/install.go
View File

@@ -1,376 +0,0 @@
package crowdsec
import (
"bytes"
"embed"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
"time"
)
//go:embed fs/*
var configFiles embed.FS
// Config holds all configuration values
type Config struct {
DomainName string
EnrollmentKey string
TurnstileSiteKey string
TurnstileSecretKey string
GID string
CrowdsecIP string
TraefikBouncerKey string
PangolinIP string
}
// DockerContainer represents a Docker container
type DockerContainer struct {
NetworkSettings struct {
Networks map[string]struct {
IPAddress string `json:"IPAddress"`
} `json:"Networks"`
} `json:"NetworkSettings"`
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func run() error {
// Create configuration
config := &Config{}
// Run installation steps
if err := backupConfig(); err != nil {
return fmt.Errorf("backup failed: %v", err)
}
if err := createPangolinNetwork(); err != nil {
return fmt.Errorf("network creation failed: %v", err)
}
if err := modifyDockerCompose(); err != nil {
return fmt.Errorf("docker-compose modification failed: %v", err)
}
if err := createConfigFiles(*config); err != nil {
return fmt.Errorf("config file creation failed: %v", err)
}
if err := retrieveIPs(config); err != nil {
return fmt.Errorf("IP retrieval failed: %v", err)
}
if err := retrieveBouncerKey(config); err != nil {
return fmt.Errorf("bouncer key retrieval failed: %v", err)
}
if err := replacePlaceholders(config); err != nil {
return fmt.Errorf("placeholder replacement failed: %v", err)
}
if err := deployStack(); err != nil {
return fmt.Errorf("deployment failed: %v", err)
}
if err := verifyDeployment(); err != nil {
return fmt.Errorf("verification failed: %v", err)
}
printInstructions()
return nil
}
func backupConfig() error {
// Backup docker-compose.yml
if _, err := os.Stat("docker-compose.yml"); err == nil {
if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil {
return fmt.Errorf("failed to backup docker-compose.yml: %v", err)
}
}
// Backup config directory
if _, err := os.Stat("config"); err == nil {
cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to backup config directory: %v", err)
}
}
return nil
}
func createPangolinNetwork() error {
// Check if network exists
cmd := exec.Command("docker", "network", "inspect", "pangolin")
if err := cmd.Run(); err == nil {
fmt.Println("pangolin network already exists")
return nil
}
// Create network
cmd = exec.Command("docker", "network", "create", "pangolin")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to create pangolin network: %v", err)
}
return nil
}
func modifyDockerCompose() error {
// Read existing docker-compose.yml
content, err := os.ReadFile("docker-compose.yml")
if err != nil {
return fmt.Errorf("failed to read docker-compose.yml: %v", err)
}
// Verify required services exist
requiredServices := []string{"services:", "pangolin:", "gerbil:", "traefik:"}
for _, service := range requiredServices {
if !bytes.Contains(content, []byte(service)) {
return fmt.Errorf("required service %s not found in docker-compose.yml", service)
}
}
// Add crowdsec service
modified := addCrowdsecService(string(content))
// Write modified content
if err := os.WriteFile("docker-compose.yml", []byte(modified), 0644); err != nil {
return fmt.Errorf("failed to write modified docker-compose.yml: %v", err)
}
return nil
}
func retrieveIPs(config *Config) error {
// Start required containers
cmd := exec.Command("docker", "compose", "up", "-d", "pangolin", "crowdsec")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to start containers: %v", err)
}
defer exec.Command("docker", "compose", "down").Run()
// Wait for containers to start
time.Sleep(10 * time.Second)
// Get Pangolin IP
pangolinIP, err := getContainerIP("pangolin")
if err != nil {
return fmt.Errorf("failed to get pangolin IP: %v", err)
}
config.PangolinIP = pangolinIP
// Get CrowdSec IP
crowdsecIP, err := getContainerIP("crowdsec")
if err != nil {
return fmt.Errorf("failed to get crowdsec IP: %v", err)
}
config.CrowdsecIP = crowdsecIP
return nil
}
func retrieveBouncerKey(config *Config) error {
// Start crowdsec container
cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to start crowdsec: %v", err)
}
defer exec.Command("docker", "compose", "down").Run()
// Wait for container to start
time.Sleep(10 * time.Second)
// Get bouncer key
output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output()
if err != nil {
return fmt.Errorf("failed to get bouncer key: %v", err)
}
// Parse key from output
lines := strings.Split(string(output), "\n")
for _, line := range lines {
if strings.Contains(line, "key:") {
config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1])
break
}
}
return nil
}
func replacePlaceholders(config *Config) error {
// Get user input
fmt.Print("Enter your Domain Name (e.g., pangolin.example.com): ")
fmt.Scanln(&config.DomainName)
fmt.Print("Enter your CrowdSec Enrollment Key: ")
fmt.Scanln(&config.EnrollmentKey)
fmt.Print("Enter your Cloudflare Turnstile Site Key: ")
fmt.Scanln(&config.TurnstileSiteKey)
fmt.Print("Enter your Cloudflare Turnstile Secret Key: ")
fmt.Scanln(&config.TurnstileSecretKey)
fmt.Print("Enter your GID (or leave empty for default 1000): ")
gid := ""
fmt.Scanln(&gid)
if gid == "" {
config.GID = "1000"
} else {
config.GID = gid
}
return nil
}
func deployStack() error {
cmd := exec.Command("docker", "compose", "up", "-d")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to deploy stack: %v", err)
}
fmt.Println("Stack deployed. Waiting 2 minutes for services to initialize...")
time.Sleep(2 * time.Minute)
return nil
}
func verifyDeployment() error {
resp, err := exec.Command("curl", "-s", "http://localhost:6060/metrics").Output()
if err != nil {
return fmt.Errorf("failed to get metrics: %v", err)
}
if !bytes.Contains(resp, []byte("appsec")) {
return fmt.Errorf("appsec metrics not found in response")
}
return nil
}
func printInstructions() {
fmt.Println(`
--- Testing Instructions ---
1. Test Captcha Implementation:
docker exec crowdsec cscli decisions add --ip YOUR_IP --type captcha -d 1h
(Replace YOUR_IP with your actual IP address)
2. Verify decisions:
docker exec -it crowdsec cscli decisions list
3. Test security by accessing DOMAIN_NAME/.env (should return 403)
(Replace DOMAIN_NAME with the domain you entered)
--- Troubleshooting ---
1. If encountering 403 errors:
- Check Traefik logs: docker compose logs traefik -f
- Verify CrowdSec logs: docker compose logs crowdsec
2. For plugin errors:
- Verify http notifications are commented out in profiles.yaml
- Restart services: docker compose restart traefik crowdsec
3. For Captcha issues:
- Ensure Turnstile is configured in non-interactive mode
- Verify captcha.html configuration
- Check container network connectivity
Useful Commands:
- View Traefik logs: docker compose logs traefik -f
- View CrowdSec logs: docker compose logs crowdsec
- List decisions: docker exec -it crowdsec cscli decisions list
- Check metrics: curl http://localhost:6060/metrics | grep appsec
`)
}
// Helper functions
func copyFile(src, dst string) error {
source, err := os.Open(src)
if err != nil {
return err
}
defer source.Close()
destination, err := os.Create(dst)
if err != nil {
return err
}
defer destination.Close()
_, err = io.Copy(destination, source)
return err
}
func getContainerIP(containerName string) (string, error) {
output, err := exec.Command("docker", "inspect", containerName).Output()
if err != nil {
return "", err
}
var containers []DockerContainer
if err := json.Unmarshal(output, &containers); err != nil {
return "", err
}
if len(containers) == 0 {
return "", fmt.Errorf("no container found")
}
for _, network := range containers[0].NetworkSettings.Networks {
return network.IPAddress, nil
}
return "", fmt.Errorf("no IP address found")
}
func addCrowdsecService(content string) string {
// Implementation of adding crowdsec service to docker-compose.yml
// This would involve string manipulation or template rendering
// The actual implementation would depend on how you want to structure the docker-compose modifications
return content + `
crowdsec:
image: crowdsecurity/crowdsec:latest
container_name: crowdsec
environment:
GID: "${GID-1000}"
COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules
ENROLL_INSTANCE_NAME: "pangolin-crowdsec"
PARSERS: crowdsecurity/whitelists
ENROLL_KEY: ${ENROLLMENT_KEY}
ACQUIRE_FILES: "/var/log/traefik/*.log"
ENROLL_TAGS: docker
networks:
- pangolin
healthcheck:
test: ["CMD", "cscli", "capi", "status"]
depends_on:
- gerbil
labels:
- "traefik.enable=false"
volumes:
- ./config/crowdsec:/etc/crowdsec
- ./config/crowdsec/db:/var/lib/crowdsec/data
- ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro
- ./config/crowdsec_logs/syslog:/var/log/syslog:ro
- ./config/crowdsec_logs:/var/log
- ./config/traefik/logs:/var/log/traefik
ports:
- 9090:9090
- 6060:6060
expose:
- 9090
- 6060
- 7422
restart: unless-stopped
command: -t`
}

2
install/crowdsec/local_api_credentials.yaml Normal file
View File

@@ -0,0 +1,2 @@
url: http://0.0.0.0:9090
login: localhost

25
install/crowdsec/profiles.yaml Normal file
View File

@@ -0,0 +1,25 @@
name: captcha_remediation
filters:
- Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() contains "http"
decisions:
- type: captcha
duration: 4h
on_success: break
---
name: default_ip_remediation
filters:
- Alert.Remediation == true && Alert.GetScope() == "Ip"
decisions:
- type: ban
duration: 4h
on_success: break
---
name: default_range_remediation
filters:
- Alert.Remediation == true && Alert.GetScope() == "Range"
decisions:
- type: ban
duration: 4h
on_success: break

87
install/crowdsec/traefik_config.yml Normal file
View File

@@ -0,0 +1,87 @@
api:
insecure: true
dashboard: true
providers:
http:
endpoint: "http://pangolin:3001/api/v1/traefik-config"
pollInterval: "5s"
file:
filename: "/etc/traefik/dynamic_config.yml"
experimental:
plugins:
badger:
moduleName: "github.com/fosrl/badger"
version: "{{.BadgerVersion}}"
crowdsec: # CrowdSec plugin configuration added
moduleName: "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
version: "v1.3.5"
log:
level: "INFO"
format: "json" # Log format changed to json for better parsing
accessLog: # We enable access logs as json
filePath: "/var/log/traefik/access.log"
format: json
filters:
statusCodes:
- "200-299" # Success codes
- "400-499" # Client errors
- "500-599" # Server errors
retryAttempts: true
minDuration: "100ms" # Increased to focus on slower requests
bufferingSize: 100 # Add buffering for better performance
fields:
defaultMode: drop # Start with dropping all fields
names:
ClientAddr: keep # Keep client address for IP tracking
ClientHost: keep # Keep client host for IP tracking
RequestMethod: keep # Keep request method for tracking
RequestPath: keep # Keep request path for tracking
RequestProtocol: keep # Keep request protocol for tracking
DownstreamStatus: keep # Keep downstream status for tracking
DownstreamContentSize: keep # Keep downstream content size for tracking
Duration: keep # Keep request duration for tracking
ServiceName: keep # Keep service name for tracking
StartUTC: keep # Keep start time for tracking
TLSVersion: keep # Keep TLS version for tracking
TLSCipher: keep # Keep TLS cipher for tracking
RetryAttempts: keep # Keep retry attempts for tracking
headers:
defaultMode: drop # Start with dropping all headers
names:
User-Agent: keep # Keep user agent for tracking
X-Real-Ip: keep # Keep real IP for tracking
X-Forwarded-For: keep # Keep forwarded IP for tracking
X-Forwarded-Proto: keep # Keep forwarded protocol for tracking
Content-Type: keep # Keep content type for tracking
Authorization: redact # Redact sensitive information
Cookie: redact # Redact sensitive information
certificatesResolvers:
letsencrypt:
acme:
httpChallenge:
entryPoint: web
email: "{{.LetsEncryptEmail}}"
storage: "/letsencrypt/acme.json"
caServer: "https://acme-v02.api.letsencrypt.org/directory"
entryPoints:
web:
address: ":80"
websecure:
address: ":443"
transport:
respondingTimeouts:
readTimeout: "30m"
http:
tls:
certResolver: "letsencrypt"
middlewares: # CHANGE MADE HERE (BOUNCER ENABLED) !!!
- crowdsec@file
serversTransport:
insecureSkipVerify: true