diff --git a/.env.example b/.env.example index fae54787..ea26fd22 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ PUBLIC_APP_URL=http://localhost -TRUST_PROXY=false \ No newline at end of file +TRUST_PROXY=false +MAXMIND_LICENSE_KEY= \ No newline at end of file diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index d9c4cf96..be1bf6f6 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -40,9 +40,6 @@ jobs: registry: ghcr.io username: ${{github.repository_owner}} password: ${{secrets.GITHUB_TOKEN}} - - - name: Download GeoLite2 City database - run: MAXMIND_LICENSE_KEY=${{ secrets.MAXMIND_LICENSE_KEY }} sh scripts/download-ip-database.sh - name: Build and push uses: docker/build-push-action@v4 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 41a3faed..5aff7f4b 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -16,9 +16,6 @@ jobs: cache: 'npm' cache-dependency-path: frontend/package-lock.json - - name: Create dummy GeoLite2 City database - run: touch ./backend/GeoLite2-City.mmdb - - name: Build Docker Image run: docker build -t stonith404/pocket-id . diff --git a/Dockerfile b/Dockerfile index 8232a094..63460552 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,6 @@ COPY --from=frontend-builder /app/frontend/package.json ./frontend/package.json COPY --from=backend-builder /app/backend/pocket-id-backend ./backend/pocket-id-backend COPY --from=backend-builder /app/backend/migrations ./backend/migrations -COPY --from=backend-builder /app/backend/GeoLite2-City.mmdb ./backend/GeoLite2-City.mmdb COPY --from=backend-builder /app/backend/email-templates ./backend/email-templates COPY --from=backend-builder /app/backend/images ./backend/images diff --git a/README.md b/README.md index 935500ef..a1283af7 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,6 @@ Required tools: cd .. pm2 start pocket-id-backend --name pocket-id-backend - # Optional: Download the GeoLite2 city database. - # If not downloaded the ip location in the audit log will be empty. - MAXMIND_LICENSE_KEY= sh scripts/download-ip-database.sh - # Start the frontend cd ../frontend npm install @@ -130,9 +126,6 @@ docker compose up -d cd .. pm2 start pocket-id-backend --name pocket-id-backend - # Optional: Update the GeoLite2 city database - MAXMIND_LICENSE_KEY= sh scripts/download-ip-database.sh - # Start the frontend cd ../frontend npm install @@ -146,17 +139,20 @@ docker compose up -d ## Environment variables -| Variable | Default Value | Recommended to change | Description | -| ---------------------- | ----------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PUBLIC_APP_URL` | `http://localhost` | yes | The URL where you will access the app. | -| `TRUST_PROXY` | `false` | yes | Whether the app is behind a reverse proxy. | -| `PUID` and `PGID` | `1000` | yes | The user and group ID of the user who should run Pocket ID inside the Docker container and owns the files that are mounted with the volume. You can get the `PUID` and `GUID` of your user on your host machine by using the command `id`. For more information see [this article](https://docs.linuxserver.io/general/understanding-puid-and-pgid/#using-the-variables). | -| `DB_PATH` | `data/pocket-id.db` | no | The path to the SQLite database. | -| `UPLOAD_PATH` | `data/uploads` | no | The path where the uploaded files are stored. | -| `INTERNAL_BACKEND_URL` | `http://localhost:8080` | no | The URL where the backend is accessible. | -| `CADDY_PORT` | `80` | no | The port on which Caddy should listen. Caddy is only active inside the Docker container. If you want to change the exposed port of the container then you sould change this variable. | -| `PORT` | `3000` | no | The port on which the frontend should listen. | -| `BACKEND_PORT` | `8080` | no | The port on which the backend should listen. | +| Variable | Default Value | Recommended to change | Description | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PUBLIC_APP_URL` | `http://localhost` | yes | The URL where you will access the app. | +| `TRUST_PROXY` | `false` | yes | Whether the app is behind a reverse proxy. | +| `MAXMIND_LICENSE_KEY` | `-` | yes | # License key for MaxMind GeoIP2lite. | +| License key for the GeoLite2 database. This is necessary to get the location of the ip adresses in the audit log. If not specified, the location will be unknown. You can get the license key here. | +| `PUID` and `PGID` | `1000` | yes | The user and group ID of the user who should run Pocket ID inside the Docker container and owns the files that are mounted with the volume. You can get the `PUID` and `GUID` of your user on your host machine by using the command `id`. For more information see [this article](https://docs.linuxserver.io/general/understanding-puid-and-pgid/#using-the-variables). | +| `DB_PATH` | `data/pocket-id.db` | no | The path to the SQLite database. | +| `UPLOAD_PATH` | `data/uploads` | no | The path where the uploaded files are stored. | +| `INTERNAL_BACKEND_URL` | `http://localhost:8080` | no | The URL where the backend is accessible. | +| `GEOLITE_DB_PATH` | `data/GeoLite2-City.mmdb` | no | The path where the GeoLite2 database should be stored. | +| `CADDY_PORT` | `80` | no | The port on which Caddy should listen. Caddy is only active inside the Docker container. If you want to change the exposed port of the container then you sould change this variable. | +| `PORT` | `3000` | no | The port on which the frontend should listen. | +| `BACKEND_PORT` | `8080` | no | The port on which the backend should listen. | ## Contribute diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go index 82be7bed..93e42960 100644 --- a/backend/internal/bootstrap/router_bootstrap.go +++ b/backend/internal/bootstrap/router_bootstrap.go @@ -35,7 +35,8 @@ func initRouter(db *gorm.DB, appConfigService *service.AppConfigService) { log.Fatalf("Unable to create email service: %s", err) } - auditLogService := service.NewAuditLogService(db, appConfigService, emailService) + geoLiteService := service.NewGeoLiteService() + auditLogService := service.NewAuditLogService(db, appConfigService, emailService, geoLiteService) jwtService := service.NewJwtService(appConfigService) webauthnService := service.NewWebAuthnService(db, jwtService, auditLogService, appConfigService) userService := service.NewUserService(db, jwtService, auditLogService) diff --git a/backend/internal/common/env_config.go b/backend/internal/common/env_config.go index 94f0ce4a..e8a2eb78 100644 --- a/backend/internal/common/env_config.go +++ b/backend/internal/common/env_config.go @@ -14,6 +14,8 @@ type EnvConfigSchema struct { Port string `env:"BACKEND_PORT"` Host string `env:"HOST"` EmailTemplatesPath string `env:"EMAIL_TEMPLATES_PATH"` + MaxMindLicenseKey string `env:"MAXMIND_LICENSE_KEY"` + GeoLiteDBPath string `env:"GEOLITE_DB_PATH"` } var EnvConfig = &EnvConfigSchema{ @@ -24,6 +26,8 @@ var EnvConfig = &EnvConfigSchema{ Port: "8080", Host: "localhost", EmailTemplatesPath: "./email-templates", + MaxMindLicenseKey: "", + GeoLiteDBPath: "data/GeoLite2-City.mmdb", } func init() { diff --git a/backend/internal/service/audit_log_service.go b/backend/internal/service/audit_log_service.go index 09973a2c..695f4088 100644 --- a/backend/internal/service/audit_log_service.go +++ b/backend/internal/service/audit_log_service.go @@ -2,28 +2,27 @@ package service import ( userAgentParser "github.com/mileusna/useragent" - "github.com/oschwald/maxminddb-golang/v2" "github.com/stonith404/pocket-id/backend/internal/model" "github.com/stonith404/pocket-id/backend/internal/utils" "github.com/stonith404/pocket-id/backend/internal/utils/email" "gorm.io/gorm" "log" - "net/netip" ) type AuditLogService struct { db *gorm.DB appConfigService *AppConfigService emailService *EmailService + geoliteService *GeoLiteService } -func NewAuditLogService(db *gorm.DB, appConfigService *AppConfigService, emailService *EmailService) *AuditLogService { - return &AuditLogService{db: db, appConfigService: appConfigService, emailService: emailService} +func NewAuditLogService(db *gorm.DB, appConfigService *AppConfigService, emailService *EmailService, geoliteService *GeoLiteService) *AuditLogService { + return &AuditLogService{db: db, appConfigService: appConfigService, emailService: emailService, geoliteService: geoliteService} } // Create creates a new audit log entry in the database func (s *AuditLogService) Create(event model.AuditLogEvent, ipAddress, userAgent, userID string, data model.AuditLogData) model.AuditLog { - country, city, err := s.GetIpLocation(ipAddress) + country, city, err := s.geoliteService.GetLocationByIP(ipAddress) if err != nil { log.Printf("Failed to get IP location: %v\n", err) } @@ -97,29 +96,3 @@ func (s *AuditLogService) DeviceStringFromUserAgent(userAgent string) string { ua := userAgentParser.Parse(userAgent) return ua.Name + " on " + ua.OS + " " + ua.OSVersion } - -func (s *AuditLogService) GetIpLocation(ipAddress string) (country, city string, err error) { - db, err := maxminddb.Open("GeoLite2-City.mmdb") - if err != nil { - return "", "", err - } - defer db.Close() - - addr := netip.MustParseAddr(ipAddress) - - var record struct { - City struct { - Names map[string]string `maxminddb:"names"` - } `maxminddb:"city"` - Country struct { - Names map[string]string `maxminddb:"names"` - } `maxminddb:"country"` - } - - err = db.Lookup(addr).Decode(&record) - if err != nil { - return "", "", err - } - - return record.Country.Names["en"], record.City.Names["en"], nil -} diff --git a/backend/internal/service/geolite_service.go b/backend/internal/service/geolite_service.go new file mode 100644 index 00000000..d1766527 --- /dev/null +++ b/backend/internal/service/geolite_service.go @@ -0,0 +1,142 @@ +package service + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "github.com/oschwald/maxminddb-golang/v2" + "github.com/stonith404/pocket-id/backend/internal/common" + "io" + "log" + "net/http" + "net/netip" + "os" + "path/filepath" + "time" +) + +type GeoLiteService struct{} + +// NewGeoLiteService initializes a new GeoLiteService instance and starts a goroutine to update the GeoLite2 City database. +func NewGeoLiteService() *GeoLiteService { + service := &GeoLiteService{} + + go func() { + if err := service.updateDatabase(); err != nil { + log.Printf("Failed to update GeoLite2 City database: %v\n", err) + } + }() + + return service +} + +// GetLocationByIP returns the country and city of the given IP address. +func (s *GeoLiteService) GetLocationByIP(ipAddress string) (country, city string, err error) { + db, err := maxminddb.Open(common.EnvConfig.GeoLiteDBPath) + if err != nil { + return "", "", err + } + defer db.Close() + + addr := netip.MustParseAddr(ipAddress) + + var record struct { + City struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"city"` + Country struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + } + + err = db.Lookup(addr).Decode(&record) + if err != nil { + return "", "", err + } + + return record.Country.Names["en"], record.City.Names["en"], nil +} + +// UpdateDatabase checks the age of the database and updates it if it's older than 14 days. +func (s *GeoLiteService) updateDatabase() error { + if s.isDatabaseUpToDate() { + log.Println("GeoLite2 City database is up-to-date.") + return nil + } + + log.Println("Updating GeoLite2 City database...") + + // Download and extract the database + downloadUrl := fmt.Sprintf( + "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz", + common.EnvConfig.MaxMindLicenseKey, + ) + // Download the database tar.gz file + resp, err := http.Get(downloadUrl) + if err != nil { + return fmt.Errorf("failed to download database: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download database, received HTTP %d", resp.StatusCode) + } + + // Extract the database file directly to the target path + if err := s.extractDatabase(resp.Body); err != nil { + return fmt.Errorf("failed to extract database: %w", err) + } + + log.Println("GeoLite2 City database successfully updated.") + return nil +} + +// isDatabaseUpToDate checks if the database file is older than 14 days. +func (s *GeoLiteService) isDatabaseUpToDate() bool { + info, err := os.Stat(common.EnvConfig.GeoLiteDBPath) + if err != nil { + // If the file doesn't exist, treat it as not up-to-date + return false + } + return time.Since(info.ModTime()) < 14*24*time.Hour +} + +// extractDatabase extracts the database file from the tar.gz archive directly to the target location. +func (s *GeoLiteService) extractDatabase(reader io.Reader) error { + gzr, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzr.Close() + + tarReader := tar.NewReader(gzr) + + // Iterate over the files in the tar archive + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read tar archive: %w", err) + } + + // Check if the file is the GeoLite2-City.mmdb file + if header.Typeflag == tar.TypeReg && filepath.Base(header.Name) == "GeoLite2-City.mmdb" { + outFile, err := os.Create(common.EnvConfig.GeoLiteDBPath) + if err != nil { + return fmt.Errorf("failed to create target database file: %w", err) + } + defer outFile.Close() + + // Write the file contents directly to the target location + if _, err := io.Copy(outFile, tarReader); err != nil { + return fmt.Errorf("failed to write database file: %w", err) + } + return nil + } + } + + return errors.New("GeoLite2-City.mmdb not found in archive") +} diff --git a/scripts/download-ip-database.sh b/scripts/download-ip-database.sh deleted file mode 100644 index 19718497..00000000 --- a/scripts/download-ip-database.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Check if the license key environment variable is set -if [ -z "$MAXMIND_LICENSE_KEY" ]; then - echo "Error: MAXMIND_LICENSE_KEY environment variable is not set." - echo "Please set it using 'export MAXMIND_LICENSE_KEY=your_license_key' and try again." - exit 1 -fi -echo $MAXMIND_LICENSE_KEY -# GeoLite2 City Database URL -URL="https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz" - -# Download directory -DOWNLOAD_DIR="./geolite2_db" -TARGET_PATH=./backend/GeoLite2-City.mmdb -mkdir -p $DOWNLOAD_DIR - -# Download the database -echo "Downloading GeoLite2 City database..." -curl -L -o "$DOWNLOAD_DIR/GeoLite2-City.tar.gz" "$URL" - -# Extract the downloaded file -echo "Extracting GeoLite2 City database..." -tar -xzf "$DOWNLOAD_DIR/GeoLite2-City.tar.gz" -C $DOWNLOAD_DIR --strip-components=1 - -mv "$DOWNLOAD_DIR/GeoLite2-City.mmdb" $TARGET_PATH - -# Clean up -rm -rf "$DOWNLOAD_DIR" - -echo "GeoLite2 City database downloaded and extracted to $TARGET_PATH" \ No newline at end of file