[client] Feature/upload bundle (#3734)

Add an upload bundle option with the flag --upload-bundle; by default, the upload will use a NetBird address, which can be replaced using the flag --upload-bundle-url.

The upload server is available under the /upload-server path. The release change will push a docker image to netbirdio/upload image repository.

The server supports using s3 with pre-signed URL for direct upload and local file for storing bundles.

client/server/debug.go

@@ -4,16 +4,24 @@ package server
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/client/internal/debug"
 	"github.com/netbirdio/netbird/client/proto"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/upload-server/types"
 )
 
+const maxBundleUploadSize = 50 * 1024 * 1024
+
 // DebugBundle creates a debug bundle and returns the location.
 func (s *Server) DebugBundle(_ context.Context, req *proto.DebugBundleRequest) (resp *proto.DebugBundleResponse, err error) {
 	s.mutex.Lock()
@@ -42,7 +50,102 @@ func (s *Server) DebugBundle(_ context.Context, req *proto.DebugBundleRequest) (
 		return nil, fmt.Errorf("generate debug bundle: %w", err)
 	}
 
-	return &proto.DebugBundleResponse{Path: path}, nil
+	if req.GetUploadURL() == "" {
+
+		return &proto.DebugBundleResponse{Path: path}, nil
+	}
+	key, err := uploadDebugBundle(context.Background(), req.GetUploadURL(), s.config.ManagementURL.String(), path)
+	if err != nil {
+		return &proto.DebugBundleResponse{Path: path, UploadFailureReason: err.Error()}, nil
+	}
+
+	return &proto.DebugBundleResponse{Path: path, UploadedKey: key}, nil
+}
+
+func uploadDebugBundle(ctx context.Context, url, managementURL, filePath string) (key string, err error) {
+	response, err := getUploadURL(ctx, url, managementURL)
+	if err != nil {
+		return "", err
+	}
+
+	err = upload(ctx, filePath, response)
+	if err != nil {
+		return "", err
+	}
+	return response.Key, nil
+}
+
+func upload(ctx context.Context, filePath string, response *types.GetURLResponse) error {
+	fileData, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("open file: %w", err)
+	}
+
+	defer fileData.Close()
+
+	stat, err := fileData.Stat()
+	if err != nil {
+		return fmt.Errorf("stat file: %w", err)
+	}
+
+	if stat.Size() > maxBundleUploadSize {
+		return fmt.Errorf("file size exceeds maximum limit of %d bytes", maxBundleUploadSize)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "PUT", response.URL, fileData)
+	if err != nil {
+		return fmt.Errorf("create PUT request: %w", err)
+	}
+
+	req.ContentLength = stat.Size()
+	req.Header.Set("Content-Type", "application/octet-stream")
+
+	putResp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("upload failed: %v", err)
+	}
+	defer putResp.Body.Close()
+
+	if putResp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(putResp.Body)
+		return fmt.Errorf("upload status %d: %s", putResp.StatusCode, string(body))
+	}
+	return nil
+}
+
+func getUploadURL(ctx context.Context, url string, managementURL string) (*types.GetURLResponse, error) {
+	id := getURLHash(managementURL)
+	getReq, err := http.NewRequestWithContext(ctx, "GET", url+"?id="+id, nil)
+	if err != nil {
+		return nil, fmt.Errorf("create GET request: %w", err)
+	}
+
+	getReq.Header.Set(types.ClientHeader, types.ClientHeaderValue)
+
+	resp, err := http.DefaultClient.Do(getReq)
+	if err != nil {
+		return nil, fmt.Errorf("get presigned URL: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("get presigned URL status %d: %s", resp.StatusCode, string(body))
+	}
+
+	urlBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response body: %w", err)
+	}
+	var response types.GetURLResponse
+	if err := json.Unmarshal(urlBytes, &response); err != nil {
+		return nil, fmt.Errorf("unmarshal response: %w", err)
+	}
+	return &response, nil
+}
+
+func getURLHash(url string) string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(url)))
 }
 
 // GetLogLevel gets the current logging level for the server.

client/server/debug_test.go

@@ -0,0 +1,49 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/upload-server/server"
+	"github.com/netbirdio/netbird/upload-server/types"
+)
+
+func TestUpload(t *testing.T) {
+	if os.Getenv("DOCKER_CI") == "true" {
+		t.Skip("Skipping upload test on docker ci")
+	}
+	testDir := t.TempDir()
+	testURL := "http://localhost:8080"
+	t.Setenv("SERVER_URL", testURL)
+	t.Setenv("STORE_DIR", testDir)
+	srv := server.NewServer()
+	go func() {
+		if err := srv.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			t.Errorf("Failed to start server: %v", err)
+		}
+	}()
+	t.Cleanup(func() {
+		if err := srv.Stop(); err != nil {
+			t.Errorf("Failed to stop server: %v", err)
+		}
+	})
+
+	file := filepath.Join(t.TempDir(), "tmpfile")
+	fileContent := []byte("test file content")
+	err := os.WriteFile(file, fileContent, 0640)
+	require.NoError(t, err)
+	key, err := uploadDebugBundle(context.Background(), testURL+types.GetURLPath, testURL, file)
+	require.NoError(t, err)
+	id := getURLHash(testURL)
+	require.Contains(t, key, id+"/")
+	expectedFilePath := filepath.Join(testDir, key)
+	createdFileContent, err := os.ReadFile(expectedFilePath)
+	require.NoError(t, err)
+	require.Equal(t, fileContent, createdFileContent)
+}