jarvis-home/internal/documents/extract.go

package documents

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"net/mail"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

var tags = regexp.MustCompile(`(?s)<[^>]*>`)

func Extract(name string, data []byte) (string, error) {
	ext := strings.ToLower(filepath.Ext(name))
	switch ext {
	case ".txt", ".md", ".markdown", ".csv", ".json", ".yaml", ".yml", ".xml", ".ics", ".log":
		return cleanText(string(data)), nil
	case ".html", ".htm":
		return cleanText(tags.ReplaceAllString(string(data), " ")), nil
	case ".docx":
		return extractDOCX(data)
	case ".pdf":
		return externalExtract(name, data, "pdftotext", func(path string) *exec.Cmd {
			return exec.Command("pdftotext", "-layout", path, "-")
		})
	case ".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff":
		return externalExtract(name, data, "tesseract", func(path string) *exec.Cmd {
			return exec.Command("tesseract", path, "stdout", "-l", "deu+eng")
		})
	case ".eml":
		msg, err := mail.ReadMessage(bytes.NewReader(data))
		if err != nil {
			return "", err
		}
		body, err := io.ReadAll(io.LimitReader(msg.Body, 4<<20))
		if err != nil {
			return "", err
		}
		return cleanText("Subject: " + msg.Header.Get("Subject") + "\nFrom: " + msg.Header.Get("From") + "\n\n" + string(body)), nil
	default:
		return "", fmt.Errorf("Dateityp %q wird noch nicht direkt unterstützt", ext)
	}
}

func extractDOCX(data []byte) (string, error) {
	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return "", err
	}
	for _, f := range zr.File {
		if f.Name != "word/document.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			return "", err
		}
		defer rc.Close()
		dec := xml.NewDecoder(rc)
		var b strings.Builder
		for {
			tok, err := dec.Token()
			if err == io.EOF {
				break
			}
			if err != nil {
				return "", err
			}
			switch t := tok.(type) {
			case xml.CharData:
				b.WriteString(string(t))
			case xml.EndElement:
				if t.Name.Local == "p" || t.Name.Local == "tr" {
					b.WriteByte('\n')
				}
			}
		}
		return cleanText(b.String()), nil
	}
	return "", fmt.Errorf("word/document.xml fehlt")
}

func externalExtract(name string, data []byte, tool string, build func(string) *exec.Cmd) (string, error) {
	if _, err := exec.LookPath(tool); err != nil {
		return "", fmt.Errorf("%s ist nicht installiert; für %s bitte lokal installieren", tool, filepath.Ext(name))
	}
	f, err := os.CreateTemp("", "homehub-*"+filepath.Ext(name))
	if err != nil {
		return "", err
	}
	path := f.Name()
	defer os.Remove(path)
	if _, err := f.Write(data); err != nil {
		f.Close()
		return "", err
	}
	if err := f.Close(); err != nil {
		return "", err
	}
	out, err := build(path).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%s: %w: %s", tool, err, string(out))
	}
	return cleanText(string(out)), nil
}

func cleanText(s string) string {
	s = strings.ReplaceAll(s, "\x00", "")
	lines := strings.Split(s, "\n")
	out := make([]string, 0, len(lines))
	blank := false
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			if !blank {
				out = append(out, "")
			}
			blank = true
			continue
		}
		blank = false
		out = append(out, line)
	}
	return strings.TrimSpace(strings.Join(out, "\n"))
}