From ee1c895a229d73187c6e46b1ecc1c695e5f8d60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Thu, 8 Oct 2026 17:44:34 +0200 Subject: [PATCH] textfile: replace utfbom with a focused UTF-8 BOM helper (#2552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- go.mod | 1 - go.sum | 2 - internal/collector/textfile/textfile.go | 30 ++- internal/collector/textfile/textfile_test.go | 68 ++++--- internal/collector/textfile/utf8bom.go | 49 +++++ internal/collector/textfile/utf8bom_test.go | 195 +++++++++++++++++++ 6 files changed, 300 insertions(+), 45 deletions(-) create mode 100644 internal/collector/textfile/utf8bom.go create mode 100644 internal/collector/textfile/utf8bom_test.go diff --git a/go.mod b/go.mod index 359b6597..c592e9ce 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.27.1 require ( github.com/alecthomas/kingpin/v2 v2.4.0 github.com/bmatcuk/doublestar/v4 v4.10.2 - github.com/dimchansky/utfbom v1.1.1 github.com/go-ole/go-ole v1.3.1-0.20250305162226-6867ec158e36 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 diff --git a/go.sum b/go.sum index c74fc0e6..e5470f87 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,6 @@ github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/go-ole/go-ole v1.3.1-0.20250305162226-6867ec158e36 h1:PTRdMI7SqYj0FIO+xrw1gWgjB3ygEh1ME45PB0T+npQ= github.com/go-ole/go-ole v1.3.1-0.20250305162226-6867ec158e36/go.mod h1:2ip94f67up8I6Po4OnI0rjMeULNtsGHkQOLwhRZl5qk= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= diff --git a/internal/collector/textfile/textfile.go b/internal/collector/textfile/textfile.go index 4d4760a1..ad6785ea 100644 --- a/internal/collector/textfile/textfile.go +++ b/internal/collector/textfile/textfile.go @@ -18,6 +18,7 @@ package textfile import ( + "bufio" "errors" "fmt" "io" @@ -31,7 +32,6 @@ import ( "time" "github.com/alecthomas/kingpin/v2" - "github.com/dimchansky/utfbom" "github.com/prometheus-community/windows_exporter/internal/mi" "github.com/prometheus-community/windows_exporter/internal/types" "github.com/prometheus/client_golang/prometheus" @@ -393,22 +393,22 @@ func scrapeFile(path string, logger *slog.Logger) ([]*dto.MetricFamily, error) { return nil, err } + defer func() { + if err := file.Close(); err != nil { + logger.Warn("error closing file "+path, + slog.Any("err", err), + ) + } + }() + parser := expfmt.NewTextParser(model.UTF8Validation) - r, encoding := utfbom.Skip(carriageReturnFilteringReader{r: file}) - if err = checkBOM(encoding); err != nil { + r := bufio.NewReader(carriageReturnFilteringReader{r: file}) + if err := skipUTF8BOM(r); err != nil { return nil, err } parsedFamilies, err := parser.TextToMetricFamilies(r) - - closeErr := file.Close() - if closeErr != nil { - logger.Warn("error closing file "+path, - slog.Any("err", closeErr), - ) - } - if err != nil { return nil, err } @@ -439,14 +439,6 @@ func scrapeFile(path string, logger *slog.Logger) ([]*dto.MetricFamily, error) { return families_array, nil } -func checkBOM(encoding utfbom.Encoding) error { - if encoding == utfbom.Unknown || encoding == utfbom.UTF8 { - return nil - } - - return errors.New(encoding.String()) -} - func getDefaultPath() string { execPath, _ := os.Executable() diff --git a/internal/collector/textfile/textfile_test.go b/internal/collector/textfile/textfile_test.go index dc671c2a..f7832239 100644 --- a/internal/collector/textfile/textfile_test.go +++ b/internal/collector/textfile/textfile_test.go @@ -19,10 +19,12 @@ package textfile import ( "io" + "log/slog" + "os" + "path/filepath" "strings" "testing" - "github.com/dimchansky/utfbom" dto "github.com/prometheus/client_model/go" ) @@ -42,33 +44,53 @@ func TestCRFilter(t *testing.T) { } } -func TestCheckBOM(t *testing.T) { +func TestScrapeFileBOM(t *testing.T) { t.Parallel() - testdata := []struct { - encoding utfbom.Encoding - err string + for _, tc := range []struct { + name string + prefix string + err string }{ - {utfbom.Unknown, ""}, - {utfbom.UTF8, ""}, - {utfbom.UTF16BigEndian, "UTF16BigEndian"}, - {utfbom.UTF16LittleEndian, "UTF16LittleEndian"}, - {utfbom.UTF32BigEndian, "UTF32BigEndian"}, - {utfbom.UTF32LittleEndian, "UTF32LittleEndian"}, - } - for _, d := range testdata { - err := checkBOM(d.encoding) - if d.err == "" && err != nil { - t.Error(err) - } + {name: "plain UTF-8"}, + {name: "UTF-8 BOM", prefix: "\xef\xbb\xbf"}, + {name: "UTF-16 BE", prefix: "\xfe\xff", err: "UTF16BigEndian"}, + {name: "UTF-16 LE", prefix: "\xff\xfe", err: "UTF16LittleEndian"}, + {name: "UTF-32 BE", prefix: "\x00\x00\xfe\xff", err: "UTF32BigEndian"}, + {name: "UTF-32 LE", prefix: "\xff\xfe\x00\x00", err: "UTF32LittleEndian"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - if d.err != "" && err == nil { - t.Errorf("Missing expected error %s", d.err) - } + path := filepath.Join(t.TempDir(), "metric.prom") + content := tc.prefix + "# TYPE metric gauge\r\nmetric 1\r\n" - if err != nil && !strings.Contains(err.Error(), d.err) { - t.Error(err) - } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + + families, err := scrapeFile(path, slog.New(slog.DiscardHandler)) + if tc.err != "" { + if err == nil || err.Error() != tc.err { + t.Fatalf("got error %v, want %s", err, tc.err) + } + + return + } + + if err != nil { + t.Fatal(err) + } + + if len(families) != 1 || families[0].GetName() != "metric" { + t.Fatalf("unexpected metric families: %v", families) + } + + metrics := families[0].GetMetric() + if len(metrics) != 1 || metrics[0].GetGauge().GetValue() != 1 { + t.Errorf("unexpected metrics: %v", metrics) + } + }) } } diff --git a/internal/collector/textfile/utf8bom.go b/internal/collector/textfile/utf8bom.go new file mode 100644 index 00000000..14b2762a --- /dev/null +++ b/internal/collector/textfile/utf8bom.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textfile + +import ( + "bufio" + "bytes" + "errors" + "io" +) + +// skipUTF8BOM removes one leading UTF-8 BOM and rejects UTF-16/32 BOMs. +// The buffered reader is also reused by the metrics parser. +func skipUTF8BOM(r *bufio.Reader) error { + prefix, err := r.Peek(4) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + + switch { + case bytes.HasPrefix(prefix, []byte("\x00\x00\xfe\xff")): + return errors.New("UTF32BigEndian") + case bytes.HasPrefix(prefix, []byte("\xff\xfe\x00\x00")): + return errors.New("UTF32LittleEndian") + case bytes.HasPrefix(prefix, []byte("\xfe\xff")): + return errors.New("UTF16BigEndian") + case bytes.HasPrefix(prefix, []byte("\xff\xfe")): + return errors.New("UTF16LittleEndian") + case bytes.HasPrefix(prefix, []byte("\xef\xbb\xbf")): + _, err = r.Discard(3) + + return err + default: + return nil + } +} diff --git a/internal/collector/textfile/utf8bom_test.go b/internal/collector/textfile/utf8bom_test.go new file mode 100644 index 00000000..72f0ed97 --- /dev/null +++ b/internal/collector/textfile/utf8bom_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textfile + +import ( + "bufio" + "errors" + "io" + "strings" + "testing" + "testing/iotest" + + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" +) + +func TestSkipUTF8BOM(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + input string + want string + err string + }{ + {name: "empty"}, + {name: "one byte", input: "a", want: "a"}, + {name: "two bytes", input: "ab", want: "ab"}, + {name: "three bytes", input: "abc", want: "abc"}, + {name: "plain metrics", input: "metric 1\n", want: "metric 1\n"}, + {name: "UTF-8", input: "\xef\xbb\xbfmetric 1\n", want: "metric 1\n"}, + {name: "UTF-8 BOM only", input: "\xef\xbb\xbf"}, + {name: "one partial BOM byte", input: "\xef", want: "\xef"}, + {name: "two partial BOM bytes", input: "\xef\xbb", want: "\xef\xbb"}, + {name: "mismatched BOM", input: "\xef\xbbx", want: "\xef\xbbx"}, + {name: "embedded BOM", input: "a\xef\xbb\xbf", want: "a\xef\xbb\xbf"}, + {name: "repeated BOM", input: "\xef\xbb\xbf\xef\xbb\xbf", want: "\xef\xbb\xbf"}, + {name: "UTF-16 BE", input: "\xfe\xff", err: "UTF16BigEndian"}, + {name: "UTF-16 LE", input: "\xff\xfe", err: "UTF16LittleEndian"}, + {name: "UTF-16 BE with data", input: "\xfe\xff\x00a", err: "UTF16BigEndian"}, + {name: "UTF-16 LE with data", input: "\xff\xfea\x00", err: "UTF16LittleEndian"}, + {name: "UTF-32 BE", input: "\x00\x00\xfe\xff", err: "UTF32BigEndian"}, + {name: "UTF-32 LE", input: "\xff\xfe\x00\x00", err: "UTF32LittleEndian"}, + {name: "partial UTF-32 BE", input: "\x00\x00\xfe", want: "\x00\x00\xfe"}, + {name: "partial UTF-32 LE", input: "\xff\xfe\x00", err: "UTF16LittleEndian"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + for _, fragmented := range []bool{false, true} { + var input io.Reader = strings.NewReader(tc.input) + if fragmented { + input = iotest.OneByteReader(input) + } + + r := bufio.NewReader(input) + + err := skipUTF8BOM(r) + if tc.err != "" { + if err == nil || err.Error() != tc.err { + t.Fatalf("fragmented=%t: got error %v, want %s", fragmented, err, tc.err) + } + + continue + } + + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(got) != tc.want { + t.Errorf("fragmented=%t: got %q, want %q", fragmented, got, tc.want) + } + } + }) + } +} + +func TestSkipUTF8BOMReadError(t *testing.T) { + t.Parallel() + + readErr := errors.New("read failed") + + for _, tc := range []struct { + name string + input string + want string + }{ + {name: "before prefix"}, + {name: "partial prefix", input: "\xef\xbb"}, + {name: "after plain prefix", input: "metric 1\n", want: "metric 1\n"}, + {name: "after UTF-8 prefix", input: "\xef\xbb\xbfmetric 1\n", want: "metric 1\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + input := io.MultiReader(strings.NewReader(tc.input), iotest.ErrReader(readErr)) + r := bufio.NewReader(input) + + err := skipUTF8BOM(r) + if len(tc.input) < 4 { + if !errors.Is(err, readErr) { + t.Fatalf("got error %v, want %v", err, readErr) + } + + return + } + + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(r) + if !errors.Is(err, readErr) { + t.Errorf("got error %v, want %v", err, readErr) + } + + if string(got) != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func BenchmarkSkipUTF8BOM(b *testing.B) { + for _, tc := range []struct { + name string + input string + }{ + {name: "plain", input: "metric 1\n"}, + {name: "UTF-8 BOM", input: "\xef\xbb\xbfmetric 1\n"}, + } { + b.Run(tc.name, func(b *testing.B) { + input := strings.NewReader(tc.input) + r := bufio.NewReader(input) + + b.ReportAllocs() + + for b.Loop() { + input.Reset(tc.input) + r.Reset(input) + + if err := skipUTF8BOM(r); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkBOMParser(b *testing.B) { + for _, tc := range []struct { + name string + prefix string + }{ + {name: "plain"}, + {name: "UTF-8 BOM", prefix: "\xef\xbb\xbf"}, + } { + b.Run(tc.name, func(b *testing.B) { + input := tc.prefix + "# HELP metric Example metric.\n# TYPE metric gauge\nmetric{label=\"value\"} 1\n" + + b.ReportAllocs() + + for b.Loop() { + r := bufio.NewReader(strings.NewReader(input)) + if err := skipUTF8BOM(r); err != nil { + b.Fatal(err) + } + + parser := expfmt.NewTextParser(model.UTF8Validation) + if _, err := parser.TextToMetricFamilies(r); err != nil { + b.Fatal(err) + } + } + }) + } +}