Add HostProcess Container Configuration for k8s

Co-authored-by: Brian Redmond <brianisrunning@gmail.com>
Signed-off-by: Brian Redmond <brianisrunning@gmail.com>
Signed-off-by: James Sturtevant <jstur@microsoft.com>

.github/workflows/ci.yml

@@ -11,6 +11,10 @@ on:
       - published
       - edited
 
+permissions:
+  contents: read
+  packages: write
+  
 jobs:
   test:
     runs-on: windows-2019
@@ -34,7 +38,7 @@ jobs:
         run: make e2e-test
 
   lint:
-    runs-on: windows-2019
+    runs-on: windows-2022
     steps:
       # `gofmt` linter run by golangci-lint fails on CRLF line endings (the default for Windows)
       - name: Set git to use LF
@@ -73,7 +77,7 @@ jobs:
           ignore_words_list: calle
 
   build:
-    runs-on: windows-2019
+    runs-on: windows-2022
     needs:
       - test
       - lint
@@ -90,6 +94,7 @@ jobs:
 
       - name: Install Build deps
         run: |
+          dotnet tool install --global GitVersion.Tool --version 5.*
           go get github.com/prometheus/promu@v0.11.1
           go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.2.0
           # GOPATH\bin dir must be added to PATH else the `promu` and `goversioninfo` commands won't be found
@@ -99,13 +104,14 @@ jobs:
         run: |
           $ErrorActionPreference = "Stop"
 
-          gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru
+          dotnet-gitversion /output json /showvariable FullSemVer | Set-Content VERSION -PassThru
           $Version = Get-Content VERSION
           # Windows versioninfo resources need the file version by parts (but product version is free text)
           $VersionParts = ($Version -replace '^v?([0-9\.]+).*$','$1').Split(".")
           goversioninfo.exe -ver-major $VersionParts[0] -ver-minor $VersionParts[1] -ver-patch $VersionParts[2] -product-version $Version -platform-specific
 
           make crossbuild
+          make build-all
           # GH requires all files to have different names, so add version/arch to differentiate
           foreach($Arch in "amd64","386") {
             Move-Item output\$Arch\windows_exporter.exe output\windows_exporter-$Version-$Arch.exe
@@ -133,6 +139,21 @@ jobs:
 
           promu checksum output\
 
+      - name: Login to GitHub container registry
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push Latest image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          VERSION=latest make push-all
+
       - name: Release
         if: startsWith(github.ref, 'refs/tags/')
         env:
@@ -140,3 +161,4 @@ jobs:
         run: |
           $TagName = $env:GITHUB_REF -replace 'refs/tags/', ''
           Get-ChildItem -Path output\* -Include @('windows_exporter*.msi', 'windows_exporter*.exe', 'sha256sums.txt') | Foreach-Object {gh release upload $TagName $_}
+          make push-all

Dockerfile

@@ -0,0 +1,9 @@
+# Note this image doesn't really matter for hostprocess but it is good to build per OS version
+# the files in the image are copied to $env:CONTAINER_SANDBOX_MOUNT_POINT on the host
+# but the file system is the Host NOT the container
+ARG BASE="mcr.microsoft.com/windows/nanoserver:1809"
+FROM $BASE
+
+ENV PATH="C:\Windows\system32;C:\Windows;"
+COPY output/amd64/windows_exporter.exe /windows_exporter.exe
+ENTRYPOINT ["windows_exporter.exe"]

Makefile

@@ -1,4 +1,15 @@
 export GOOS=windows
+export DOCKER_IMAGE_NAME ?= windows-exporter
+export DOCKER_REPO ?= ghcr.io/prometheus-community
+
+VERSION?=$(shell cat VERSION)
+DOCKER?=docker
+
+# Image Variables for Hostprocess Container
+# Windows image build is heavily influenced by https://github.com/kubernetes/kubernetes/blob/master/cluster/images/etcd/Makefile
+OS=1809
+ALL_OS:= 1809 ltsc2022
+BASE_IMAGE=mcr.microsoft.com/windows/nanoserver
 
 .PHONY: build
 build: windows_exporter.exe
@@ -26,3 +37,23 @@ crossbuild:
 	# on Windows, so for now, we'll just build twice
 	GOARCH=amd64 promu build --prefix=output/amd64
 	GOARCH=386   promu build --prefix=output/386
+
+build-image: crossbuild
+	$(DOCKER) build --build-arg=BASE=$(BASE_IMAGE):$(OS) -f Dockerfile -t $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$(OS) .
+
+sub-build-%:
+	$(MAKE) OS=$* build-image
+
+build-all: $(addprefix sub-build-,$(ALL_OS))
+
+push:
+	set -x; \
+	for osversion in ${ALL_OS}; do \
+		$(DOCKER) push $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
+		$(DOCKER) manifest create --amend $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION) $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
+		full_version=`$(DOCKER) manifest inspect $(BASE_IMAGE):$${osversion} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'` || true; \
+		$(DOCKER) manifest annotate --os windows --arch amd64 --os-version $${full_version} $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)  $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)-$${osversion}; \
+	done
+	$(DOCKER) manifest push --purge $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(VERSION)
+
+push-all: build-all push

README.md

@@ -115,6 +115,11 @@ On some older versions of Windows you may need to surround parameter values with
 msiexec /i C:\Users\Administrator\Downloads\windows_exporter.msi ENABLED_COLLECTORS="ad,iis,logon,memory,process,tcp,thermalzone" TEXTFILE_DIR="C:\custom_metrics\"
 ```
 
+
+## Kubernetes Implementation
+
+See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernetes.md).
+
 ## Supported versions
 
 windows_exporter supports Windows Server versions 2008R2 and later, and desktop Windows version 7 and later.

exporter.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"os/user"
 	"sort"
 	"strconv"
 	"strings"
@@ -345,6 +346,16 @@ func main() {
 		log.Fatalf("Couldn't load collectors: %s", err)
 	}
 
+	u, err := user.Current()
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	log.Infof("Running as %v", u.Username)
+	if strings.Contains(u.Username, "ContainerAdministrator") || strings.Contains(u.Username, "ContainerUser") {
+		log.Warnf("Running as a preconfigured Windows Container user. This may mean you do not have Windows HostProcess containers configured correctly and some functionality will not work as expected.")
+	}
+
 	log.Infof("Enabled collectors: %v", strings.Join(keys(collectors), ", "))
 
 	h := &metricsHandler{

kubernetes/kubernetes.md

@@ -0,0 +1,91 @@
+# windows_exporter on Kubernetes
+
+With Kubernetes supporting HostProcess containers on Windows nodes (as of [v1.22](https://kubernetes.io/blog/2021/08/16/windows-hostprocess-containers/), it is useful to run the `windows_exporter` as a container on Windows to export metrics for your Prometheus implementation.  Read the [Kubernetes HostProcess documentation](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) for more information.
+
+Requirements:
+
+- Kubernetes 1.22+
+- containerd 1.6 Beta+
+- WindowsHostProcessContainers feature-gate turned on for `kube-apiserver` and `kubelet`
+
+> IMPORTANT: This does not work unless you are specifically targeting Host Process Containers with Containerd (Docker doesn't have support).  The image will build but will **not** be able to access the host.
+
+## Container Image
+
+The image is multi arch image (WS 2019, WS 2022) built on Windows. To build the images:
+
+```
+DOCKER_REPO=<your repo> make push-all
+```
+
+If you don't have a version of `make` on your Windows machine, You can use WSL to build the image with Windows Containers by creating a symbolic link to the docker cli and then override the docker command in the `Makefile`: 
+
+On Windows:
+```
+Item -ItemType SymbolicLink -Path "c:\docker" -Target "C:\Program Files\Docker\Docker\resources\bin\docker.exe"
+
+In WSL:
+```
+DOCKER_REPO=<your repo> DOCKER=/mnt/c/docker make push-all 
+```
+
+## Kubernetes Quick Start
+
+Before beginning you need to deploy the [prometheus operator](https://github.com/prometheus-operator/prometheus-operator) to your cluster. As a quick start, you can use a project like https://github.com/prometheus-operator/kube-prometheus. The export itself doesn't have any dependency on prometheus operator and the exporter image can be used in manual configurations.
+
+### Windows Exporter DaemonSet
+
+This create a deployment on every node. A config map is created for to handle the configuration of the Windows exporter with [configuration file](../README.md#using-a-configuration-file).  Adjust the configuration file for the collectors you are interested in.
+
+```bash
+kubectl apply -f kubernetes/windows-exporter-daemonset.yaml
+```
+
+> Note: This example manifest deploys the latest bleeding edge image `ghcr.io/prometheus-community/windows-exporter:latest` built from the main branch.  You should update this to use a released version which you can find at https://github.com/prometheus-community/windows_exporter/releases
+
+#### Configuring the firewall
+The firewall on the node needs to be configured  to allow connections on the node: `New-NetFirewallRule -DisplayName 'windows-exporter' -Direction inbound -Profile Any -Action Allow -LocalPort 9182 -Protocol TCP` 
+
+You could do this by adding an init container but if you remove the deployment at a later date you will need to remove the firewall rule manually. The following could be added to the `windows-exporter-daemonset.yaml`:
+
+```
+apiVersion: apps/v1
+kind: DaemonSet
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: configure-firewall
+          image: mcr.microsoft.com/windows/nanoserver:1809
+          command: ["powershell"]
+          args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"]
+```
+
+### Prometheus PodMonitor
+
+Create the [Pod Monitor](https://prometheus-operator.dev/docs/operator/design/#podmonitor) to configure the scraping:
+
+```bash
+kubectl apply -f windows-exporter-podmonitor.yaml
+```
+
+### View Metrics
+
+Open Prometheus with 
+
+```
+kubectl --namespace monitoring port-forward svc/prometheus-k8s 9091:9090
+```
+
+Navigate to prometheus UI and add a query to see node cpu (replacing with your ip address)
+
+```
+sum by (mode) (irate(windows_cpu_time_total{instance="10.1.0.5:9182"}[5m]))
+```
+
+![windows cpu total time graph in prometheus ui](https://user-images.githubusercontent.com/648372/140547130-b535c766-6479-47d3-b2d3-cd8a551647df.png)
+
+
+## Configuring TLS
+
+It is possible to configure TLS of the solution using `--web.config.file`.  Read more at https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md

kubernetes/windows-exporter-daemonset.yaml

@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: windows-exporter
+  name: windows-exporter
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app: windows-exporter
+  template:
+    metadata:
+      labels:
+        app: windows-exporter
+    spec:
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\system"
+      hostNetwork: true
+      initContainers:
+        - name: configure-firewall
+          image: mcr.microsoft.com/windows/nanoserver:1809
+          command: ["powershell"]
+          args: ["New-NetFirewallRule", "-DisplayName", "'windows-exporter'", "-Direction", "inbound", "-Profile", "Any", "-Action", "Allow", "-LocalPort", "9182", "-Protocol", "TCP"]
+      containers:
+      - args: 
+        - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml
+        name: windows-exporter
+        image: ghcr.io/prometheus-community/windows-exporter:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9182
+          hostPort: 9182
+          name: http
+        volumeMounts:
+        - name:  windows-exporter-config
+          mountPath: /config.yml
+          subPath: config.yml
+      nodeSelector:
+        kubernetes.io/os: windows
+      volumes:
+      - name: windows-exporter-config
+        configMap:
+          name: windows-exporter-config
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: windows-exporter-config
+  namespace: monitoring
+  labels:
+    app: windows-exporter
+data:
+  config.yml: |
+    collectors:
+      enabled: '[defaults],container'
+    collector:
+      service:
+        services-where: "Name='containerd' or Name='kubelet'"

kubernetes/windows-exporter-podmonitor.yaml

@@ -0,0 +1,15 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    app: windows-exporter
+  name: windows-exporter
+  namespace: monitoring
+spec:
+  jobLabel: windows-exporter
+  selector:
+    matchLabels:
+      app: windows-exporter
+  podMetricsEndpoints:
+  - port: http
+    scheme: http