Prometheus Operator

$ kubectl get node
NAME                                                STATUS   ROLES    AGE   VERSION
ip-10-xxx-xxx-xxx.ap-northeast-2.compute.internal   Ready    <none>   8d    v1.28.1-eks-43840fb
ip-10-xxx-xxx-xx.ap-northeast-2.compute.internal    Ready    <none>   8d    v1.28.1-eks-43840fb
ip-10-xxx-xxx-xx.ap-northeast-2.compute.internal    Ready    <none>   8d    v1.28.1-eks-43840fb

# Download kube-prometheus-stack chart
git clone https://github.com/prometheus-community/helm-charts.git
cd ./helm-charts/kube-prometheus-stack/

#  Install the kube-prometheus-stack chart on your EKS cluster
helm upgrade \
  --install \
  --create-namespace monitoring \
  -f values.yaml \
  -n monitoring \
  kube-prometheus-stack . \
  --wait

$ helm list -n monitoring
NAME                   NAMESPACE   REVISION  UPDATED                               STATUS    CHART                         APP VERSION
kube-prometheus-stack  monitoring  1         2023-09-27 14:58:32.286992 +0900 KST  deployed  kube-prometheus-stack-51.2.0  v0.68.0

$ kubectl get pod \
    -n monitoring \
    -l app=kube-prometheus-stack-operator

NAME                                              READY   STATUS    RESTARTS   AGE
kube-prometheus-stack-operator-8449959549-865vh   1/1     Running   0          8d

kubectl api-resources --api-group monitoring.coreos.com

NAME                  SHORTNAMES   APIVERSION                       NAMESPACED   KIND
alertmanagerconfigs   amcfg        monitoring.coreos.com/v1alpha1   true         AlertmanagerConfig
alertmanagers         am           monitoring.coreos.com/v1         true         Alertmanager
podmonitors           pmon         monitoring.coreos.com/v1         true         PodMonitor
probes                prb          monitoring.coreos.com/v1         true         Probe
prometheusagents      promagent    monitoring.coreos.com/v1alpha1   true         PrometheusAgent
prometheuses          prom         monitoring.coreos.com/v1         true         Prometheus
prometheusrules       promrule     monitoring.coreos.com/v1         true         PrometheusRule
scrapeconfigs         scfg         monitoring.coreos.com/v1alpha1   true         ScrapeConfig
servicemonitors       smon         monitoring.coreos.com/v1         true         ServiceMonitor
thanosrulers          ruler        monitoring.coreos.com/v1         true         ThanosRuler

# values.yaml for kube-prometheus-stack
## Install Prometheus Operator CRDs
##
crds:
  enabled: true

$ kubectl get prometheus \
    -n monitoring \
    -o wide

NAME                               VERSION   DESIRED   READY   RECONCILED   AVAILABLE   AGE   PAUSED
kube-prometheus-stack-prometheus   v2.47.0   1         1       True         True        10d   false

$ kubectl get statefulset \
    -n monitoring \
    -o wide

NAME                                              READY   AGE    CONTAINERS                     IMAGES
alertmanager-kube-prometheus-stack-alertmanager   1/1     9d     alertmanager,config-reloader   quay.io/prometheus/alertmanager:v0.26.0,quay.io/prometheus-operator/prometheus-config-reloader:v0.68.0
prometheus-kube-prometheus-stack-prometheus       1/1     7d7h   prometheus,config-reloader     quay.io/prometheus/prometheus:v2.47.0,quay.io/prometheus-operator/prometheus-config-reloader:v0.68.0

  # valeus.yaml for kube-prometheus-stack
  prometheusOperator:
    ...
    prometheusSpec:
-     serviceMonitorSelectorNilUsesHelmValues: true
+     serviceMonitorSelectorNilUsesHelmValues: false

helm upgrade \
  kube-prometheus-stack . \
  -n monitoring \
  -f values.yaml \
  --wait

$ helm list -n actions-runner-system
NAME                       NAMESPACE              REVISION  UPDATED                               STATUS    CHART                             APP VERSION
actions-runner-controller  actions-runner-system  26        2023-10-04 20:07:28.715411 +0900 KST  deployed  actions-runner-controller-0.23.3  0.27.4

$ kubectl api-resources --api-group actions.summerwind.dev
NAME                          SHORTNAMES   APIVERSION                        NAMESPACED   KIND
horizontalrunnerautoscalers   hra          actions.summerwind.dev/v1alpha1   true         HorizontalRunnerAutoscaler
runnerdeployments             rdeploy      actions.summerwind.dev/v1alpha1   true         RunnerDeployment
runnerreplicasets             rrs          actions.summerwind.dev/v1alpha1   true         RunnerReplicaSet
runners                                    actions.summerwind.dev/v1alpha1   true         Runner
runnersets                                 actions.summerwind.dev/v1alpha1   true         RunnerSet

$ kubectl get pod -n actions-runner
NAME                                READY   STATUS    RESTARTS   AGE
github-actions-runner-ph5xr-2sp9k   2/2     Running   0          3h32m
github-actions-runner-ph5xr-z4r7n   2/2     Running   0          3h32m

$ kubectl get rdeploy -n actions-runner
NAME                    ENTERPRISE   ORGANIZATION   REPOSITORY   GROUP   LABELS                                                                DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
github-actions-runner   spacex                                           ["support-horizontal-runner-autoscaling","ubuntu-22.04","v2.307.1"]   2         2         2            2           62d

# values.yaml for actions-runner-controller
...
# Metrics service resource
metrics:
  serviceAnnotations: {}
- serviceMonitor: false
+ serviceMonitor: true
  serviceMonitorLabels: {}
- port: 8443
+ port: 8080
  proxy:
-   enabled: true
+   enabled: false  
    image:
      repository: quay.io/brancz/kube-rbac-proxy
      tag: v0.13.1
...

helm upgrade \
  --namespace actions-runner-system \
  --create-namespace \
  actions-runner-controller . \
  -f values.yaml \
  --wait

kubectl get svc,servicemonitor \
  -n actions-runner-system \
  -o wide

NAME                                                TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE   SELECTOR
service/actions-runner-controller-metrics-service   ClusterIP   xxx.xx.xxx.212   <none>        8080/TCP   99d   app.kubernetes.io/instance=actions-runner-controller,app.kubernetes.io/name=actions-runner-controller
service/actions-runner-controller-webhook           ClusterIP   xxx.xx.xx.83     <none>        443/TCP    99d   app.kubernetes.io/instance=actions-runner-controller,app.kubernetes.io/name=actions-runner-controller

NAME                                                                             AGE
servicemonitor.monitoring.coreos.com/actions-runner-controller-service-monitor   100m

kubectl logs \
  -f prometheus-kube-prometheus-stack-prometheus-0 \
  -n monitoring \
  | grep discovery

...
ts=2023-10-04T12:40:42.618Z caller=kubernetes.go:329 level=info component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/actions-runner-system/actions-runner-controller-service-monitor/0 msg="Using pod service account via in-cluster config"

...
scrape_configs:
- job_name: serviceMonitor/actions-runner-system/actions-runner-controller-service-monitor/0
  honor_labels: false
  kubernetes_sd_configs:
  - role: endpoints
    namespaces:
      names:
      - actions-runner-system
  metrics_path: /metrics
  relabel_configs:
...

$ kubectl get svc -n monitoring kube-prometheus-stack-prometheus
NAME                               TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
kube-prometheus-stack-prometheus   ClusterIP   xxx.xx.xxx.131   <none>        9090/TCP,8080/TCP   10d

kubectl port-forward service/kube-prometheus-stack-prometheus \
  9090:9090 \
  -n monitoring

open http://localhost:9090

# Chart.yaml for kube-prometheus-stack chart
dependencies:
  - name: crds
    version: "0.0.0"
    condition: crds.enabled
  - name: kube-state-metrics
    version: "5.12.*"
    repository: https://prometheus-community.github.io/helm-charts
    condition: kubeStateMetrics.enabled
  - name: prometheus-node-exporter
    version: "4.23.*"
    repository: https://prometheus-community.github.io/helm-charts
    condition: nodeExporter.enabled
  - name: grafana
    version: "6.59.*"
    repository: https://grafana.github.io/helm-charts
    condition: grafana.enabled
  - name: prometheus-windows-exporter
    repository: https://prometheus-community.github.io/helm-charts
    version: "0.1.*"
    condition: windowsMonitoring.enabled

# values.yaml for kube-prometheus-stack chart
grafana:
  ...
  persistence:
    enabled: true
    storageClassName: gp3
    size: 2Gi
    accessModes:
      - ReadWriteOnce

$ kubectl get pv -n monitoring
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                                                                                     STORAGECLASS   REASON   AGE
pvc-2a2df390-7d05-47cf-87fc-b36cb53c69f4   2Gi        RWO            Delete           Bound    monitoring/kube-prometheus-stack-grafana                                                                  gp3                     152m
pvc-5c13c09e-3844-43f0-ba75-aa27f8e980d2   200Gi      RWO            Delete           Bound    monitoring/prometheus-kube-prometheus-stack-prometheus-db-prometheus-kube-prometheus-stack-prometheus-0   gp3                     12d

$ kubectl exec -it -n monitoring kube-prometheus-stack-grafana-6d7bc8d4d4-zzspn -- df -h /var/lib/grafana
Filesystem                Size      Used Available Use% Mounted on
/dev/nvme1n1              1.9G      2.2M      1.9G   0% /var/lib/grafana

# values.yaml for kube-prometheus-stack
## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml
##
grafana:
  enabled: true
  namespaceOverride: ""

  ...

  ## Timezone for the default dashboards
  ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg
  ##
  defaultDashboardsTimezone: Asia/Seoul

  adminPassword: example-password

# values for kube-prometheus-stack
...
grafana:
  # assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret
  # values are not exposed in the rendered grafana.ini configmap. It is enabled by default.
  #
  # To pass values into grafana.ini without exposing them in a configmap, use variable expansion:
  # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion
  #
  # Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap,
  # you can disable this check by setting assertNoLeakedSecrets to false.
  assertNoLeakedSecrets: false
  ...

helm upgrade \
  kube-prometheus-stack . \
  --install \
  --create-namespace \
  --namespace monitoring \
  --values values.yaml \
  --wait

Error: UPGRADE FAILED: execution error at (kube-prometheus-stack/charts/grafana/templates/statefulset.yaml:28:28): Sensitive key 'auth.google.client_secret' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets

  {{- if $.Values.assertNoLeakedSecrets -}}
      {{- $grafanaIni := index .Values "grafana.ini" -}}
      {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}}
        {{- $currentMap := $grafanaIni -}}
        {{- $shouldContinue := true -}}
        {{- range $index, $elem := $secret.path -}}
          {{- if and $shouldContinue (hasKey $currentMap $elem) -}}
            {{- if eq (len $secret.path) (add1 $index) -}}
              {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}}
                {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}}
              {{- end -}}
            {{- else -}}
              {{- $currentMap = index $currentMap $elem -}}
            {{- end -}}
          {{- else -}}
              {{- $shouldContinue = false -}}
          {{- end -}}
        {{- end -}}
      {{- end -}}
  {{- end -}}
{{- end -}}

kubectl create secret generic kube-prometheus-stack-grafana-oauth \
  --from-literal GF_AUTH_GOOGLE_CLIENT_ID="<REDACTED>" \
  --from-literal GF_AUTH_GOOGLE_CLIENT_SECRET="<REDACTED>" \
  --namespace monitoring

secret/kube-prometheus-stack-grafana-oauth created

# values.yaml for kube-promethue-stack
...
## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml
##
grafana:
  envFromSecret: kube-prometheus-stack-grafana-oauth

  grafana.ini:
    security:
      cookie_secure: false
    dashboard:
      min_refresh_interal: 60s
    auth.google:
      enabled: true
      allow_sign_up: true
      auto_login: false
      # Variable expansion is only availble in Grafana 7.1+
      # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion
      client_id: $__env{GF_AUTH_GOOGLE_CLIENT_ID}
      client_secret: $__env{GF_AUTH_GOOGLE_CLIENT_SECRET}
      scopes: openid email profile
      auth_url: https://accounts.google.com/o/oauth2/v2/auth
      token_url: https://oauth2.googleapis.com/token
      api_url: https://openidconnect.googleapis.com/v1/userinfo

      client_id: $__env{GF_AUTH_GOOGLE_CLIENT_ID}

helm upgrade \
  kube-prometheus-stack . \
  --install \
  --create-namespace \
  --namespace monitoring \
  --values values.yaml \
  --wait

Release "kube-prometheus-stack" has been upgraded. Happy Helming!
NAME: kube-prometheus-stack
LAST DEPLOYED: Thu Feb 15 15:43:27 2024
NAMESPACE: monitoring
STATUS: deployed
REVISION: 9
NOTES:
kube-prometheus-stack has been installed. Check its status by running:
  kubectl --namespace monitoring get pods -l "release=kube-prometheus-stack"

Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator.

$ kubectl exec -it -n monitoring kube-prometheus-stack-grafana-0 -- env | grep GF_AUTH_GOOGLE_CLIENT_
GF_AUTH_GOOGLE_CLIENT_ID=<REDACTED>-<REDACTED>.apps.googleusercontent.com
GF_AUTH_GOOGLE_CLIENT_SECRET=<REDACTED>