add kubernetes-event-exporter

update prometheus port config

kubernetes/apps/jellyfin/app/release.yaml

@@ -25,12 +25,14 @@ spec:
           accessModes:
             - ReadWriteOnce
           volumeName: jellyfin-config
+          storageClassName: local-path
       data:
         enabled: true
         volumeClaimSpec:
           accessModes:
             - ReadWriteOnce
           volumeName: jellyfin-data
+          storageClassName: local-path
     jellyfin:
       mediaVolumes:
         - name: movies

kubernetes/apps/kustomization.yaml

@@ -12,7 +12,6 @@ resources:
   - ./nexus/nexus.yaml
   - ./podinfo/podinfo.yaml
   - ./qbittorrent/qbittorrent.yaml
-  - ./redis/redis.yaml
   - ./snippet-box/snippet-box.yaml
   - ./sonarqube/sonarqube.yaml
   - ./uptime-kuma/uptime-kuma.yaml

kubernetes/apps/redis/app/volume.yaml

@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: redis-master-pvc
-  namespace: redis
-  labels:
-    name: redis-master-pvc
-spec:
-  storageClassName: local-path
-  volumeMode: Filesystem
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 8Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: redis-replica-pvc
-  namespace: redis
-  labels:
-    name: redis-replica-pvc
-spec:
-  storageClassName: local-path
-  volumeMode: Filesystem
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 8Gi

kubernetes/apps/sonarqube/app/release.yaml

@@ -34,7 +34,7 @@ spec:
         elastic: 30091
     persistence:
       enabled: true
-      storageClass: "local-path"
+      storageClass: local-path
       size: "32Gi"
       existingClaim: "sonarqube-pvc"
     postgresql:

kubernetes/infrastructure/consul/app/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul
+  namespace: consul
+  labels:
+    app: consul
+spec:
+  ports:
+    - name: http
+      protocol: TCP
+      port: 8500
+      targetPort: 8500
+  selector:
+    app: consul
+  type: ClusterIP

kubernetes/infrastructure/consul/app/statefulset.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: consul
+  namespace: consul
+  labels:
+    app: consul
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: consul
+  template:
+    metadata:
+      labels:
+        app: consul
+    spec:
+      containers:
+        - name: consul
+          image: 'consul:1.15.4'
+          args:
+            - agent
+          ports:
+            - name: http
+              containerPort: 8500
+              protocol: TCP
+          env:
+            - name: TZ
+              value: Australia/Sydney
+          volumeMounts:
+            - name: consul-data
+              mountPath: /consul/data
+            - name: consul-config
+              mountPath: /consul/config
+          imagePullPolicy: IfNotPresent
+      volumes:
+        - name: consul-data
+          hostPath:
+            path: /mnt/nfs/AppData/consul/data
+            type: Directory
+        - name: consul-config
+          hostPath:
+            path: /mnt/nfs/AppData/consul/config
+            type: Directory
+      restartPolicy: Always
+  serviceName: consul

kubernetes/infrastructure/consul/consul.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: consul
+  namespace: consul
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  path: ./kubernetes/infrastructure/consul/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: flux-system
+  dependsOn:
+    - name: namespaces
+      namespace: flux-system

kubernetes/infrastructure/grafana-dashboards/dashboards/8919-node-exporter-20240520.yaml

@@ -0,0 +1,113 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: flask-consul
+  namespace: consul
+  labels:
+    app: flask-consul
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flask-consul
+  template:
+    metadata:
+      labels:
+        app: flask-consul
+    spec:
+      initContainers:
+        - name: wait-for-consul
+          image: busybox
+          command:
+            - sh
+            - '-c'
+            - >-
+              for i in \$(seq 1 60); do nc -z -w3 consul 8500 && exit 0 ||
+              sleep 5; done; exit 1
+          imagePullPolicy: IfNotPresent
+      containers:
+        - name: flask-consul
+          image: 'edeedeeed/flask_consul:v0.1.0'
+          ports:
+            - name: http-2026
+              containerPort: 2026
+              protocol: TCP
+          env:
+            - name: admin_passwd
+              value: ${dashboard_8919_admin_passwd}
+            - name: consul_token
+              value: ${dashboard_8919_consul_token}
+            - name: consul_url
+              value: 'http://consul:8500/v1'
+            - name: log_level
+              value: INFO
+            - name: TZ
+              value: Australia/Sydney
+          imagePullPolicy: Always
+      restartPolicy: Always
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: flask-consul
+  namespace: consul
+  labels:
+    app: flask-consul
+spec:
+  ports:
+    - name: http-2026
+      protocol: TCP
+      port: 2026
+      targetPort: 2026
+  selector:
+    app: flask-consul
+  type: ClusterIP
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: nginx-consul
+  namespace: consul
+  labels:
+    app: nginx-consul
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx-consul
+  template:
+    metadata:
+      labels:
+        app: nginx-consul
+    spec:
+      containers:
+        - name: nginx-consul
+          image: 'nicholasjackson/nginx-consul:v0.1.0'
+          ports:
+            - name: http-1026
+              containerPort: 1026
+              protocol: TCP
+          env:
+            - name: TZ
+              value: Australia/Sydney
+          imagePullPolicy: Always
+      restartPolicy: Always
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: nginx-consul
+  namespace: consul
+  labels:
+    app: consul
+spec:
+  ports:
+    - name: nginx-consul
+      protocol: TCP
+      port: 1026
+      targetPort: 1026
+      nodePort: 31026
+  selector:
+    app: nginx-consul
+  type: NodePort
+  externalTrafficPolicy: Cluster

kubernetes/infrastructure/grafana-dashboards/grafana-dashboards.yaml

@@ -0,0 +1,48 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: grafana-dashboards-secrets
+  namespace: flux-system
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  targetNamespace: prometheus
+  path: ./grafana-dashboards
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: home-cluster-ops-secrets
+  dependsOn:
+    - name: repositories
+      namespace: flux-system
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: grafana-dashboards
+  namespace: prometheus
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  path: ./kubernetes/infrastructure/grafana-dashboards/dashboards
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: flux-system
+  dependsOn:
+    - name: namespaces
+      namespace: flux-system
+    - name: grafana-dashboards-secrets
+      namespace: flux-system
+  postBuild:
+    substituteFrom:
+      - kind: Secret
+        name: grafana-dashboards-secrets

kubernetes/infrastructure/kustomization.yaml

@@ -3,11 +3,16 @@ kind: Kustomization
 resources:
   - ./cert-manager/cert-manager.yaml
   #  - ./cilium/cilium.yaml
-  - ./minio/minio.yaml
+  - ./consul/consul.yaml
+  - ./grafana-dashboards/grafana-dashboards.yaml
   - ./ingress-nginx/ingress-nginx.yaml
   - ./ingress-nginx/ingress-nginx-config.yaml
+  - ./local-path-provisioner/local-path-provisioner.yaml
+  - ./minio/minio.yaml
   - ./namespaces/namespaces.yaml
   - ./postgresql/postgresql.yaml
   - ./prometheus/prometheus.yaml
   - ./prometheus-alertmanager/prometheus-alertmanager.yaml
+  - ./prometheus-exporters/prometheus-exporters.yaml
+  - ./redis/redis.yaml
   - ./repositories/repositories.yaml

kubernetes/infrastructure/local-path-provisioner/app/release.yaml

@@ -0,0 +1,21 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  interval: 1h
+  driftDetection:
+    mode: enabled
+  chart:
+    spec:
+      chart: local-path-provisioner
+      version: master-head
+      sourceRef:
+        kind: GitRepository
+        namespace: flux-system
+        name: local-path-provisioner
+      interval: 24h
+  values:
+    storageClass:
+      reclaimPolicy: Retain

kubernetes/infrastructure/local-path-provisioner/local-path-provisioner.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/gitrepository_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  path: ./kubernetes/infrastructure/local-path-provisioner/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: flux-system
+  dependsOn:
+    - name: namespaces
+      namespace: flux-system

kubernetes/infrastructure/minio/minio.yaml

@@ -42,6 +42,8 @@ spec:
       namespace: flux-system
     - name: minio-secrets
       namespace: flux-system
+    - name: local-path-provisioner
+      namespace: local-path-storage
   postBuild:
     substituteFrom:
       - kind: Secret

kubernetes/infrastructure/namespaces/namespaces/consul.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: consul

kubernetes/infrastructure/namespaces/namespaces/local-path-storage.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage

kubernetes/infrastructure/postgresql/postgresql.yaml

@@ -42,6 +42,10 @@ spec:
   dependsOn:
     - name: postgresql-secrets
       namespace: flux-system
+    - name: namespaces
+      namespace: flux-system
+    - name: local-path-provisioner
+      namespace: local-path-storage
   postBuild:
     substituteFrom:
       - kind: Secret

kubernetes/infrastructure/prometheus-alertmanager/app/deployment.yaml

@@ -19,7 +19,7 @@ spec:
         runAsGroup: 0
       containers:
         - name: prometheus-alertmanager
-          image: prom/prometheus-alertmanager:v0.27.0
+          image: prom/alertmanager:v0.27.0
           args:
             - "--config.file=/etc/prometheus/prometheus-alertmanager.yaml"
           ports:

kubernetes/infrastructure/prometheus-exporters/exporters/kubernetes-event-exporter.yaml

@@ -0,0 +1,29 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kubernetes-event-exporter
+  namespace: prometheus
+spec:
+  releaseName: kubernetes-event-exporter
+  chart:
+    spec:
+      chart: kubernetes-event-exporter
+      sourceRef:
+        kind: HelmRepository
+        name: bitnami
+  interval: 60m
+  values:
+    config:
+      receivers:
+        - name: "dump"
+          file:
+            layout:
+              message: "{{ .Message }}"
+              reason: "{{ .Reason }}"
+              type: "{{ .Type }}"
+              count: "{{ .Count }}"
+              kind: "{{ .InvolvedObject.Kind }}"
+              name: "{{ .InvolvedObject.Name }}"
+              namespace: "{{ .Namespace }}"
+              component: "{{ .Source.Component }}"
+              host: "{{ .Source.Host }}"

kubernetes/infrastructure/prometheus-exporters/exporters/node-exporter.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-node-exporter
+  namespace: prometheus
+  labels:
+    app: prometheus-node-exporter
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: prometheus-node-exporter
+  template:
+    metadata:
+      labels:
+        app: prometheus-node-exporter
+    spec:
+      hostNetwork: true
+      hostPID: true
+      hostIPC: true
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app: prometheus-node-exporter
+      containers:
+        - name: prometheus-node-exporter
+          image: prom/node-exporter
+          args:
+            - "--path.rootfs=/host"
+          volumeMounts:
+            - name: node-volume
+              mountPath: /host
+              readOnly: true
+              mountPropagation: HostToContainer
+          securityContext:
+            privileged: true
+      volumes:
+        - name: node-volume
+          hostPath:
+            path: /
+            type: Directory

kubernetes/infrastructure/prometheus-exporters/prometheus-exporters.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: prometheus-exporters
+  namespace: prometheus
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  path: ./kubernetes/infrastructure/prometheus-exporters/exporters
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: flux-system
+  dependsOn:
+    - name: namespaces
+      namespace: flux-system

kubernetes/infrastructure/prometheus/app/deployment.yaml

@@ -41,6 +41,21 @@ spec:
               mountPath: /etc/prometheus/
             - name: prometheus-storage-volume
               mountPath: /prometheus/
+        - name: grafana
+          image: grafana/grafana:11.0.1
+          ports:
+            - containerPort: 3000
+          volumeMounts:
+            - name: grafana-config-volume
+              mountPath: /etc/grafana
+            - name: grafana-data-volume
+              mountPath: /var/lib/grafana
+            - name: grafana-logs-volume
+              mountPath: /var/log/grafana
+            - name: grafana-plugins-volume
+              mountPath: /var/lib/grafana/plugins
+            - name: grafana-provisioning-volume
+              mountPath: /etc/grafana/provisioning
       volumes:
         - name: prometheus-config-volume
           hostPath:
@@ -49,4 +64,24 @@ spec:
         - name: prometheus-storage-volume
           hostPath:
             path: /mnt/nfs/AppData/prometheus/storage
+            type: Directory
+        - name: grafana-config-volume
+          hostPath:
+            path: /mnt/nfs/AppData/grafana/config
+            type: Directory
+        - name: grafana-data-volume
+          hostPath:
+            path: /mnt/nfs/AppData/grafana/data
+            type: Directory
+        - name: grafana-logs-volume
+          hostPath:
+            path: /mnt/nfs/AppData/grafana/logs
+            type: Directory
+        - name: grafana-plugins-volume
+          hostPath:
+            path: /mnt/nfs/AppData/grafana/plugins
+            type: Directory
+        - name: grafana-provisioning-volume
+          hostPath:
+            path: /mnt/nfs/AppData/grafana/provisioning
             type: Directory

kubernetes/infrastructure/prometheus/app/ingress.yaml

@@ -1,31 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: prometheus-ingress
-  namespace: prometheus
-  annotations:
-    nginx.ingress.kubernetes.io/ssl-redirect: "false"
-    nginx.ingress.kubernetes.io/use-regex: "true"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: "prometheus.cluster.edward.sydney"
-      http:
-        paths:
-          - pathType: Prefix
-            path: "/"
-            backend:
-              service:
-                name: prometheus
-                port:
-                  number: 9999
-    - host: "prometheus.cluster.local"
-      http:
-        paths:
-          - pathType: Prefix
-            path: "/"
-            backend:
-              service:
-                name: prometheus
-                port:
-                  number: 9999

kubernetes/infrastructure/prometheus/app/service.yaml

@@ -9,9 +9,14 @@ metadata:
 spec:
   selector:
     app: prometheus
-  type: ClusterIP
+  type: NodePort
   ports:
     - port: 9999
       targetPort: 9090
+      nodePort: 30999
       protocol: TCP
-      name: http
+      name: http
+    - name: grafana-port
+      port: 3030
+      nodePort: 30303
+      targetPort: 3000

kubernetes/infrastructure/redis/app/release.yaml

@@ -21,6 +21,7 @@ spec:
       password: ${password}
     master:
       persistence:
+        storageClass: local-path
         existingClaim: redis-master-pvc
         selector:
           matchLabels:
@@ -30,6 +31,7 @@ spec:
     replica:
       replicaCount: 1
       persistence:
+        storageClass: local-path
         existingClaim: redis-replica-pvc
         selector:
           matchLabels:

kubernetes/infrastructure/redis/app/volume.yaml

@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: redis-master-pv
+  namespace: redis
+  labels:
+    type: local
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  capacity:
+    storage: 8Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  local:
+    path: "/mnt/nfs/AppData/redis/master"
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: redis-master-pvc
+    namespace: redis
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - rpi5-cluster-node-3
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: redis-master-pvc
+  namespace: redis
+  labels:
+    name: redis-master-pvc
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 8Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: redis-replica-pv
+  namespace: redis
+  labels:
+    type: local
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  capacity:
+    storage: 8Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  local:
+    path: "/mnt/nfs/AppData/redis/replica"
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: redis-replica-pvc
+    namespace: redis
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - rpi5-cluster-node-3
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: redis-replica-pvc
+  namespace: redis
+  labels:
+    name: redis-replica-pvc
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 8Gi

kubernetes/infrastructure/redis/redis.yaml

@@ -31,7 +31,7 @@ spec:
   interval: 10m
   timeout: 1m30s
   retryInterval: 30s
-  path: ./kubernetes/apps/redis/app
+  path: ./kubernetes/infrastructure/redis/app
   prune: true
   sourceRef:
     kind: GitRepository

kubernetes/infrastructure/repositories/repos/local-path-provisioner.yaml

@@ -0,0 +1,13 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: local-path-provisioner
+  namespace: flux-system
+spec:
+  interval: 5m
+  ref:
+    branch: main
+  secretRef:
+    name: flux-system
+  timeout: 60s
+  url: https://github.com/rancher/local-path-provisioner.git