From b6f00d0140d3d1f0a45da258c6eba121826bdbfc Mon Sep 17 00:00:00 2001 From: Edward Cheng Date: Wed, 26 Jun 2024 14:35:22 +1000 Subject: [PATCH] add infra app prometheus --- kubernetes/infrastructure/kustomization.yaml | 1 + .../namespaces/namespaces/prometheus.yaml | 4 + .../prometheus/app/deployment.yaml | 47 ++++++ .../prometheus/app/ingress.yaml | 31 ++++ .../prometheus/app/prometheus.yaml | 137 ++++++++++++++++++ .../infrastructure/prometheus/app/role.yaml | 33 +++++ .../prometheus/app/service.yaml | 16 ++ .../infrastructure/prometheus/prometheus.yaml | 18 +++ 8 files changed, 287 insertions(+) create mode 100644 kubernetes/infrastructure/namespaces/namespaces/prometheus.yaml create mode 100644 kubernetes/infrastructure/prometheus/app/deployment.yaml create mode 100644 kubernetes/infrastructure/prometheus/app/ingress.yaml create mode 100644 kubernetes/infrastructure/prometheus/app/prometheus.yaml create mode 100644 kubernetes/infrastructure/prometheus/app/role.yaml create mode 100644 kubernetes/infrastructure/prometheus/app/service.yaml create mode 100644 kubernetes/infrastructure/prometheus/prometheus.yaml diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 8fe7257..460ca2e 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -8,4 +8,5 @@ resources: - ./ingress-nginx/ingress-nginx-config.yaml - ./namespaces/namespaces.yaml - ./postgresql/postgresql.yaml + - ./prometheus/prometheus.yaml - ./repositories/repositories.yaml \ No newline at end of file diff --git a/kubernetes/infrastructure/namespaces/namespaces/prometheus.yaml b/kubernetes/infrastructure/namespaces/namespaces/prometheus.yaml new file mode 100644 index 0000000..ea5d16f --- /dev/null +++ b/kubernetes/infrastructure/namespaces/namespaces/prometheus.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: prometheus diff --git a/kubernetes/infrastructure/prometheus/app/deployment.yaml b/kubernetes/infrastructure/prometheus/app/deployment.yaml new file mode 100644 index 0000000..5cbb4b4 --- /dev/null +++ b/kubernetes/infrastructure/prometheus/app/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: prometheus + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--storage.tsdb.retention.time=12h" + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9999 + resources: + requests: + cpu: 500m + memory: 500M + limits: + cpu: 1 + memory: 1Gi + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + - name: prometheus-storage-volume + hostPath: + path: /mnt/nfs/AppData/prometheus + type: Directory \ No newline at end of file diff --git a/kubernetes/infrastructure/prometheus/app/ingress.yaml b/kubernetes/infrastructure/prometheus/app/ingress.yaml new file mode 100644 index 0000000..bbc5d82 --- /dev/null +++ b/kubernetes/infrastructure/prometheus/app/ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: capacitor-ingress + namespace: capacitor + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: "prometheus.cluster.edward.sydney" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: prometheus + port: + number: 30999 + - host: "prometheus.cluster.local" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: prometheus + port: + number: 30999 \ No newline at end of file diff --git a/kubernetes/infrastructure/prometheus/app/prometheus.yaml b/kubernetes/infrastructure/prometheus/app/prometheus.yaml new file mode 100644 index 0000000..a294edc --- /dev/null +++ b/kubernetes/infrastructure/prometheus/app/prometheus.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + name: prometheus-server-conf + namespace: prometheus +data: + prometheus.rules: |- + groups: + - name: rpi5 cluster alert + rules: + - alert: High Pod Memory + expr: sum(container_memory_usage_bytes) > 1 + for: 1m + labels: + severity: slack + annotations: + summary: High Memory Usage + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" + scrape_configs: + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_endpoints_name] + regex: 'node-exporter' + action: keep + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name \ No newline at end of file diff --git a/kubernetes/infrastructure/prometheus/app/role.yaml b/kubernetes/infrastructure/prometheus/app/role.yaml new file mode 100644 index 0000000..f914d54 --- /dev/null +++ b/kubernetes/infrastructure/prometheus/app/role.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: prometheus \ No newline at end of file diff --git a/kubernetes/infrastructure/prometheus/app/service.yaml b/kubernetes/infrastructure/prometheus/app/service.yaml new file mode 100644 index 0000000..e87e722 --- /dev/null +++ b/kubernetes/infrastructure/prometheus/app/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: prometheus + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9999' +spec: + selector: + app: prometheus + type: NodePort + ports: + - port: 9999 + targetPort: 9999 + nodePort: 30999 \ No newline at end of file diff --git a/kubernetes/infrastructure/prometheus/prometheus.yaml b/kubernetes/infrastructure/prometheus/prometheus.yaml new file mode 100644 index 0000000..9fc4a95 --- /dev/null +++ b/kubernetes/infrastructure/prometheus/prometheus.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: prometheus + namespace: prometheus +spec: + interval: 10m + timeout: 1m30s + retryInterval: 30s + path: ./kubernetes/infrastructure/prometheus/app + prune: true + sourceRef: + kind: GitRepository + namespace: flux-system + name: flux-system + dependsOn: + - name: namespaces + namespace: flux-system