Merge pull request #420 from 3dwardch3ng/infra/logstash

add infra app logstash for omada syslog

kubernetes/infrastructure/logstash/app/ingress.yaml

@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: logstash-ingress
+  namespace: logstash
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: "omada.logstash.cluster.edward.sydney"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: logstash
+                port:
+                  number: 8008
+    - host: "monitor.omada.logstash.cluster.edward.sydney"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: logstash
+                port:
+                  number: 9600

kubernetes/infrastructure/logstash/app/pipelines.yaml

@@ -11,12 +11,29 @@ data:
       path.config: "/opt/bitnami/logstash/config/misc.conf"
   omada.conf: |
     input {
-      file {
-        path => "/tmp/omada"
+      tcp {
+        port => 1514
+        type => syslog
+      }
+      udp {
+        port => 1514
+        type => syslog
+      }
+      http {
+        port => 8008
       }
     }
     output {
-      stdout { }
+      s3 { 
+        access_key_id => ${omada_s3_access_key_id}
+        bucket => "logstash"
+        canned_acl => "bucket-owner-read"
+        endpoint => "http://minio.minio.svc.cluster.local:19000"
+        prefix => "omada/%{+YYYY}/%{+MM}/%{+dd}"
+        region: "ap-southeast-2"
+        secret_access_key => ${omada_s3_secret_access_key}
+        time_file => 60
+      }
     }
   misc.conf: |
     input {

kubernetes/infrastructure/logstash/app/release.yaml

@@ -0,0 +1,67 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: logstash
+  namespace: logstash
+spec:
+  releaseName: logstash
+  chart:
+    spec:
+      chart: logstash
+      sourceRef:
+        kind: HelmRepository
+        name: bitnami
+        namespace: flux-system
+  interval: 1h
+  install:
+    remediation:
+      retries: 3
+  values:
+    existingConfiguration: logstash-pipelines
+    serviceAccount:
+      name: logstash
+    containerPorts:
+      - name: omada-http
+        containerPort: 8008
+        protocol: TCP
+      - name: monitoring
+        containerPort: 9600
+        protocol: TCP
+      - name: omada-syslog-udp
+        containerPort: 1514
+        protocol: UDP
+      - name: omada-syslog-tcp
+        containerPort: 1514
+        protocol: TCP
+    podSecurityContext:
+      fsGroup: 1000
+    containerSecurityContext:
+      runAsUser: 1000
+      runAsGroup: 1000
+    service:
+      type: LoadBalancer
+      ports:
+        - name: omada-http
+          port: 8080
+          targetPort: http
+          protocol: TCP
+        - name: monitoring
+          port: 9600
+          targetPort: monitoring
+          protocol: TCP
+        - name: omada-syslog-udp
+          port: 1514
+          targetPort: syslog-udp
+          protocol: UDP
+        - name: omada-syslog-tcp
+          port: 1514
+          targetPort: syslog-tcp
+          protocol: TCP
+    persistence:
+      enabled: true
+      existingClaim: logstash-pvc
+      size: 16Gi
+    volumePermissions:
+      enabled: true
+      securityContext:
+        runAsUser: 1000

kubernetes/infrastructure/logstash/app/volume.yaml

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: logstash-pv
+  namespace: logstash
+  labels:
+    type: local
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  capacity:
+    storage: 16Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  local:
+    path: "/mnt/nfs/AppData/logstash/data"
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: logstash-pvc
+    namespace: logstash
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - rpi5-cluster-node-1
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: logstash-pvc
+  namespace: logstash
+  labels:
+    name: logstash-pvc
+spec:
+  storageClassName: local-path
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 16Gi

kubernetes/infrastructure/logstash/logstash.yaml

@@ -0,0 +1,50 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: logstash-secrets
+  namespace: flux-system
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  targetNamespace: logstash
+  path: ./logstash
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: home-cluster-ops-secrets
+  dependsOn:
+    - name: repositories
+      namespace: flux-system
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: logstash
+  namespace: logstash
+spec:
+  interval: 10m
+  timeout: 1m30s
+  retryInterval: 30s
+  path: ./kubernetes/infrastructure/logstash/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    namespace: flux-system
+    name: flux-system
+  dependsOn:
+    - name: namespaces
+      namespace: flux-system
+    - name: logstash-secrets
+      namespace: flux-system
+    - name: local-path-provisioner
+      namespace: local-path-storage
+  postBuild:
+    substituteFrom:
+      - kind: Secret
+        name: logstash-secrets