diff --git a/kubernetes/infrastructure/logstash/app/ingress.yaml b/kubernetes/infrastructure/logstash/app/ingress.yaml new file mode 100644 index 0000000..6b68287 --- /dev/null +++ b/kubernetes/infrastructure/logstash/app/ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: logstash-ingress + namespace: logstash + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: "omada.logstash.cluster.edward.sydney" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: logstash + port: + number: 8008 + - host: "monitor.omada.logstash.cluster.edward.sydney" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: logstash + port: + number: 9600 \ No newline at end of file diff --git a/kubernetes/infrastructure/logstash/app/pipelines.yaml b/kubernetes/infrastructure/logstash/app/pipelines.yaml index 4a951ea..efcdac9 100644 --- a/kubernetes/infrastructure/logstash/app/pipelines.yaml +++ b/kubernetes/infrastructure/logstash/app/pipelines.yaml @@ -11,12 +11,29 @@ data: path.config: "/opt/bitnami/logstash/config/misc.conf" omada.conf: | input { - file { - path => "/tmp/omada" + tcp { + port => 1514 + type => syslog + } + udp { + port => 1514 + type => syslog + } + http { + port => 8008 } } output { - stdout { } + s3 { + access_key_id => ${omada_s3_access_key_id} + bucket => "logstash" + canned_acl => "bucket-owner-read" + endpoint => "http://minio.minio.svc.cluster.local:19000" + prefix => "omada/%{+YYYY}/%{+MM}/%{+dd}" + region: "ap-southeast-2" + secret_access_key => ${omada_s3_secret_access_key} + time_file => 60 + } } misc.conf: | input { diff --git a/kubernetes/infrastructure/logstash/app/release.yaml b/kubernetes/infrastructure/logstash/app/release.yaml new file mode 100644 index 0000000..99ed6e1 --- /dev/null +++ b/kubernetes/infrastructure/logstash/app/release.yaml @@ -0,0 +1,67 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: logstash + namespace: logstash +spec: + releaseName: logstash + chart: + spec: + chart: logstash + sourceRef: + kind: HelmRepository + name: bitnami + namespace: flux-system + interval: 1h + install: + remediation: + retries: 3 + values: + existingConfiguration: logstash-pipelines + serviceAccount: + name: logstash + containerPorts: + - name: omada-http + containerPort: 8008 + protocol: TCP + - name: monitoring + containerPort: 9600 + protocol: TCP + - name: omada-syslog-udp + containerPort: 1514 + protocol: UDP + - name: omada-syslog-tcp + containerPort: 1514 + protocol: TCP + podSecurityContext: + fsGroup: 1000 + containerSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + service: + type: LoadBalancer + ports: + - name: omada-http + port: 8080 + targetPort: http + protocol: TCP + - name: monitoring + port: 9600 + targetPort: monitoring + protocol: TCP + - name: omada-syslog-udp + port: 1514 + targetPort: syslog-udp + protocol: UDP + - name: omada-syslog-tcp + port: 1514 + targetPort: syslog-tcp + protocol: TCP + persistence: + enabled: true + existingClaim: logstash-pvc + size: 16Gi + volumePermissions: + enabled: true + securityContext: + runAsUser: 1000 \ No newline at end of file diff --git a/kubernetes/infrastructure/logstash/app/volume.yaml b/kubernetes/infrastructure/logstash/app/volume.yaml new file mode 100644 index 0000000..7500a61 --- /dev/null +++ b/kubernetes/infrastructure/logstash/app/volume.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: logstash-pv + namespace: logstash + labels: + type: local +spec: + storageClassName: local-path + volumeMode: Filesystem + capacity: + storage: 16Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + local: + path: "/mnt/nfs/AppData/logstash/data" + claimRef: + apiVersion: v1 + kind: PersistentVolumeClaim + name: logstash-pvc + namespace: logstash + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - rpi5-cluster-node-1 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: logstash-pvc + namespace: logstash + labels: + name: logstash-pvc +spec: + storageClassName: local-path + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 16Gi \ No newline at end of file diff --git a/kubernetes/infrastructure/logstash/logstash.yaml b/kubernetes/infrastructure/logstash/logstash.yaml new file mode 100644 index 0000000..b916d5f --- /dev/null +++ b/kubernetes/infrastructure/logstash/logstash.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: logstash-secrets + namespace: flux-system +spec: + interval: 10m + timeout: 1m30s + retryInterval: 30s + targetNamespace: logstash + path: ./logstash + prune: true + sourceRef: + kind: GitRepository + namespace: flux-system + name: home-cluster-ops-secrets + dependsOn: + - name: repositories + namespace: flux-system + decryption: + provider: sops + secretRef: + name: sops-age +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: logstash + namespace: logstash +spec: + interval: 10m + timeout: 1m30s + retryInterval: 30s + path: ./kubernetes/infrastructure/logstash/app + prune: true + sourceRef: + kind: GitRepository + namespace: flux-system + name: flux-system + dependsOn: + - name: namespaces + namespace: flux-system + - name: logstash-secrets + namespace: flux-system + - name: local-path-provisioner + namespace: local-path-storage + postBuild: + substituteFrom: + - kind: Secret + name: logstash-secrets \ No newline at end of file