rework Loki

loki/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+generators:
+  - ./secret-generator.yaml
+resources:
+  - ./namespace.yaml

loki/secret-generator.yaml

@@ -0,0 +1,10 @@
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: secret-generator
+  annotations:
+    config.kubernetes.io/function: |
+      exec:
+        path: ksops
+files:
+  - ./secret.yaml

loki/secret.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    namespace: logs
+    name: loki-bucket-secret
+type: Opaque
+stringData:
+    S3_LOKI_ACCESS_KEY_ID: ENC[AES256_GCM,data:EK9PR0EZO6gwCFAB/DRg1OuAm10=,iv:kk8xWafJlJpkgty8m2klv/EMSHZvxm/rYDlkUG0Hoh4=,tag:tvRhZ3Han4Bh02f0wZx9sw==,type:str]
+    S3_LOKI_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:OyQ5QTQz0Bznki15bkOfMm9StPfXGRbJyyAnVPnif+MixMss5Ugvuw==,iv:s4B4HQBKR2hV6GIWD6Vvfx4TkijVdchV0nP4YDf+FvY=,tag:QrlLcr2rqpMMVo+GHzG/aw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1mraede6gqxkh2rkeq5fjrcflp7emenl2qn885asxvtx5erga2pdqujuexz
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoeDNSblZkQ3BndWNEcXdl
+            Q3l1ZkU5NEV3UTkvY2dmK2xWK1RLRFVUOFM0CkFWV1RodkZHbGtEVm1CUmlkRGZK
+            YTNhT0hCVERSRUdmQVdVOVlPYVc2bG8KLS0tIDdVdmpiT1hxZExadUdMcWl5Mkdq
+            ZjFFWHh2bWNBSE5MeEJ1S00zOWpGU3cKPOjdYqv1dKlpfrE4CwyjTM27W/O2ax9k
+            XHVBDtg9E9g4d5E1Mf3o+bfWl96wVteVr/W5ZaGE/WRSqIa+nBC4rA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1s9nvc4rxj3kaj4apmzzn8fmjrudrvdhgu70rg04we9hyse5aadsq7kmckn
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1TXVpRlFTYVRQR3Nid1Ir
+            ODRuTFVVQkFpSCtIZzR3YkRPR05vbituOFRBCllQVkdja1RHaXl5c2VERlV4SjE5
+            UWt1RkNlRHF6WFJwMk9qZU1XZEwwTjgKLS0tIHBWUSs5a0VWNHkzakNzZmdmUlkw
+            Rk9FSTFMYnA1V1pyakxDSlVzVzhKS3cKSFJ0H4Z5Q8FzrpRWUUQgpBj+nSdNT9yR
+            ZeF3+hZZCQwIaMTxq8RXq6zyUtQsLRXUnPl+59NmPmNKaPkKi8X9tA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1z5wtjmk0jw0j9qz9k5rrnp30nzqxrl3v6wgl7eryvqus28zekp4qpx9jc2
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwZUZMYXZHWXpYN0ZYNE5J
+            bmIxWWhGcko2enJBdnpjdTFUY3RwMnZIaFc0Cmo1NHZVMzFxK2Y3K3UxWHB3UnQr
+            UmV6bzl2Rk1xMVF6NXM4MmdMeWVKZnMKLS0tIFBEZG5RTE4zNXNJYTlxd2VxRW5p
+            YklZWHhseUp6d1AwRHp6QnRBL3JWZ3MKO19rYxmCZDro3x83aM5oHI26TpITblCU
+            X40297QCc2pWWfYSfGa3H3IUZMFe5STvPGoKUhRWZ+XMqYb6c2kp/A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-10-03T12:15:16Z"
+    mac: ENC[AES256_GCM,data:59efYmiDE6RzTR6H7qd9ujfmRXmW7NNQVdX4viU5OWMHTGCN0Yf7rDFbPLSlagUFbAImGZgcSZ6hD9qwk4Xl4PIoVwQ6P9nGky6tI4ljEjQPvzceSSeMvhDyvrrDryr+qs2E6aeLCJF4uHXa7u/XhgvxsN6teFeFLg9SjJzjkEY=,iv:/bQd9wn+5FuPcF9HsgeZuzpzLusaJXYGQn8Pj5VHlWU=,tag:BxMtpEEG2OvGZA+n/OdJeA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.0

loki/values.yaml

@@ -1,14 +1,7 @@
-minio:
-  enabled: true
-  replicas: 3
-  drivesPerNode: 2
-  persistence:
-    size: 15Gi
-    storageClass: longhorn-local
 loki:
   auth_enabled: false
   persistence:
-    size: 30Gi
+    size: 10Gi
   schemaConfig:
     configs:
       - from: 2024-04-14
@@ -19,20 +12,86 @@ loki:
           prefix: index_
           period: 24h
   limits_config:
-    retention_period: 168h
+    retention_period: 24h
   compactor:
     compaction_interval: 15m
     retention_enabled: true
     retention_delete_delay: 1h
     retention_delete_worker_count: 150
     delete_request_store: s3
+  storage:
+    bucketNames:
+      chunks: yolokube-logs
+    s3:
+      endpoint: https://fsn1.your-objectstorage.com
+      s3: s3://yolokube-logs
+      s3forcepathstyle: true
+      accessKeyId: ${S3_LOKI_ACCESS_KEY_ID}
+      secretAccessKey: ${S3_LOKI_SECRET_ACCESS_KEY}
 monitoring:
-  selfMonitoring:
+  dashboards:
-    grafanaAgent:
+    enabled: true
-      installOperator: false
+  serviceMonitor:
+    enabled: true
+read:
+  extraArgs:
+    - '-config.expand-env=true'
+  extraEnv:
+    - name: S3_LOKI_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_ACCESS_KEY_ID
+    - name: S3_LOKI_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_SECRET_ACCESS_KEY
 write:
+  extraArgs:
+    - '-config.expand-env=true'
+  extraEnv:
+    - name: S3_LOKI_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_ACCESS_KEY_ID
+    - name: S3_LOKI_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_SECRET_ACCESS_KEY
   persistence:
     storageClass: longhorn-local
 backend:
+  extraArgs:
+    - '-config.expand-env=true'
+  extraEnv:
+    - name: S3_LOKI_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_ACCESS_KEY_ID
+    - name: S3_LOKI_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_SECRET_ACCESS_KEY
+  persistence:
+    storageClass: longhorn-local
+compactor:
+  extraArgs:
+    - '-config.expand-env=true'
+  extraEnv:
+    - name: S3_LOKI_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_ACCESS_KEY_ID
+    - name: S3_LOKI_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-bucket-secret
+          key: S3_LOKI_SECRET_ACCESS_KEY
   persistence:
     storageClass: longhorn-local