From da23145624ee99820e94290d97da113bce454e4a Mon Sep 17 00:00:00 2001 From: Aaron Riedel Date: Thu, 3 Oct 2024 14:16:28 +0200 Subject: [PATCH] rework Loki --- loki/kustomization.yaml | 6 +++ loki/secret-generator.yaml | 10 +++++ loki/secret.yaml | 47 ++++++++++++++++++++++ loki/values.yaml | 81 ++++++++++++++++++++++++++++++++------ 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 loki/kustomization.yaml create mode 100644 loki/secret-generator.yaml create mode 100644 loki/secret.yaml diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml new file mode 100644 index 0000000..c04c4da --- /dev/null +++ b/loki/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +generators: + - ./secret-generator.yaml +resources: + - ./namespace.yaml diff --git a/loki/secret-generator.yaml b/loki/secret-generator.yaml new file mode 100644 index 0000000..486c4c1 --- /dev/null +++ b/loki/secret-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: viaduct.ai/v1 +kind: ksops +metadata: + name: secret-generator + annotations: + config.kubernetes.io/function: | + exec: + path: ksops +files: + - ./secret.yaml diff --git a/loki/secret.yaml b/loki/secret.yaml new file mode 100644 index 0000000..8ea6ad1 --- /dev/null +++ b/loki/secret.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Secret +metadata: + namespace: logs + name: loki-bucket-secret +type: Opaque +stringData: + S3_LOKI_ACCESS_KEY_ID: ENC[AES256_GCM,data:EK9PR0EZO6gwCFAB/DRg1OuAm10=,iv:kk8xWafJlJpkgty8m2klv/EMSHZvxm/rYDlkUG0Hoh4=,tag:tvRhZ3Han4Bh02f0wZx9sw==,type:str] + S3_LOKI_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:OyQ5QTQz0Bznki15bkOfMm9StPfXGRbJyyAnVPnif+MixMss5Ugvuw==,iv:s4B4HQBKR2hV6GIWD6Vvfx4TkijVdchV0nP4YDf+FvY=,tag:QrlLcr2rqpMMVo+GHzG/aw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1mraede6gqxkh2rkeq5fjrcflp7emenl2qn885asxvtx5erga2pdqujuexz + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoeDNSblZkQ3BndWNEcXdl + Q3l1ZkU5NEV3UTkvY2dmK2xWK1RLRFVUOFM0CkFWV1RodkZHbGtEVm1CUmlkRGZK + YTNhT0hCVERSRUdmQVdVOVlPYVc2bG8KLS0tIDdVdmpiT1hxZExadUdMcWl5Mkdq + ZjFFWHh2bWNBSE5MeEJ1S00zOWpGU3cKPOjdYqv1dKlpfrE4CwyjTM27W/O2ax9k + XHVBDtg9E9g4d5E1Mf3o+bfWl96wVteVr/W5ZaGE/WRSqIa+nBC4rA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1s9nvc4rxj3kaj4apmzzn8fmjrudrvdhgu70rg04we9hyse5aadsq7kmckn + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1TXVpRlFTYVRQR3Nid1Ir + ODRuTFVVQkFpSCtIZzR3YkRPR05vbituOFRBCllQVkdja1RHaXl5c2VERlV4SjE5 + UWt1RkNlRHF6WFJwMk9qZU1XZEwwTjgKLS0tIHBWUSs5a0VWNHkzakNzZmdmUlkw + Rk9FSTFMYnA1V1pyakxDSlVzVzhKS3cKSFJ0H4Z5Q8FzrpRWUUQgpBj+nSdNT9yR + ZeF3+hZZCQwIaMTxq8RXq6zyUtQsLRXUnPl+59NmPmNKaPkKi8X9tA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1z5wtjmk0jw0j9qz9k5rrnp30nzqxrl3v6wgl7eryvqus28zekp4qpx9jc2 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwZUZMYXZHWXpYN0ZYNE5J + bmIxWWhGcko2enJBdnpjdTFUY3RwMnZIaFc0Cmo1NHZVMzFxK2Y3K3UxWHB3UnQr + UmV6bzl2Rk1xMVF6NXM4MmdMeWVKZnMKLS0tIFBEZG5RTE4zNXNJYTlxd2VxRW5p + YklZWHhseUp6d1AwRHp6QnRBL3JWZ3MKO19rYxmCZDro3x83aM5oHI26TpITblCU + X40297QCc2pWWfYSfGa3H3IUZMFe5STvPGoKUhRWZ+XMqYb6c2kp/A== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-10-03T12:15:16Z" + mac: ENC[AES256_GCM,data:59efYmiDE6RzTR6H7qd9ujfmRXmW7NNQVdX4viU5OWMHTGCN0Yf7rDFbPLSlagUFbAImGZgcSZ6hD9qwk4Xl4PIoVwQ6P9nGky6tI4ljEjQPvzceSSeMvhDyvrrDryr+qs2E6aeLCJF4uHXa7u/XhgvxsN6teFeFLg9SjJzjkEY=,iv:/bQd9wn+5FuPcF9HsgeZuzpzLusaJXYGQn8Pj5VHlWU=,tag:BxMtpEEG2OvGZA+n/OdJeA==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.0 diff --git a/loki/values.yaml b/loki/values.yaml index 061e415..852e060 100644 --- a/loki/values.yaml +++ b/loki/values.yaml @@ -1,14 +1,7 @@ -minio: - enabled: true - replicas: 3 - drivesPerNode: 2 - persistence: - size: 15Gi - storageClass: longhorn-local loki: auth_enabled: false persistence: - size: 30Gi + size: 10Gi schemaConfig: configs: - from: 2024-04-14 @@ -19,20 +12,84 @@ loki: prefix: index_ period: 24h limits_config: - retention_period: 168h + retention_period: 24h compactor: compaction_interval: 15m retention_enabled: true retention_delete_delay: 1h retention_delete_worker_count: 150 delete_request_store: s3 + storage: + s3: + endpoint: https://fsn1.your-objectstorage.com + bucketnames: yolokube-logs + s3forcepathstyle: true + accessKeyId: ${S3_LOKI_ACCESS_KEY_ID} + secretAccessKey: ${S3_LOKI_SECRET_ACCESS_KEY} monitoring: - selfMonitoring: - grafanaAgent: - installOperator: false + dashboards: + enabled: true + serviceMonitor: + enabled: true +read: + extraArgs: + - '-config.expand-env=true' + extraEnv: + - name: S3_LOKI_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_ACCESS_KEY_ID + - name: S3_LOKI_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_SECRET_ACCESS_KEY write: + extraArgs: + - '-config.expand-env=true' + extraEnv: + - name: S3_LOKI_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_ACCESS_KEY_ID + - name: S3_LOKI_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_SECRET_ACCESS_KEY persistence: storageClass: longhorn-local backend: + extraArgs: + - '-config.expand-env=true' + extraEnv: + - name: S3_LOKI_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_ACCESS_KEY_ID + - name: S3_LOKI_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_SECRET_ACCESS_KEY + persistence: + storageClass: longhorn-local +compactor: + extraArgs: + - '-config.expand-env=true' + extraEnv: + - name: S3_LOKI_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_ACCESS_KEY_ID + - name: S3_LOKI_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: loki-bucket-secret + key: S3_LOKI_SECRET_ACCESS_KEY persistence: storageClass: longhorn-local