From dd006d9344fc2fef69492fab3eb1840aa40d5d41 Mon Sep 17 00:00:00 2001 From: Kentaro1043 <71170923+Kentaro1043@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:04:58 +0900 Subject: [PATCH 1/3] =?UTF-8?q?loki=E3=81=AE=E3=83=AD=E3=82=B0=E3=82=92S3?= =?UTF-8?q?=E3=81=AB=E4=BF=9D=E5=AD=98=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monitor/loki/config/config.yaml | 37 ++++++++++++++++++------ monitor/loki/ksops.yaml | 11 +++++++ monitor/loki/kustomization.yaml | 3 ++ monitor/loki/secrets/s3-credentials.yaml | 32 ++++++++++++++++++++ monitor/loki/stateful-set.yaml | 12 ++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 monitor/loki/ksops.yaml create mode 100644 monitor/loki/secrets/s3-credentials.yaml diff --git a/monitor/loki/config/config.yaml b/monitor/loki/config/config.yaml index cc34e121..db8469f0 100644 --- a/monitor/loki/config/config.yaml +++ b/monitor/loki/config/config.yaml @@ -5,9 +5,27 @@ common: path_prefix: /var/loki replication_factor: 1 storage: - filesystem: - chunks_directory: /loki-data/chunks - rules_directory: /loki-data/rules + s3: + endpoint: https://s3.ap-northeast-1.wasabisys.com + region: ap-northeast-1 + bucketnames: trap-loki + access_key_id: ${S3_ACCESS_KEY} + secret_access_key: ${S3_SECRET_KEY} + s3forcepathstyle: true + +storage_config: + filesystem: + directory: /loki-data/chunks + hedging: + at: 250ms + max_per_second: 20 + up_to: 3 + +ruler: + storage: + type: local + local: + directory: /loki-data/rules limits_config: max_cache_freshness_per_query: 10m @@ -42,18 +60,19 @@ schema_config: index: prefix: index_ period: 24h + - from: "2024-12-13" + object_store: s3 + store: tsdb + schema: v13 + index: + prefix: index_ + period: 24h server: log_level: warn grpc_listen_port: 9095 http_listen_port: 3100 -storage_config: - hedging: - at: 250ms - max_per_second: 20 - up_to: 3 - table_manager: retention_deletes_enabled: true # 7 days diff --git a/monitor/loki/ksops.yaml b/monitor/loki/ksops.yaml new file mode 100644 index 00000000..714c4f4c --- /dev/null +++ b/monitor/loki/ksops.yaml @@ -0,0 +1,11 @@ +apiVersion: viaduct.ai/v1 +kind: ksops +metadata: + name: ksops + annotations: + config.kubernetes.io/function: | + exec: + path: ksops + +files: + - ./secrets/s3-credentials.yaml diff --git a/monitor/loki/kustomization.yaml b/monitor/loki/kustomization.yaml index 35064e61..e7ca89bd 100644 --- a/monitor/loki/kustomization.yaml +++ b/monitor/loki/kustomization.yaml @@ -10,3 +10,6 @@ configMapGenerator: - name: loki-runtime-config files: - config/runtime-config.yaml + +generators: + - ksops.yaml diff --git a/monitor/loki/secrets/s3-credentials.yaml b/monitor/loki/secrets/s3-credentials.yaml new file mode 100644 index 00000000..ef5c92f3 --- /dev/null +++ b/monitor/loki/secrets/s3-credentials.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Secret +metadata: + name: my-secret + annotations: + # kustomizeによってSecret名にhash suffixを付けさせる設定 + # Secretの中身が変更されたとき、自動リロードが可能になる + # kustomize設定のnameReferenceで、Secretを読む側のフィールドを参照する必要あり + kustomize.config.k8s.io/needs-hash: "true" +stringData: + s3-access-key: ENC[AES256_GCM,data:GsSaKLRolz266FoNo7C/UhKJpmk=,iv:4XLXRRpEY8x7rIIZDgsAgVsUDrxw0dAB73AVZvwj+ds=,tag:VSajBS+c8PtfsT3yhlUstA==,type:str] + s3-secret-key: ENC[AES256_GCM,data:YCSRynS4ZUZ+t/TroP2umh2vTo2he09UtesyP9RhDKqsE9Ch9kWFOQ==,iv:ioa0o5L0koKQtyJR1D5ZTs1XO7TiF5WQirAEfVZTZK8=,tag:n4dMowfEvqTx9P1rAjdySw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age156red4ptw5huzpwlfnrukg4htuucdweu9jg8usjz98ggmeyedces3xqplq + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxQUMxa1JIbzd0OHN4UGRi + OW14ait6K0dkNGVsQXUzS2NqckQvREdRTnlBCkp3bWhxaGttZDE2RU1sMDNLa3dj + L1NQdWJLZnVJcEg1eXRFUjJpRTVtS3cKLS0tIGpMQ2RuQVJyRHd6SWVCL2tlZ2gz + Vnd1SWdJSFNwaS92TWpBZ3pzcjdvVlkKXlirgs/ia+bfkZ5VljptL6edz+XSevhP + O40ug/XU3loMm9qcYo0dtrwkXBZfcK/1beX6ZcTqHVfj6peBqS7gTQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-12-12T00:38:43Z" + mac: ENC[AES256_GCM,data:goQ+VVtPmWnYQ9YJtlLeCkm7/tvtzN1WTjin7kutQGtY6gkhazjqvuloUJl6KcOgeDGGcnbRdv7I6zv30GPVn+6s5lGT68MqQSncrVTwk4xOmFhQXjBRdQtLNpkWBgyPrrpXbjxWbl9/AvUe+56/KYECWM91Eksvy7hKQMQuozA=,iv:8Y19KEgxm43Bx+cMNrmAjrDm83j1+rQS6XThtKXKWpk=,tag:mV7autFZWiFDNyJnYYoh+A==,type:str] + pgp: [] + unencrypted_regex: ^(apiVersion|metadata|kind|type)$ + version: 3.9.2 diff --git a/monitor/loki/stateful-set.yaml b/monitor/loki/stateful-set.yaml index 20b2085a..a88f57c8 100644 --- a/monitor/loki/stateful-set.yaml +++ b/monitor/loki/stateful-set.yaml @@ -89,6 +89,18 @@ spec: periodSeconds: 1 failureThreshold: 60 + env: + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: s3-access-key + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: s3-secret-key + volumeMounts: - name: tmp mountPath: /tmp From ea3fcb902f358d78f16d38fea36a9e7754740871 Mon Sep 17 00:00:00 2001 From: Kentaro1043 <71170923+Kentaro1043@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:38:40 +0900 Subject: [PATCH 2/3] =?UTF-8?q?secret=E3=81=AE=E5=90=8D=E5=89=8D=E3=82=92?= =?UTF-8?q?=E5=A4=89=E3=81=88=E3=81=A6=E5=86=8D=E5=BA=A6=E6=9A=97=E5=8F=B7?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monitor/loki/secrets/s3-credentials.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/monitor/loki/secrets/s3-credentials.yaml b/monitor/loki/secrets/s3-credentials.yaml index ef5c92f3..b39616dc 100644 --- a/monitor/loki/secrets/s3-credentials.yaml +++ b/monitor/loki/secrets/s3-credentials.yaml @@ -1,15 +1,15 @@ apiVersion: v1 kind: Secret metadata: - name: my-secret + name: s3-credentials annotations: # kustomizeによってSecret名にhash suffixを付けさせる設定 # Secretの中身が変更されたとき、自動リロードが可能になる # kustomize設定のnameReferenceで、Secretを読む側のフィールドを参照する必要あり kustomize.config.k8s.io/needs-hash: "true" stringData: - s3-access-key: ENC[AES256_GCM,data:GsSaKLRolz266FoNo7C/UhKJpmk=,iv:4XLXRRpEY8x7rIIZDgsAgVsUDrxw0dAB73AVZvwj+ds=,tag:VSajBS+c8PtfsT3yhlUstA==,type:str] - s3-secret-key: ENC[AES256_GCM,data:YCSRynS4ZUZ+t/TroP2umh2vTo2he09UtesyP9RhDKqsE9Ch9kWFOQ==,iv:ioa0o5L0koKQtyJR1D5ZTs1XO7TiF5WQirAEfVZTZK8=,tag:n4dMowfEvqTx9P1rAjdySw==,type:str] + s3-access-key: ENC[AES256_GCM,data:DfporcfBusm/WLQhswKga0gc/1w=,iv:XyPqmSPfGnKYEU9S3WZvZb9emvrbeBTS1Bjh8htHdIs=,tag:MI9vR2dkzO/jSHfwtyVzRw==,type:str] + s3-secret-key: ENC[AES256_GCM,data:AXgXqwHH5apIm3L1TATAFnC/XH71AYt9lefKXZq412UqxP5dWdNK3g==,iv:OmQiywnxB00szqz1SH3fUCdsM+SxsY/msWJ6pDTrqWA=,tag:wALnssWcwqMHp0dizoWI+A==,type:str] sops: kms: [] gcp_kms: [] @@ -19,14 +19,14 @@ sops: - recipient: age156red4ptw5huzpwlfnrukg4htuucdweu9jg8usjz98ggmeyedces3xqplq enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxQUMxa1JIbzd0OHN4UGRi - OW14ait6K0dkNGVsQXUzS2NqckQvREdRTnlBCkp3bWhxaGttZDE2RU1sMDNLa3dj - L1NQdWJLZnVJcEg1eXRFUjJpRTVtS3cKLS0tIGpMQ2RuQVJyRHd6SWVCL2tlZ2gz - Vnd1SWdJSFNwaS92TWpBZ3pzcjdvVlkKXlirgs/ia+bfkZ5VljptL6edz+XSevhP - O40ug/XU3loMm9qcYo0dtrwkXBZfcK/1beX6ZcTqHVfj6peBqS7gTQ== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArOURSZUVjWm5hZ1NNK1ZR + Q2tjSkJNN01ucUNHQ21kSHRJTHo2L1pRN0RFClpMbmxXZ0JKWTVNUGpuWndqQTFG + QmcxaDFNZ3FKL1hDZEs4QnhkTlZFSTAKLS0tIFZqUHVmMjhKTEpUK1FPRWw5bzFJ + ZVhENlNMWk9JdEdCWjlrN3VCSDZRaHcKhKCNs5bWWEoIZ2akoKJ9ZCDJLEGyqBP/ + +KZUHSRXBHxLusjAWfEt1DiZuWbk+rQBotR40H7f8cHM85K+Ps7gJQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-12-12T00:38:43Z" - mac: ENC[AES256_GCM,data:goQ+VVtPmWnYQ9YJtlLeCkm7/tvtzN1WTjin7kutQGtY6gkhazjqvuloUJl6KcOgeDGGcnbRdv7I6zv30GPVn+6s5lGT68MqQSncrVTwk4xOmFhQXjBRdQtLNpkWBgyPrrpXbjxWbl9/AvUe+56/KYECWM91Eksvy7hKQMQuozA=,iv:8Y19KEgxm43Bx+cMNrmAjrDm83j1+rQS6XThtKXKWpk=,tag:mV7autFZWiFDNyJnYYoh+A==,type:str] + lastmodified: "2024-12-12T08:37:51Z" + mac: ENC[AES256_GCM,data:+JpJxgqkRhYwb1zTQF4Kjd6sunKbEDAXDDjeyqrdc7ft03gnYYJQtT+TVwDDwIETeDPpgACtbMjyyGZNCoHna9ERGY57srg7XzsjqRwJgja2xHw548vFYFBVqbkzANBt9QaHOZRWd5Ez2VlKUECHVxuoo74mZPfTJ5q3GmIl0I8=,iv:m+nlDp8MZKBvhl2Y0IapE6w1Dzl4RLRKdnv5ePc4M6Y=,tag:c4dYOJbuM09lO7GUKfitqA==,type:str] pgp: [] unencrypted_regex: ^(apiVersion|metadata|kind|type)$ version: 3.9.2 From 8e6312900f611998d50953c9f710bea95787f87b Mon Sep 17 00:00:00 2001 From: Kentaro1043 <71170923+Kentaro1043@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:38:55 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E4=BF=9D=E5=AD=98=E5=85=88=E3=82=92s3?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B=E3=81=AE=E3=82=921=E6=97=A5?= =?UTF-8?q?=E9=81=85=E3=82=89=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monitor/loki/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/loki/config/config.yaml b/monitor/loki/config/config.yaml index db8469f0..b904f652 100644 --- a/monitor/loki/config/config.yaml +++ b/monitor/loki/config/config.yaml @@ -60,7 +60,7 @@ schema_config: index: prefix: index_ period: 24h - - from: "2024-12-13" + - from: "2024-12-14" object_store: s3 store: tsdb schema: v13