工作原理图
官方文档
https://grafana.com/docs/loki/latest/installation/helm/install-scalable/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
[root@ip-172-31-31-169 loki]# cat values.yaml
loki:
storage:
bucketNames:
chunks: chunks
ruler: ruler
admin: admin
type: s3
s3:
endpoint: loki-nprod.s3.me-central-1.amazonaws.com
region: me-central-1
secretAccessKey: www
accessKeyId: www
s3ForcePathStyle: false
insecure: false
helm install --values values.yaml loki grafana/loki
# or upgrade for existing installations
helm upgrade --values values.yaml loki grafana/loki
[root@ip-172-31-31-169 loki]# helm list -aq
ecr-chart-demo
loki
[root@ip-172-31-31-169 loki]# helm delete loki
release "loki" uninstalled
Grafana Loki笔记02: 使用Helm安装Loki
https://blog.frognew.com/2023/05/loki-02-install.html
-n loki --create-namespace
helm install -n loki --create-namespace --values values.yaml loki grafana/loki
[root@ip-172-31-31-169 loki]# kubectl get all -n loki
NAME READY STATUS RESTARTS AGE
pod/loki-backend-0 1/1 Running 0 3m9s
pod/loki-backend-1 1/1 Running 0 3m9s
pod/loki-backend-2 1/1 Running 0 3m9s
pod/loki-canary-nxzt7 1/1 Running 0 3m9s
pod/loki-canary-vkc5r 1/1 Running 0 3m9s
pod/loki-canary-z58sj 1/1 Running 0 3m9s
pod/loki-gateway-88747474b-sr5cf 1/1 Running 0 3m9s
pod/loki-grafana-agent-operator-864d5f9d87-d62sc 1/1 Running 0 3m9s
pod/loki-logs-6ssfz 2/2 Running 0 3m1s
pod/loki-logs-j9fhp 2/2 Running 0 3m1s
pod/loki-logs-lnkcp 2/2 Running 0 3m1s
pod/loki-read-5bc7b5ccb7-bwfzv 1/1 Running 0 3m9s
pod/loki-read-5bc7b5ccb7-dd856 1/1 Running 0 3m9s
pod/loki-read-5bc7b5ccb7-jqnrl 1/1 Running 0 3m9s
pod/loki-write-0 1/1 Running 0 3m9s
pod/loki-write-1 1/1 Running 0 3m9s
pod/loki-write-2 1/1 Running 0 3m9s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/loki-backend ClusterIP 10.100.42.40 <none> 3100/TCP,9095/TCP 3m9s
service/loki-backend-headless ClusterIP None <none> 3100/TCP,9095/TCP 3m9s
service/loki-canary ClusterIP 10.100.39.183 <none> 3500/TCP 3m9s
service/loki-gateway ClusterIP 10.100.133.224 <none> 80/TCP 3m9s
service/loki-memberlist ClusterIP None <none> 7946/TCP 3m9s
service/loki-read ClusterIP 10.100.131.233 <none> 3100/TCP,9095/TCP 3m9s
service/loki-read-headless ClusterIP None <none> 3100/TCP,9095/TCP 3m9s
service/loki-write ClusterIP 10.100.134.225 <none> 3100/TCP,9095/TCP 3m9s
service/loki-write-headless ClusterIP None <none> 3100/TCP,9095/TCP 3m9s
service/query-scheduler-discovery ClusterIP None <none> 3100/TCP,9095/TCP 3m9s
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/loki-canary 3 3 3 3 3 <none> 3m9s
daemonset.apps/loki-logs 3 3 3 3 3 <none> 3m1s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/loki-gateway 1/1 1 1 3m9s
deployment.apps/loki-grafana-agent-operator 1/1 1 1 3m9s
deployment.apps/loki-read 3/3 3 3 3m9s
NAME DESIRED CURRENT READY AGE
replicaset.apps/loki-gateway-88747474b 1 1 1 3m9s
replicaset.apps/loki-grafana-agent-operator-864d5f9d87 1 1 1 3m9s
replicaset.apps/loki-read-5bc7b5ccb7 3 3 3 3m9s
NAME READY AGE
statefulset.apps/loki-backend 3/3 3m9s
statefulset.apps/loki-write 3/3 3m9s
[root@ip-172-31-31-169 monitor]# helm delete loki -n loki
release "loki" uninstalled
以上不再适用，已卸载其他案例整理
Kubernetes 可观测性第 1 部分：事件、日志以及与 Slack、OpenAI 和 Grafana 的集成
https://betterprogramming.pub/kubernetes-observability-part-1-events-logs-integration-with-slack-openai-and-grafana-62068cf43ec
在 10 分钟内使用 Grafana Loki 和 Promtail 进行 Kubernetes 日志记录
https://akyriako.medium.com/kubernetes-logging-with-grafana-loki-promtail-in-under-10-minutes-d2847d526f9einstall loki-distributed chart （推荐）
mkdir loki-distributed
cd loki-distributed/
helm show values grafana/loki-distributed > loki-distributed-overrides.yaml
vim loki-distributed-overrides.yamlChange the object and shared store target to s3
Add the configuration of your storage, pointing to the designated S3 bucket.
StorageConfig with AWS S3 :
https://github.com/grafana/loki/issues/7335
storageConfig:
boltdb_shipper:
shared_store: s3
active_index_directory: /var/loki/index
cache_location: /var/loki/cache
cache_ttl: 168h
filesystem:
directory: /var/loki/chunks
aws:
s3: s3://name
endpoint: https://s3.eu-west-3.amazonaws.com
region: eu-west-3
bucketnames: name
access_key_id: ${LOKI_S3_ACCESS_KEY_ID}
secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY}
s3forcepathstyle: true
insecure: false
sse_encryption: false aws:
s3: s3://loki-nprod
endpoint: https://loki-nprod.s3.me-central-1.amazonaws.com
region: me-central-1
bucketnames: loki-nprod
access_key_id: www
secret_access_key: www
s3forcepathstyle: true
insecure: false
sse_encryption: falseConfigure the compactor
compactor:
working_directory: /loki/compactor
shared_store: s3
compaction_interval: 5m
[root@ip-172-31-31-169 loki-distributed]# helm upgrade --install --values loki-distributed-overrides.yaml loki grafana/loki-distributed -n grafana-loki --create-namespace
Release "loki" does not exist. Installing it now.
NAME: loki
LAST DEPLOYED: Tue Jul 11 03:58:43 2023
NAMESPACE: grafana-loki
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
Welcome to Grafana Loki
Chart version: 0.69.16
Loki version: 2.8.2
***********************************************************************
Installed components:
* gateway
* ingester
* distributor
* querier
* query-frontend
* compactorerror
[root@ip-172-31-31-169 loki-distributed]# kubectl logs -f loki-loki-distributed-ingester-0 -n grafana-loki
Unrecognized storage client s3, choose one of: aws, cassandra, inmemory, gcp, bigtable, bigtable-hashed
error creating index client重新配置stroge
https://grafana.com/docs/loki/latest/configuration/examples/
aws:
s3: s3://AK:SK@loki-nprod.s3.me-central-1.amazonaws.com/loki-nprod
s3forcepathstyle: trueerror
[root@ip-172-31-31-169 loki-distributed]# kubectl logs -f loki-loki-distributed-querier-0 -n grafana-loki
failed parsing config: /etc/loki/config/config.yaml: parse "s3://AK:SK@loki-nprod.s3.me-central-1.amazonaws.com/loki-nprod": invalid port ":0DvuC3I+DR7oeROs2" after host. Use `-config.expand-env=true` flag if you want to expand environment variables in your config file
https://github.com/grafana/loki/issues/9143
[root@ip-172-31-31-169 loki-distributed]# cat loki-s3-credentials-secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: loki-s3-credentials
namespace: grafana-loki
type: Opaque
data:
AWS_ACCESS_KEY_ID: AK
AWS_SECRET_ACCESS_KEY: SK
重新配置成功 object_store从s3修改为aws
compactor:
working_directory: /loki/compactor
shared_store: aws
compaction_interval: 5m
schemaConfig:
configs:
- from: 2020-09-07
store: boltdb-shipper
object_store: aws
schema: v11
index:
prefix: loki_index_
period: 24h
# -- Check https://grafana.com/docs/loki/latest/configuration/#storage_config for more info on how to configure storages
storageConfig:
boltdb_shipper:
shared_store: aws
active_index_directory: /var/loki/index
cache_location: /var/loki/cache
cache_ttl: 168h
# -- Uncomment to configure each storage individually
# azure: {}
# gcs: {}
aws:
s3: s3://me-central-1
bucketnames: loki-nprod
access_key_id: AK
secret_access_key: SAK
set up Promtail on EKS
https://grafana.com/docs/loki/latest/clients/aws/eks/
[root@ip-172-31-31-169 promtail]# curl https://raw.githubusercontent.com/grafana/helm-charts/main/charts/promtail/values.yaml > values.yaml
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 18612 100 18612 0 0 39432 0 --:--:-- --:--:-- --:--:-- 39515
kubectl create namespace monitoring
[root@ip-172-31-31-169 promtail]# helm install promtail --namespace monitoring grafana/promtail -f values.yaml
NAME: promtail
LAST DEPLOYED: Mon Jul 10 10:49:58 2023
NAMESPACE: monitoring
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
Welcome to Grafana Promtail
Chart version: 6.11.5
Promtail version: 2.8.2
***********************************************************************
Verify the application is working by running these commands:
* kubectl --namespace monitoring port-forward daemonset/promtail 3101
* curl http://127.0.0.1:3101/metrics
error
Unable to fetch labels from Loki (no org id ), please check the server logs for more details
https://itnext.io/multi-tenancy-with-loki-promtail-and-grafan...
以上promtail不适用，已清理
[root@ip-172-31-31-169 monitor]# helm delete promtail -n monitoring
release "promtail" uninstalledhttps://akyriako.medium.com/kubernetes-logging-with-grafana-l...
helm show values grafana/promtail > promtail-overrides.yaml
添加loki-gateway的地址：loki-loki-distributed-gateway.grafana-loki.svc.cluster.local
[root@ip-172-31-31-169 promtail]# helm upgrade --install --values promtail-overrides.yaml promtail grafana/promtail -n grafana-loki
Release "promtail" does not exist. Installing it now.
NAME: promtail
LAST DEPLOYED: Tue Jul 11 08:04:51 2023
NAMESPACE: grafana-loki
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
Welcome to Grafana Promtail
Chart version: 6.11.5
Promtail version: 2.8.2
***********************************************************************
Verify the application is working by running these commands:
* kubectl --namespace grafana-loki port-forward daemonset/promtail 3101
* curl http://127.0.0.1:3101/metrics添加到grafana dashboard
save & test
**粗体** _斜体_ [链接](http://example.com) `代码` - 列表 > 引用。你还可以使用@来通知其他用户。