工作原理图

image.png
image.png

官方文档
https://grafana.com/docs/loki/latest/installation/helm/install-scalable/

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

[root@ip-172-31-31-169 loki]# cat values.yaml
loki:
storage:
bucketNames:
chunks: chunks
ruler: ruler
admin: admin
type: s3
s3:
endpoint: loki-nprod.s3.me-central-1.amazonaws.com
region: me-central-1
secretAccessKey: www
accessKeyId: www
s3ForcePathStyle: false
insecure: false

helm install --values values.yaml loki grafana/loki
# or upgrade for existing installations
helm upgrade --values values.yaml loki grafana/loki


[root@ip-172-31-31-169 loki]# helm list -aq
ecr-chart-demo
loki

[root@ip-172-31-31-169 loki]# helm delete loki
release "loki" uninstalled
Grafana Loki笔记02: 使用Helm安装Loki
https://blog.frognew.com/2023/05/loki-02-install.html

-n loki  --create-namespace
helm install -n loki  --create-namespace --values values.yaml loki grafana/loki

[root@ip-172-31-31-169 loki]# kubectl get all -n loki
NAME                                               READY   STATUS    RESTARTS   AGE
pod/loki-backend-0                                 1/1     Running   0          3m9s
pod/loki-backend-1                                 1/1     Running   0          3m9s
pod/loki-backend-2                                 1/1     Running   0          3m9s
pod/loki-canary-nxzt7                              1/1     Running   0          3m9s
pod/loki-canary-vkc5r                              1/1     Running   0          3m9s
pod/loki-canary-z58sj                              1/1     Running   0          3m9s
pod/loki-gateway-88747474b-sr5cf                   1/1     Running   0          3m9s
pod/loki-grafana-agent-operator-864d5f9d87-d62sc   1/1     Running   0          3m9s
pod/loki-logs-6ssfz                                2/2     Running   0          3m1s
pod/loki-logs-j9fhp                                2/2     Running   0          3m1s
pod/loki-logs-lnkcp                                2/2     Running   0          3m1s
pod/loki-read-5bc7b5ccb7-bwfzv                     1/1     Running   0          3m9s
pod/loki-read-5bc7b5ccb7-dd856                     1/1     Running   0          3m9s
pod/loki-read-5bc7b5ccb7-jqnrl                     1/1     Running   0          3m9s
pod/loki-write-0                                   1/1     Running   0          3m9s
pod/loki-write-1                                   1/1     Running   0          3m9s
pod/loki-write-2                                   1/1     Running   0          3m9s

NAME                                TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
service/loki-backend                ClusterIP   10.100.42.40     <none>        3100/TCP,9095/TCP   3m9s
service/loki-backend-headless       ClusterIP   None             <none>        3100/TCP,9095/TCP   3m9s
service/loki-canary                 ClusterIP   10.100.39.183    <none>        3500/TCP            3m9s
service/loki-gateway                ClusterIP   10.100.133.224   <none>        80/TCP              3m9s
service/loki-memberlist             ClusterIP   None             <none>        7946/TCP            3m9s
service/loki-read                   ClusterIP   10.100.131.233   <none>        3100/TCP,9095/TCP   3m9s
service/loki-read-headless          ClusterIP   None             <none>        3100/TCP,9095/TCP   3m9s
service/loki-write                  ClusterIP   10.100.134.225   <none>        3100/TCP,9095/TCP   3m9s
service/loki-write-headless         ClusterIP   None             <none>        3100/TCP,9095/TCP   3m9s
service/query-scheduler-discovery   ClusterIP   None             <none>        3100/TCP,9095/TCP   3m9s

NAME                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/loki-canary   3         3         3       3            3           <none>          3m9s
daemonset.apps/loki-logs     3         3         3       3            3           <none>          3m1s

NAME                                          READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/loki-gateway                  1/1     1            1           3m9s
deployment.apps/loki-grafana-agent-operator   1/1     1            1           3m9s
deployment.apps/loki-read                     3/3     3            3           3m9s

NAME                                                     DESIRED   CURRENT   READY   AGE
replicaset.apps/loki-gateway-88747474b                   1         1         1       3m9s
replicaset.apps/loki-grafana-agent-operator-864d5f9d87   1         1         1       3m9s
replicaset.apps/loki-read-5bc7b5ccb7                     3         3         3       3m9s

NAME                            READY   AGE
statefulset.apps/loki-backend   3/3     3m9s
statefulset.apps/loki-write     3/3     3m9s

[root@ip-172-31-31-169 monitor]# helm delete loki -n loki
release "loki" uninstalled

以上不再适用,已卸载
其他案例整理
Kubernetes 可观测性第 1 部分:事件、日志以及与 Slack、OpenAI 和 Grafana 的集成
https://betterprogramming.pub/kubernetes-observability-part-1-events-logs-integration-with-slack-openai-and-grafana-62068cf43ec

在 10 分钟内使用 Grafana Loki 和 Promtail 进行 Kubernetes 日志记录
https://akyriako.medium.com/kubernetes-logging-with-grafana-loki-promtail-in-under-10-minutes-d2847d526f9e

install loki-distributed chart (推荐)

mkdir loki-distributed
cd loki-distributed/
helm show values grafana/loki-distributed > loki-distributed-overrides.yaml

vim loki-distributed-overrides.yaml

Change the object and shared store target to s3
image.png

Add the configuration of your storage, pointing to the designated S3 bucket.
StorageConfig with AWS S3 :

https://github.com/grafana/loki/issues/7335
storageConfig:
        boltdb_shipper:
            shared_store: s3
            active_index_directory: /var/loki/index
            cache_location: /var/loki/cache
            cache_ttl: 168h
        filesystem:
            directory: /var/loki/chunks
        aws:
         s3: s3://name
         endpoint: https://s3.eu-west-3.amazonaws.com
         region: eu-west-3
         bucketnames: name
         access_key_id: ${LOKI_S3_ACCESS_KEY_ID}
         secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY}
         s3forcepathstyle: true
         insecure: false
         sse_encryption: false
    aws:
      s3: s3://loki-nprod
      endpoint: https://loki-nprod.s3.me-central-1.amazonaws.com
      region: me-central-1
      bucketnames: loki-nprod
      access_key_id: www
      secret_access_key: www
      s3forcepathstyle: true
      insecure: false
      sse_encryption: false

Configure the compactor
image.png

compactor:
  working_directory: /loki/compactor
  shared_store: s3
  compaction_interval: 5m

[root@ip-172-31-31-169 loki-distributed]# helm upgrade --install --values loki-distributed-overrides.yaml loki grafana/loki-distributed -n grafana-loki --create-namespace
Release "loki" does not exist. Installing it now.
NAME: loki
LAST DEPLOYED: Tue Jul 11 03:58:43 2023
NAMESPACE: grafana-loki
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
 Welcome to Grafana Loki
 Chart version: 0.69.16
 Loki version: 2.8.2
***********************************************************************

Installed components:
* gateway
* ingester
* distributor
* querier
* query-frontend
* compactor

error

[root@ip-172-31-31-169 loki-distributed]# kubectl logs -f loki-loki-distributed-ingester-0 -n grafana-loki
Unrecognized storage client s3, choose one of: aws, cassandra, inmemory, gcp, bigtable, bigtable-hashed
error creating index client

重新配置stroge

https://grafana.com/docs/loki/latest/configuration/examples/

    aws:
      s3: s3://AK:SK@loki-nprod.s3.me-central-1.amazonaws.com/loki-nprod
      s3forcepathstyle: true

error

[root@ip-172-31-31-169 loki-distributed]# kubectl logs -f loki-loki-distributed-querier-0 -n grafana-loki
failed parsing config: /etc/loki/config/config.yaml: parse "s3://AK:SK@loki-nprod.s3.me-central-1.amazonaws.com/loki-nprod": invalid port ":0DvuC3I+DR7oeROs2" after host. Use `-config.expand-env=true` flag if you want to expand environment variables in your config file

https://github.com/grafana/loki/issues/9143
[root@ip-172-31-31-169 loki-distributed]# cat loki-s3-credentials-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: loki-s3-credentials
  namespace: grafana-loki
type: Opaque
data:
  AWS_ACCESS_KEY_ID: AK
  AWS_SECRET_ACCESS_KEY: SK

重新配置成功 object_store从s3修改为aws

    compactor:
      working_directory: /loki/compactor
      shared_store: aws
      compaction_interval: 5m

  schemaConfig:
    configs:
    - from: 2020-09-07
      store: boltdb-shipper
      object_store: aws
      schema: v11
      index:
        prefix: loki_index_
        period: 24h

  # -- Check https://grafana.com/docs/loki/latest/configuration/#storage_config for more info on how to configure storages
  storageConfig:
    boltdb_shipper:
      shared_store: aws
      active_index_directory: /var/loki/index
      cache_location: /var/loki/cache
      cache_ttl: 168h

# -- Uncomment to configure each storage individually
#   azure: {}
#   gcs: {}
    aws:
      s3: s3://me-central-1
      bucketnames: loki-nprod
      access_key_id: AK
      secret_access_key: SAK

set up Promtail on EKS

https://grafana.com/docs/loki/latest/clients/aws/eks/

[root@ip-172-31-31-169 promtail]# curl https://raw.githubusercontent.com/grafana/helm-charts/main/charts/promtail/values.yaml > values.yaml
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 18612  100 18612    0     0  39432      0 --:--:-- --:--:-- --:--:-- 39515

kubectl create namespace monitoring

[root@ip-172-31-31-169 promtail]# helm install promtail --namespace monitoring grafana/promtail -f values.yaml
NAME: promtail
LAST DEPLOYED: Mon Jul 10 10:49:58 2023
NAMESPACE: monitoring
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
 Welcome to Grafana Promtail
 Chart version: 6.11.5
 Promtail version: 2.8.2
***********************************************************************

Verify the application is working by running these commands:
* kubectl --namespace monitoring port-forward daemonset/promtail 3101
* curl http://127.0.0.1:3101/metrics
error  
Unable to fetch labels from Loki (no org id ), please check the server logs for more details

https://itnext.io/multi-tenancy-with-loki-promtail-and-grafan...

以上promtail不适用,已清理
[root@ip-172-31-31-169 monitor]# helm delete promtail -n monitoring
release "promtail" uninstalled

https://akyriako.medium.com/kubernetes-logging-with-grafana-l...

helm show values grafana/promtail > promtail-overrides.yaml

添加loki-gateway的地址:loki-loki-distributed-gateway.grafana-loki.svc.cluster.local

image.png

[root@ip-172-31-31-169 promtail]# helm upgrade --install --values promtail-overrides.yaml promtail grafana/promtail -n grafana-loki
Release "promtail" does not exist. Installing it now.
NAME: promtail
LAST DEPLOYED: Tue Jul 11 08:04:51 2023
NAMESPACE: grafana-loki
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
***********************************************************************
 Welcome to Grafana Promtail
 Chart version: 6.11.5
 Promtail version: 2.8.2
***********************************************************************

Verify the application is working by running these commands:
* kubectl --namespace grafana-loki port-forward daemonset/promtail 3101
* curl http://127.0.0.1:3101/metrics

添加到grafana dashboard
image.png
save & test
image.png


EngineerLeo
598 声望38 粉丝

专注于云原生、AI等相关技术