Loki

distributor           #分发器。接收日志，根据一致性哈希路由到 ingester
ingester              #写入实例。内存接收日志，生成 chunk 日志块
consistent hash ring  #一致性哈希环。Loki 用于分片日志流的机制
replication factor    #复制因子。日志副本数量
quorum                #法定多数。写成功需要应答的副本数量
chunk                 #日志块。Loki存储日志的最小单元，租户+标签唯一
query-frontend        #查询前端。接收用户查询，拆分查询任务
query-scheduler       #查询调度器。分发子查询任务给 querier
querier               #查询器。执行实际查询，合并 ingester 与后端存储数据
backing store         #后端存储。对象存储，保存已经落盘的 chunk
deduplicates          #去重。处理多副本带来的重复日志
X-Scope-OrgID         #租户请求头。Loki 多租户识别标识

FLOG_VERSION=0.4.4
GO_VERSION=1.25.0
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

# 安装 go 语言
wget https://studygolang.com/dl/golang/go${GO_VERSION}.linux-${ARCH}.tar.gz
tar xf go${GO_VERSION}.linux-${ARCH}.tar.gz  -C /usr/local/
设置环境变量
cat >/etc/profile.d/go.sh<<\EOF
export GOROOT=/usr/local/go
export GOPATH=/go/lib:/go/goproject
#export GOBIN=/go/gobin
export PATH=$PATH:$GOROOT/bin:$GOBIN
EOF
mkdir /go/{lib,goproject,gobin} -pv
mkdir /go/goproject/src -pv
# 生效
source /etc/profile
go env -w GOPROXY=https://goproxy.cn,direct

# 编译安装 flog
mkdir -p $GOPATH/src/flog
cd $GOPATH/src/flog
git clone --depth 1 --branch v${FLOG_VERSION} https://github.com/mingrammer/flog.git
cd flog
CGO_ENABLED=0 go build -o flog .
cp ./flog /usr/local/bin/
flog -h

cat > /lib/systemd/system/flog.service <<EOF
[Unit]
Description=Flog fake log generator for loki test
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/flog -f json -d 200ms -l # 每 200 ms 生成日志
Restart=always
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now flog
systemctl status flog


# 安装 rsyslog
apt update
apt install rsyslog -y
systemctl enable --now rsyslog
systemctl status rsyslog

# 查看 json 日志
journalctl -u flog -f
tail -f /var/log/syslog

flog -f json -t log -o /var/log/flog/access_json.log -d 300ms -l -w &
flog -f apache_common -t log -o /var/log/flog/access_apache_common.log -d 300ms -l -w &
flog -f apache_combined -t log -o /var/log/flog/access_apache_combined.log -d 300ms -l -w &


#查看日志
head -1 /var/log/flog/access_json.log
head -1 /var/log/flog/access_apache_common.log
head -1 /var/log/flog/access_apache_combined.log

#启动两个容器生成日志
docker run -d --name mynginx -p 80:80 \
       --label service=nginx \
       --label env=prod \
       --label version=1.30.0 \
       ealen/echo-server:latest

docker run -d --name myapp -p 8080:80 \
       --label service=myapp \
       --label env=dev \
       --label ver1.0 \
       nbilal786/myapp:latest


# 安装 nerdctl 启动容器
apt install -y wget slirp4netns
cd /tmp
wget https://files.m.daocloud.io/github.com/containerd/nerdctl/releases/download/v2.3.4/nerdctl-2.3.4-linux-${ARCH}.tar.gz
tar -zxvf nerdctl-2.3.4-linux-${ARCH}.tar.gz
mv nerdctl /usr/local/bin/
chmod +x /usr/local/bin/nerdctl
sysctl -a  |grep ip_forward # 确认开启内核 IP 转发


# mynginx
nerdctl run -d --name mynginx -p 80:80 \
       --label service=nginx \
       --label env=prod \
       --label version=1.30.0 \
       ealen/echo-server:latest

# myapp
nerdctl run -d --name myapp -p 8080:80 \
       --label service=myapp \
       --label env=dev \
       --label ver=1.0 \
       nbilal786/myapp:latest

root@master01:~# kubectl  get pod
NAME                     READY   STATUS    RESTARTS       AGE
echo-647c98d9f-zrpdg     1/1     Running   0              29m
myapp-8b88f5cdd-5nbx4    1/1     Running   0              3m50s

#访问容器,生成容器日志
while true; do curl 10.103.236.202;sleep 1;do &
while true; do curl 10.103.236.202:8080;sleep 1;do &                                          

ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)
wget https://dl.minio.org.cn/server/minio/release/linux-${ARCH}/minio

chmod +x minio
cp minio /usr/local/bin
#MINIO_ROOT_USER=admin MINIO_ROOT_PASSWORD=password ./minio server /mnt/data --console-address ":9001" --address ":9000"
# 9001 为管理控制台，9000 为 api 端口

# 编译功能完善的 minio
MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
git clone --depth 1 --branch ${MINIO_VERSION} https://github.com/minio/minio.git
cd minio
CGO_ENABLED=0 go build -o minio .
cp ./minio /usr/local/bin/
minio --version

# 生成存储和启动配置
mkdir /data/minio -pv
groupadd -r minio-user
useradd -M -r -g minio-user minio-user
chown minio-user:minio-user /data/minio

cat >/etc/default/minio<<EOF
MINIO_ROOT_USER=admin
MINIO_ROOT_PASSWORD=Admin@123456
MINIO_VOLUMES="/data/minio"
MINIO_OPTS="--console-address :9001 --address :9000"
EOF

cat >/lib/systemd/system/minio.service<<\EOF
[Unit]
Description=MinIO
Documentation=https://min.io/docs/minio/linux/index.html
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/usr/local
User=minio-user
Group=minio-user
ProtectProc=invisible
EnvironmentFile=-/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES
Restart=always
LimitNOFILE=65536
TasksMax=infinity
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now  minio.service
systemctl status minio.service
journalctl -f -u minio.service

Access Key: d3qFSWw6zosYK4xyHeZC
Secret Key: iJ8gwfA5lZ3xhhf0n0EarjAkJgByNJKOnZWjJsUd

# 编译功能完善的 minio
MC_VERSION='RELEASE.2025-04-16T18-13-26Z'
git clone --depth 1 --branch ${MC_VERSION} https://github.com/minio/mc.git
cd mc
CGO_ENABLED=0 go build -o mc .
cp ./mc /usr/local/bin/
mc --version

# 配置客户端连接
echo 10.103.236.202  minio.jasper.org >> /etc/hosts
mc alias set minio http://minio.jasper.org:9000 d3qFSWw6zosYK4xyHeZC iJ8gwfA5lZ3xhhf0n0EarjAkJgByNJKOnZWjJsUd
mc alias ls minio

# 测试
mc admin info minio # 查看磁盘空间等信息
mc ls minio # 查看桶信息

LOKI_VERSION=3.7.8
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/loki-linux-${ARCH}.zip
unzip loki-linux-${ARCH}.zip
install -m 755 loki-linux-${ARCH}  /usr/local/bin/loki
loki --version

echo 10.103.236.202  minio.jasper.org >> /etc/hosts

mkdir -p /etc/loki

cat >/etc/loki/loki.yaml <<\EOF
auth_enabled: false

server:
  http_listen_port: 3100
  http_listen_address: 0.0.0.0
  log_level: info

common:
  path_prefix: /data/loki
  # minio 对象存储
  storage:
    s3:
      endpoint: minio.jasper.org:9000
      bucketnames: loki
      access_key_id: d3qFSWw6zosYK4xyHeZC
      secret_access_key: iJ8gwfA5lZ3xhhf0n0EarjAkJgByNJKOnZWjJsUd
      # minio 使用 http
      insecure: true
      # 强制 path style
      s3forcepathstyle: true
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

# loki v3.x 推荐 TSDB 索引
schema_config:
  configs:
    - from: 2024-01-01
      # TSDB 索引
      store: tsdb
      # 对象存储改为 mino
      object_store: s3
      schema: v13
      index:
        prefix: index_
        period: 24h

# 写入组件
ingester:
  # WAL 仍然保留本地
  wal:
    enabled: true
    dir: /data/loki/wal
  chunk_idle_period: 5m
  max_chunk_age: 1h

# TSDB 压缩
compactor:
  # 本地临时目录
  working_directory: /data/loki/compactor
  compaction_interval: 10m
  retention_enabled: true
  # 删除请求存储
  delete_request_store: s3

limits_config:
  # 日志保存 30 天
  retention_period: 30d
  max_query_series: 10000
  ingestion_rate_mb: 10
  ingestion_burst_size_mb: 20

# 告警规则
ruler:
  alertmanager_url: http://127.0.0.1:9093
  storage:
    type: local
    local:
      directory: /etc/loki/rules
  rule_path: /data/loki/rules-temp
  ring:
    kvstore:
      store: inmemory
  enable_api: true
EOF

# 语法检查
loki -config.file=/etc/loki/loki.yaml -verify-config

auth_enabled: false                             #关闭Loki内置认证,不需要登录鉴权,内网环境使用

server:                                         #HTTP服务配置块
  http_listen_port: 3100                        #Loki对外HTTP端口,默认3100
  http_listen_address: 0.0.0.0                  #监听所有网卡,允许外部访问
  log_level: info                               #Loki自身日志输出级别info/debug/warn/error

common:                                         #公共配置,多个组件共享
  path_prefix: /data/loki                       #本地数据根目录,wal、临时文件存放根路径
  # minio 对象存储
  storage:                                      #对象存储配置,日志chunk、索引存 MinIO S3 兼容在储存
    s3:
      endpoint: minio.jasper.org:9000           #MinIO服务地址端口
      bucketnames: loki                         #MinIO桶名称,loki所有数据存在这个bucket
      access_key_id: d3qFSWw6zosYK4xyHeZC
      secret_access_key: iJ8gwfA5lZ3xhhf0n0EarjAkJgByNJKOnZWjJsUd
      # minio 使用 http
      insecure: true                            #true 使用http,关闭https证书校验
      # 强制 path style
      s3forcepathstyle: true                    #MinIO必须开启,使用path模式访问s3,不使用虚拟host模式
  replication_factor: 1                         #副本数,单机部署写1;集群部署大于1
  ring:                                         #一致性ring配置,单机使用内存存储ring信息
    kvstore:
      store: inmemory                           #ring元数据存内存,单机专用;集群改用etcd

# loki v3.x 推荐 TSDB 索引
schema_config:                                  #索引schema配置,定义索引存储格式
  configs:
    - from: 2024-01-01                          #该schema生效起始时间,早于该时间的数据不使用此配置
      # TSDB 索引
      store: tsdb                               #使用TSDB索引(Lokiv3推荐,替代原先boltdb-shippper)
      # 对象存储改为 mino
      object_store: s3                          #索引文件存放后端:s3(minio)
      schema: v13                               #schema版本号,v13适配tsdb
      index:
        prefix: index_                          #对象存储里索引文件名称前缀
        period: 24h                             #索引分片周期,每24小时生成一份独立索引

# 写入组件
ingester:                                       #ingester:接收日志写入,内存攒chunk,刷写到对象存储
  # WAL 仍然保留本地
  wal:
    enabled: true                               #开启WAL预写日志,宕机防止内存日志丢失
    dir: /data/loki/wal                         #WAL本地磁盘存储目录
  chunk_idle_period: 5m                         #chunk空闲5分钟,没有新数据就刷盘到对象存储
  max_chunk_age: 1h                             #chunk最长存活1小时,无论是否空闲强制刷盘

# TSDB 压缩
compactor:                                      #TSDB压缩组件,合并小索引块、执行数据过期删除
  # 本地临时目录
  working_directory: /data/loki/compactor       #compactor工作临时目录,压缩过程中间文件
  compaction_interval: 10m                      #压缩任务执行间隔,每10分钟跑一次压缩
  retention_enabled: true                       #开启数据过期清理,配合retention_period删除旧片日志
  # 删除请求存储
  delete_request_store: s3                      #删除请求记录保存到s3对象存储

limits_config:                                  #全局限流、配额、保存周期配置
  # 日志保存 30 天
  retention_period: 30d                         #全局日志保留时长30天,到期自动删除
  max_query_series: 10000                       #查询最大返回时序序列数量,防止大查询压垮loki
  ingestion_rate_mb: 10                         #单实例日志写入速率上限10MB/s
  ingestion_burst_size_mb: 20                   #写入突发流量上限20MB

# 告警规则
ruler:                             #ruler组件:加载告警规则,执行LogQL,产生告警推送给aalertmanager
  alertmanager_url: http://127.0.0.1:9093       #Alertmanager接收告警的地址
  storage:                                      #ruler规则文件存储配置
    type: local                                 #规则使用本地文件模式
    local:
      directory: /etc/loki/rules                #告警规则yam1文件存放目录
  rule_path: /data/loki/rules-temp              #ruler运行时临时缓存目录
  ring:
    kvstore:
      store: inmemory                           #ruler ring,单机内存存储
  enable_api: true                              #开启ruler http API,支持curl上传校验规则,调试用

# 查找当前版本的配置
/usr/local/bin/loki -help 2>&1 | grep -- '-ui\.'

# 带上配置文件再列 target
/usr/local/bin/loki -config.file=/etc/loki/loki.yaml -list-targets

# 加载后的 ui 块实际长什么样
/usr/local/bin/loki -config.file=/etc/loki/loki.yaml -print-config-stderr 2>&1 | grep -A 25 '^ui:'

# 启动日志里 ui 相关的行
grep -i -E 'ui|cluster' /var/log/loki/loki.log | tail -40

mkdir -p /var/log/loki

cat >/lib/systemd/system/loki.service <<EOF
[Unit]
Description=Grafana Loki Log Aggregator Service
Documentation=https://grafana.com/docs/loki/latest
After=network-online.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/loki -config.file=/etc/loki/loki.yaml
Restart=always
RestartSec=5
StandardOutput=append:/var/log/loki/loki.log
StandardError=append:/var/log/loki/loki-error.log
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now loki
systemctl status loki
journalctl -f -u loki

# 自动生成相关文件
root@master01:~/.jasper# ls /data/loki/
compactor  tsdb-shipper-active  tsdb-shipper-cache  wal

# 查看 loki 相关服务
root@master01:~/.jasper# curl -s http://127.0.0.1:3100/services
querier => Running
ingester => Running
query-frontend => Running
server => Running
query-scheduler-ring => Running
ring => Running
analytics => Running
query-scheduler => Running
rule-evaluator => Running
query-frontend-tripperware => Running
compactor => Running
store => Running
ingester-querier => Running
ruler => Running
cache-generation-loader => Running
memberlist-kv => Running
distributor => Running

root@master01:~/.jasper# curl -s http://127.0.0.1:3100/ready
Ingester not ready: waiting for 15s after being ready

root@master01:~/.jasper# curl -s http://127.0.0.1:3100/ready
ready

ALLOY_VERSION=1.19.2
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://github.com/grafana/alloy/releases/download/v${ALLOY_VERSION}/alloy-linux-${ARCH}.zip
unzip alloy-linux-${ARCH}.zip
install -m 755 alloy-linux-${ARCH}  /usr/local/bin/alloy
alloy -v

#wget https://mirrors.tuna.tsinghua.edu.cn/grafana/apt/pool/main/g/grafana/grafana_13.2.2_34846740809_linux_arm64.deb
# apt install grafana_13.2.2_34846740809_linux_arm64.deb
# systemctl enable --now grafana-server

GRAFANA_VERSION=13.2.2
GRAFANA_FULLVERSION=${GRAFANA_VERSION}_34846740809
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://dl.grafana.com/grafana-enterprise/release/${GRAFANA_VERSION}/grafana-enterprise_${GRAFANA_FULLVERSION=}_linux_${ARCH}.tar.gz
tar -zxvf grafana-enterprise_${GRAFANA_FULLVERSION=}_linux_${ARCH}.tar.gz \
    --strip-components=2  \
    -C /usr/local/bin
    grafana-${GRAFANA_VERSION}/bin/grafana
grafana -v


tar xf grafana-enterprise_${GRAFANA_FULLVERSION=}_linux_${ARCH}.tar.gz
mv grafana-${GRAFANA_VERSION} /usr/local/grafana

useradd -r -s /bin/false grafana
chown -R grafana:users /usr/local/grafana

mkdir -pv /etc/grafana/provisioning/{access-control,alerting,dashboards,datasources,notifiers,plugins}
mkdir -pv /var/log/grafana /var/lib/grafana/plugins

chown -R grafana:grafana /var/log/grafana /var/lib/grafana /etc/grafana/provisioning

cat <<\EOF>  /etc/grafana/grafana.ini
[server]
# 生成外部链接（分享链接、OAuth 回调、告警通知里的跳转 URL）
root_url = http://10.103.236.201:3000
EOF



cat >/lib/systemd/system/grafana-server.service <<\EOF
[Unit]
Description=Grafana Server
After=network.target

[Service]
Type=simple
User=grafana
Group=users
#Grafana 启动的硬性依赖 /usr/local/grafana/conf/defaults.ini
#生效顺序（后者覆盖前者）:
#  conf/defaults.ini  →  grafana.ini  →  GF_* 环境变量  →  命令行 cfg: 参数
Environment=GF_PATHS_HOME=/usr/local/grafana
Environment=GF_PATHS_CONFIG=/etc/grafana/grafana.ini
Environment=GF_PATHS_DATA=/var/lib/grafana
Environment=GF_PATHS_LOGS=/var/log/grafana
Environment=GF_PATHS_PLUGINS=/var/lib/grafana/plugins
Environment=GF_PATHS_PROVISIONING=/etc/grafana/provisioning
ExecStart=/usr/local/grafana/bin/grafana server \
   --homepath=${GF_PATHS_HOME} \
   --config=${GF_PATHS_CONFIG} \
   cfg:default.log.mode=console                            \
   cfg:default.paths.data=${GF_PATHS_DATA}                   \
   cfg:default.paths.logs=${GF_PATHS_LOGS}                   \
   cfg:default.paths.plugins=${GF_PATHS_PLUGINS}             \
   cfg:default.paths.provisioning=${GF_PATHS_PROVISIONING}
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now grafana-server
systemctl status grafana-server
journalctl -u grafana-server -f
journalctl -u grafana-server -n 30 --no-pager | grep -i "config\|root_url\|listen"

ALERT_VERSION=0.34.1
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://github.com/prometheus/alertmanager/releases/download/v${ALERT_VERSION}/alertmanager-${ALERT_VERSION}.linux-${ARCH}.tar.gz

tar -zxvf alertmanager-${ALERT_VERSION}.linux-${ARCH}.tar.gz \
    --strip-components=1  \
    -C /usr/local/bin \
    alertmanager-${ALERT_VERSION}.linux-${ARCH}/{alertmanager,amtool}

useradd -r -s /bin/false alertmanager

mkdir -pv /etc/alertmanager /data/alertmanager
chown -R alertmanager:alertmanager /etc/alertmanager /data/alertmanager


cat <<\EOF>  /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 1m
  # 邮件信息
  smtp_from: "[email protected]"
  smtp_smarthost: "smtp.163.com:465"
  smtp_hello: "163.com"
  smtp_auth_username: "[email protected]"
  smtp_auth_password: "Nxxx" # 授权码
  smtp_require_tls: false
route:
  group_by: ['instance', 'cluster']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 10s
  receiver: 'email'
receivers:
  - name: 'email'
    email_configs:
    - to: '[email protected]'
      send_resolved: true
      headers: {Subject: "[WARN] {{ .CommonLabels.alertname }}"}
inhibit_rules:
  - source_matchers: [severity="critical"]
    target_matchers: [severity="warning"]
    equal: [alertname, dev, instance]
EOF

# 语法检测
amtool check-config /etc/alertmanager/alertmanager.yml

cat >/lib/systemd/system/alertmanager.service <<\EOF
[Unit]
Description=Alertmanager Server
After=network.target

[Service]
Type=simple
User=grafana
Group=users
ExecStart=/usr/local/bin/alertmanager  \
   --config.file=/etc/alertmanager/alertmanager.yml \
   --storage.path=/data/alertmanager \
   --cluster.advertise-address=0.0.0.0:9093 \
   --data.retention=240h \
   --web.route-prefix=/ \
   --web.external-url=https://alert.jasper.org
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now alertmanager
systemctl status alertmanager
journalctl -u alertmanager -f
journalctl -u alertmanager -f | grep -i "notify\|smtp"

# 重新加载配置
curl -X POST http://localhost:9093/-/reload

# 调用 api
curl --location --request POST 'http://10.103.236.201:9093/api/v2/alerts' \
--header 'Content-Type: application/json' \
--data-raw '[
    {
        "labels": {
            "alertname":"testAlert",
            "severity": "critical",
            "duty": "infra_op|fsy_rd"
        },
        "annotations": {
            "info": "The disk sda1 is running full2",
            "summary": "please check the instance example1"
        }
    }
]'

# 或者使用 amtool
amtool alert add alertname=TestEmail severity=critical service=inhouse-service \
    --annotation=summary="邮件通道测试" \
    --alertmanager.url=http://localhost:9093

# 添加告警规则文件
mkdir -p /etc/loki/rules/fake
cat >/etc/loki/rules/fake/alert-rule-demo1.yml <<EOF
groups:
  - name: http_response
    interval: 10s
    rules:
      - alert: HighPercentageError
        expr: |-
          sum(rate({filename="/var/log/flog/access_json.log", status=~"(4|5).."}[1m]))
          /
          sum(rate({filename="/var/log/flog/access_json.log"}[1m])) > 0.5
        for: 10s
        labels:
          serverty: warning
        annotations:
          summary: High request latency
  - name: nginx_availability
    interval: 10s
    rules:
      - alert: NginxToraffic
        expr: |
          absent_over_time({container="calico-node"}[1m])
        for: 10s
        labels:
          serverty: critical
          service: nginx
        annotations:
          summary: 'nginx 服务无日志'
          description: |
            nginx 1 分钟没有产生访问日志，
            可能服务异常或无法访问。
EOF

# 规则文件自动加载，无需重启 loki 服务，可能通过 API 查看告警规则生效
curl 10.103.236.201:3100/loki/api/v1/rules