Настройка Prometheus и Grafana

sudo -i
apt update && apt upgrade -y
reboot
 
apt install mc htop tcpdump curl wget traceroute net-tools iperf3 iptraf tree -y
 
nano ~/.bashrc
# Tuning Command Line History (only current user)
export HISTSIZE=10000
export HISTTIMEFORMAT="%h %d %H:%M:%S "
PROMPT_COMMAND='history -a'
export HISTIGNORE="ls:ll:history:w:htop:pwd:mc:docker ps"

source ~/.bashrc

ufw allow 9090,9093,9094,9100/tcp
ufw allow 9094/udp
ufw allow ssh
ufw enable

# https://prometheus.io/download/ -> linux -> amd64
wget https://github.com/prometheus/prometheus/releases/download/v3.8.0/prometheus-3.8.0.linux-amd64.tar.gz

mkdir /etc/prometheus /var/lib/prometheus
tar -zxf prometheus-*.linux-amd64.tar.gz
cd prometheus-3.8.0.linux-amd64/
cp prometheus promtool /usr/local/bin/
cp prometheus.yml /etc/prometheus
cd .. && rm -rf prometheus-*.linux-amd64/ && rm -f prometheus-*.linux-amd64.tar.gz

# создаем пользователя prometheus без домашней директории и без возможности входа в консоль сервера
useradd --no-create-home --shell /bin/false prometheus
chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus
chown prometheus:prometheus /usr/local/bin/{prometheus,promtool}

sudo -u prometheus /usr/local/bin/prometheus --config.file /etc/prometheus/prometheus.yml --storage.tsdb.path /var/lib/prometheus/

# time=2025-12-10T17:40:57.068+03:00 level=INFO source=main.go:1286 msg="Server is ready to receive web requests."

[Unit]
Description=Prometheus Service
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
 --config.file /etc/prometheus/prometheus.yml \
 --storage.tsdb.path /var/lib/prometheus/
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl enable prometheus
systemctl start prometheus
systemctl status prometheus

# https://prometheus.io/download/
wget https://github.com/prometheus/alertmanager/releases/download/v0.29.0/alertmanager-0.29.0.linux-amd64.tar.gz

mkdir -p /etc/alertmanager /var/lib/prometheus/alertmanager
tar -zxf alertmanager-*.linux-amd64.tar.gz
cd alertmanager-0.29.0.linux-amd64/
cp alertmanager amtool /usr/local/bin/
cp alertmanager.yml /etc/alertmanager
cd .. && rm -rf alertmanager-*.linux-amd64/ && rm -f alertmanager-*.linux-amd64.tar.gz

# создаем пользователя alertmanager без домашней директории и без возможности входа в консоль сервера
useradd --no-create-home --shell /bin/false alertmanager
chown -R alertmanager:alertmanager /etc/alertmanager /var/lib/prometheus/alertmanager
chown alertmanager:alertmanager /usr/local/bin/{alertmanager,amtool}

[Unit]
Description=Alertmanager Service
After=network.target

[Service]
EnvironmentFile=-/etc/default/alertmanager
User=alertmanager
Group=alertmanager
Type=simple
ExecStart=/usr/local/bin/alertmanager \
          --config.file=/etc/alertmanager/alertmanager.yml \
          --storage.path=/var/lib/prometheus/alertmanager \
          --cluster.advertise-address=0.0.0.0:9093 \
          $ALERTMANAGER_OPTS
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl enable alertmanager
systemctl start alertmanager
systemctl status alertmanager

# https://prometheus.io/download/
wget https://github.com/prometheus/node_exporter/releases/download/v1.10.2/node_exporter-1.10.2.linux-amd64.tar.gz

tar -zxf node_exporter-*.linux-amd64.tar.gz
cd node_exporter-1.10.2.linux-amd64/
cp node_exporter /usr/local/bin/
cd .. && rm -rf node_exporter-*.linux-amd64/ && rm -f node_exporter-*.linux-amd64.tar.gz

# создаем пользователя nodeuser без домашней директории и без возможности входа в консоль сервера
useradd --no-create-home --shell /bin/false nodeuser
chown -R nodeuser:nodeuser /usr/local/bin/node_exporter

[Unit]
Description=Node Exporter Service
After=network.target

[Service]
User=nodeuser
Group=nodeuser
Type=simple
ExecStart=/usr/local/bin/node_exporter
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl enable node_exporter
systemctl start node_exporter
systemctl status node_exporter

scrape_configs:
  ... 
  - job_name: 'linux_exporter_clients'
    scrape_interval: 5s
    static_configs:
      - targets:
          - localhost:9100
          - 10.10.1.9:9100

groups:
- name: alert.rules
  rules:
  - alert: InstanceDown
    expr: up == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      description: '{{ $labels.instance }} of job {{ $labels.job }} has been down
        for more than 1 minute.'
      summary: Instance {{ $labels.instance }} down

...
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"
  - "alert.rules.yml"
...

apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/grafana-enterprise/release/12.3.0/grafana-enterprise_12.3.0_19497075765_linux_amd64.deb
dpkg -i grafana-enterprise_12.3.0_19497075765_linux_amd64.deb

systemctl start grafana-server
systemctl enable grafana-server
systemctl status grafana-server

ufw allow 3000/tcp