prometheus remote write storage

prometheusbeat

https://github.com/infonova/prometheusbeat

prometheusbeat.yml

./prometheusbeat.yml
❯ gcat ./prometheusbeat.yml
zgrep '-v' '^[ \t]*#' ./prometheusbeat.yml |   sed 's/[ \t]*#.*//;s/[ \t]*$//;/^[ \t]*$/d'
prometheusbeat:
  listen: ":8080"
  context: "/remote_write_prometheusbeat"
setup.kibana:
output.elasticsearch:
  hosts: ["es.local.org:9123"]
  protocol: "https"
  username: "elastic"
  password: "cs123456"
  ssl.certificate_authorities: ["/opt/ca.crt"]
  ssl.certificate: "/opt/es.crt"
  ssl.key: "/opt/es.key"
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  

prometheus.yml

❯ promtool check config  prometheus.yml
global:
  scrape_interval: 8s     # By default, scrape targets every 15 seconds.
  scrape_timeout: 8s
  evaluation_interval: 5m


remote_write:
  - url: "http://local.org:18080/remote_write_prometheusbeat"
    # write_relabel_configs:
    #   - source_labels: [job]
    #     regex: 'prometheus'
    #     action: keep
    remote_timeout: 30s




scrape_configs:
    # The job name is added as a label `job=` to any timeseries scraped from this config.
   - job_name: "prometheus"
     scrape_interval: 5s
     static_configs:
        - targets: ["localhost:9090"]
   - job_name: 'elasticsearch-monitoring'
     scrape_interval: 10s
     static_configs:
       - targets: ['local.org:9114']
  

docker-prometheusbeat.yaml

prometheusbeat-7.3.1
#prometheus  写入es (prometheusbeat)
---
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:v2.54.1
    container_name: prometheus
    volumes:
      - "./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
      - "/mnt/oss/db/tsdb/prometheus:/prometheus:rw"
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
      - "--web.console.templates=/usr/share/prometheus/consoles" 
      - "--storage.tsdb.retention.time=3d"
      - "--storage.tsdb.retention.size=1GB"
      - "--storage.tsdb.wal-compression"
    healthcheck:
      test: ["CMD-SHELL", "nc -z -v localhost 9090 || exit 1"]
      interval: 5s #健康检查的执行间隔
      timeout: 2s  #健康检查命令的超时时间
      retries: 2 #失败的次数
      start_period: 15s #开始执行健康检查之前的等待时间
    ports:
      - "9090:9090"
    extra_hosts:
       - "local.org:192.168.122.1" # 添加主机名
  ##使用prometheusbeat 把prometheus的数据存储到elasticsearch中
  prometheusbeat:
    image: prometheusbeat:7.3.1
    container_name: prometheusbeat
    ports:
      - 18080:8080
    depends_on:
      - prometheus
    volumes:
      - "./prometheusbeat-7.3.1/prometheusbeat.yml:/prometheusbeat.yml"
      - '/opt/monitor/elasticsearch_exporter-1.7.0/ssl:/opt:ro'
      - "/etc/localtime:/etc/localtime"
    extra_hosts:
       - "local.org:192.168.122.1" # 添加主机名
    healthcheck:
      test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8080" ]
      interval: 3s #健康检查的执行间隔
      timeout: 2s  #健康检查命令的超时时间
      retries: 3 #失败的次数
      start_period: 6s #开始执行健康检查之前的等待时间
  elasticsearch_exporter:
    image: k8s.org/monitor/elasticsearch-exporter:v1.7.0
    container_name: es-exporter
    command:
     - '--es.uri=https://elastic:cs123456@es.local.org:9123'
     - '--es.ca=/opt/ca.crt'
     - '--es.client-private-key=/opt/es.key'
     - '--es.client-cert=/opt/es.crt'
     # - '--es.ssl-skip-verify'  #跳过证书验证
     - '--collector.clustersettings'
     - '--collector.snapshots'
     - '--web.telemetry-path=/metrics'
     - '--web.listen-address=:9114'
     - '--es.timeout=15s'
     - '--es.clusterinfo.interval=3m'
    volumes:
      - '/opt/monitor/elasticsearch_exporter-1.7.0/ssl:/opt:ro'
    extra_hosts:
      - "es.local.org:192.168.122.1" # 添加主机名
    ports:
    - "9114:9114"  

metricbeat

https://github.com/elastic/beats/tree/master/metricbeat

doc https://www.elastic.co/cn/beats/metricbeat

./metricbeat.yml

/metricbeat
❯ gcat ./metricbeat.yml
zgrep '-v' '^[ \t]*#' ./metricbeat.yml |   sed 's/[ \t]*#.*//;s/[ \t]*$//;/^[ \t]*$/d'
metricbeat.config.modules:
  path: ./modules.d/*.yml
  reload.enabled: true
  reload.period: 10s
setup.template.settings:
  index.number_of_shards: 1
  index.codec: best_compression
name: mbeat
setup.kibana:
output.elasticsearch:
  hosts: ["es.local.org:9123"]
  preset: balanced
  protocol: "https"
  username: "elastic"
  password: "cs123456"
  ssl.certificate_authorities: ["/opt/ca.crt"]
  ssl.certificate: "/opt/es.crt"
  ssl.key: "/opt/es.key"
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~
  

prometheus.yml

❯ promtool check config  prometheus.yml
global:
  scrape_interval: 8s     # By default, scrape targets every 15 seconds.
  scrape_timeout: 8s
  evaluation_interval: 5m


remote_write:
  - url: "http://local.org:9201/remote_write"
    # write_relabel_configs:
    #   - source_labels: [job]
    #     regex: 'prometheus'
    #     action: keep
    remote_timeout: 30s




scrape_configs:
    # The job name is added as a label `job=` to any timeseries scraped from this config.
   - job_name: "prometheus"
     scrape_interval: 5s
     static_configs:
        - targets: ["localhost:9090"]
   - job_name: 'elasticsearch-monitoring'
     scrape_interval: 10s
     static_configs:
       - targets: ['local.org:9114']
  

docker-compose.yaml

docker-compose.yaml
---
version: '3.8'
services:
  metricbeat:
    image: docker.elastic.co/beats/metricbeat:8.15.0
 #   image: docker.elastic.co/beats/metricbeat:7.17.1
    container_name: metricbeat
    volumes:
      - ./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
      - ./modules.d:/usr/share/metricbeat/modules.d
      - /opt/monitor/elasticsearch_exporter-1.7.0/ssl:/opt:ro
    ports:
      - 9201:9201
    extra_hosts:
       - "es.local.org:192.168.122.1" # 添加主机名
    command: metricbeat -e -c /usr/share/metricbeat/metricbeat.yml 
  prometheus:
    image: prom/prometheus:v2.54.1
    container_name: prometheus
    volumes:
      - "/mnt/oss/db/tsdb/prometheus:/prometheus:rw"
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--web.enable-admin-api"
      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
      - "--web.console.templates=/usr/share/prometheus/consoles" 
      - "--storage.tsdb.retention.time=3d"
      - "--storage.tsdb.retention.size=1GB"
      - "--storage.tsdb.wal-compression"
    ports:
      - "9090:9090"