From c8d60478c5dc7c0b3a4b66ea37ff7199bbe7e125 Mon Sep 17 00:00:00 2001
From: Arnav Jain <JainAR1@cardiff.ac.uk>
Date: Thu, 5 Dec 2024 15:41:57 +0000
Subject: [PATCH] setup for kafka, promethus, grafna
---
.DS_Store | Bin 0 -> 6148 bytes
README.md | 21 +++++++++++++++++++++
alerts.yml | 11 +++++++++++
docker-compose.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++
prometheus.yml | 14 ++++++++++++++
5 files changed, 90 insertions(+)
create mode 100644 .DS_Store
create mode 100644 README.md
create mode 100644 alerts.yml
create mode 100644 docker-compose.yml
create mode 100644 prometheus.yml
diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..a81a688b5a712f7595e5234f018b21abc7e30e60
GIT binary patch
literal 6148
zcmeHKyGjE=6g{IUf<cI2VI%AZXdr*EN-QkIB88>qWeg_SsGFdLu!8tI{(@B+!GG`%
zw6xN5X9kl^Otcb_d&Ar_J9E#S-80Ng27uY_*NZ?NK$b-?vqblWNxzgSTktK*L?b!c
z&0@J0b$U~g+Mo)k0)J5fS-Tk&&_oMyYW?m#b4`!>I#H=qZAVqMWpAIF*SHyOrTgo1
z`+Fy2d&B+%smP&*2p#?^=%S7bzDEc3-F8&-BgwWTqZHn)2?;AWW&E-Ahc-i(%s!b7
zoHIiYR}&fT=Vs`U6>;=FbIr(L)+aLL$!Xw>oR+Q0*j>c-3u0U%PQQV9o8ad#u*{el
zDrjTV@&{?wLi1;=)T}+;ZICr1|M(oXhN}mD4rfM>X7u~$)cn4_@Fmp#xB{}iY1y1Z
z^{Rj>pbC5{Ap1kWA{cs19NMje&RzkC88%yET@G#G1Rg_=i9`0#jH41A)%Yuhadfr^
zJ}&f_ICOLvfB7(eWaDor#*WVXfenWV9jaFaRDrYti|#Qm=l{;^`+u6GPpW__@UIjw
znNqp5%}?@Y>(s}|SsSn%v4}}r;?Pp){Bdk6aulzzSYuxx6=LWyamW^${t>V;s8a=g
GRe=vA_n;sE
literal 0
HcmV?d00001
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..593d33e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,21 @@
+{\rtf1\ansi\ansicpg1252\cocoartf2757
+\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\froman\fcharset0 Times-Roman;}
+{\colortbl;\red255\green255\blue255;\red0\green0\blue0;}
+{\*\expandedcolortbl;;\cssrgb\c0\c0\c0;}
+\paperw11900\paperh16840\margl1440\margr1440\vieww11520\viewh8400\viewkind0
+\deftab720
+\pard\pardeftab720\partightenfactor0
+
+\f0\fs24 \cf0 \expnd0\expndtw0\kerning0
+\outl0\strokewidth0 \strokec2 run services\
+docker-compose up -d\
+\
+kafka ui\
+http://localhost:8080\
+\
+prometheus\
+http://localhost:9090\
+\
+\
+grafana\
+http://localhost:3000}
\ No newline at end of file
diff --git a/alerts.yml b/alerts.yml
new file mode 100644
index 0000000..85f7dca
--- /dev/null
+++ b/alerts.yml
@@ -0,0 +1,11 @@
+groups:
+ - name: KafkaAlerts
+ rules:
+ - alert: HighPartitionLag
+ expr: kafka_topic_partition_lag > 100
+ for: 1m
+ labels:
+ severity: critical
+ annotations:
+ summary: "High partition lag on {{ $labels.topic }}"
+ description: "Partition lag is {{ $value }} for topic {{ $labels.topic }}."
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..d26a713
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,44 @@
+version: '3.8'
+
+services:
+ zookeeper:
+ image: confluentinc/cp-zookeeper:latest
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 2181
+ ports:
+ - "2181:2181"
+
+ kafka:
+ image: confluentinc/cp-kafka:latest
+ ports:
+ - "9092:9092"
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ depends_on:
+ - zookeeper
+
+ prometheus:
+ image: prom/prometheus:latest
+ ports:
+ - "9090:9090"
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+
+ grafana:
+ image: grafana/grafana:latest
+ ports:
+ - "3000:3000"
+
+
+ kafka-ui:
+ image: provectuslabs/kafka-ui:latest
+ ports:
+ - "8080:8080"
+ environment:
+ KAFKA_CLUSTERS_0_NAME: local
+ KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+ depends_on:
+ - kafka
diff --git a/prometheus.yml b/prometheus.yml
new file mode 100644
index 0000000..3931a2e
--- /dev/null
+++ b/prometheus.yml
@@ -0,0 +1,14 @@
+global:
+ scrape_interval: 15s
+
+scrape_configs:
+ - job_name: 'kafka'
+ static_configs:
+ - targets: ['kafka:7071']
+
+ - job_name: 'zookeeper'
+ static_configs:
+ - targets: ['zookeeper:7071']
+
+rule_files:
+ - "alerts.yml"
--
GitLab