From 48fe5993d2210236e1725b72420fd7020f9f1005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=BE=D1=81=D1=82=D0=BE=D0=B2=D0=BE=D0=B8=CC=86=20?= =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=90=D0=BD?= =?UTF-8?q?=D0=B4=D1=80=D0=B5=D0=B5=D0=B2=D0=B8=D1=87?= Date: Wed, 9 Sep 2026 17:12:08 +0300 Subject: [PATCH] [flant] init repo --- .gitignore | 3 + .helm/dashboards/perfomance.json | 1467 ++++++++++++++++++++++++++++++ .helm/dashboards/query-stat.json | 1322 +++++++++++++++++++++++++++ .helm/secret/dev/tls.crt | 1 + .helm/secret/dev/tls.key | 1 + .helm/secret/preprod/tls.crt | 1 + .helm/secret/preprod/tls.key | 1 + .helm/secret/prod/tls.crt | 1 + .helm/secret/prod/tls.key | 1 + .helm/templates/dashboard.yaml | 28 + .helm/templates/ingress.yaml | 44 + .helm/templates/sts.yaml | 129 +++ .helm/values.yaml | 47 + werf.yaml | 9 + 14 files changed, 3055 insertions(+) create mode 100644 .gitignore create mode 100644 .helm/dashboards/perfomance.json create mode 100644 .helm/dashboards/query-stat.json create mode 100644 .helm/secret/dev/tls.crt create mode 100644 .helm/secret/dev/tls.key create mode 100644 .helm/secret/preprod/tls.crt create mode 100644 .helm/secret/preprod/tls.key create mode 100644 .helm/secret/prod/tls.crt create mode 100644 .helm/secret/prod/tls.key create mode 100644 .helm/templates/dashboard.yaml create mode 100644 .helm/templates/ingress.yaml create mode 100644 .helm/templates/sts.yaml create mode 100644 .helm/values.yaml create mode 100644 werf.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6c74682 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.swp +.werf_secret_key +.idea diff --git a/.helm/dashboards/perfomance.json b/.helm/dashboards/perfomance.json new file mode 100644 index 0000000..1e2df5e --- /dev/null +++ b/.helm/dashboards/perfomance.json @@ -0,0 +1,1467 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 26, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Graph: E2E latency over time ", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "End-to-End latency of requests, showing average and key percentiles over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(vllm_e2e_request_latency_seconds_sum{namespace=\"$namespace\"}[$__interval]) / rate(vllm_e2e_request_latency_seconds_count[$__interval])", + "format": "table", + "legendFormat": "E2E Latency", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "99th percentile of End-to-End request latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "P99", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency (P99)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "90th percentile of End-to-End request latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "P90", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency (P90)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average End-to-End request latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "Average", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "(sum(increase(vllm_e2e_request_latency_seconds_sum{namespace=\"$namespace\"}[$__range])) / sum(increase(vllm_e2e_request_latency_seconds_count[$__range])))", + "legendFormat": "Average E2E Latency", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency (Avg)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "50th percentile (median) of End-to-End request latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "P50", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "E2E Latency (P50)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 8, + "panels": [], + "title": "Graph: TTFT(Time To First Token) over time ", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Time to first token (TTFT) latency, showing average and key percentiles over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "rate(vllm_time_to_first_token_seconds_sum{namespace=\"$namespace\"}[$__interval]) / rate(vllm_time_to_first_token_seconds_count[$__interval])", + "format": "table", + "legendFormat": "TTFT (Avg)", + "range": true, + "refId": "A" + } + ], + "title": "TTFT Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "99th percentile of Time To First Token latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "P99", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm_time_to_first_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "TTFT (p99)", + "range": true, + "refId": "A" + } + ], + "title": "TTFT (P99)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "90th percentile of Time To First Token latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "P90", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm_time_to_first_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "TTFT (p90)", + "range": true, + "refId": "A" + } + ], + "title": "TTFT (P90)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average Time To First Token latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "displayName": "Average", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "(sum(increase(vllm_time_to_first_token_seconds_sum{namespace=\"$namespace\"}[$__range])) / sum(increase(vllm_time_to_first_token_seconds_count[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TTFT (Avg)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "50th percentile (median) of Time To First Token latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "P50", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "orietitletChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm_time_to_first_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TTFT (P50)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 7, + "panels": [], + "title": "ITL (Iteration Latency / Time Per Output Token) over time.", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Iteration latency, or average time taken to generate a single output token, with percentiles.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 17, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "editorMode": "code", + "expr": "rate(vllm_request_time_per_output_token_seconds_sum{namespace=\"$namespace\"}[$__interval]) / rate(vllm_request_time_per_output_token_seconds_count[$__interval])", + "hide": false, + "legendFormat": "ITL (Avg)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__interval])))", + "hide": true, + "instant": false, + "legendFormat": "ITL (p50)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__interval])))", + "hide": false, + "instant": false, + "legendFormat": "ITL (p90)", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__interval])))", + "hide": false, + "instant": false, + "legendFormat": "ITL (p99)", + "range": true, + "refId": "D" + } + ], + "title": "ITL (Time Per Output Token) Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "90th percentile of Iteration Latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "ITL (P90)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "99th percentile of Iteration Latency over the selected time range.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "ITL (P99)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average Iteration Latency (time per output token) over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "editorMode": "code", + "expr": "(sum(increase(vllm_request_time_per_output_token_seconds_sum{namespace=\"$namespace\"}[$__range])) / sum(increase(vllm_request_time_per_output_token_seconds_count[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "ITL (Avg)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "50th percentile (median) of Iteration Latency over the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm_request_time_per_output_token_seconds_bucket{namespace=\"$namespace\"}[$__range])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "ITL (P50)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 6, + "panels": [], + "title": "TPS (Tokens Per Second)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of tokens processed per second, including prompt and generation phases.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tokens/sec (tps)" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "rate(vllm_generation_tokens_total{namespace=\"$namespace\"}[$__interval])", + "legendFormat": "Generation TPS", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(vllm_prompt_tokens_total{namespace=\"$namespace\"}[$__interval])", + "hide": false, + "instant": false, + "legendFormat": "Prompt TPS", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(vllm_iteration_tokens_total_count{namespace=\"$namespace\"}[$__interval])", + "hide": false, + "instant": false, + "legendFormat": "Overall Iteration TPS", + "range": true, + "refId": "C" + } + ], + "title": "TPS (Tokens Per Second) Over Time", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "$DS_PROMETHEUS", + "value": "$DS_PROMETHEUS" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "avg : Average\n0.50 : P50\n0.90 : P90\n0.99 : P99\n0.999 : Max (Approx)", + "value": "avg : Average\n0.50 : P50\n0.90 : P90\n0.99 : P99\n0.999 : Max (Approx)" + }, + "hide": 0, + "includeAll": false, + "label": "Aggregation", + "multi": false, + "name": "agg_method", + "options": [ + { + "selected": true, + "text": "avg : Average\n0.50 : P50\n0.90 : P90\n0.99 : P99\n0.999 : Max (Approx)", + "value": "avg : Average\n0.50 : P50\n0.90 : P90\n0.99 : P99\n0.999 : Max (Approx)" + } + ], + "query": "avg : Average\n0.50 : P50\n0.90 : P90\n0.99 : P99\n0.999 : Max (Approx)", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$DS_PROMETHEUS" + }, + "definition": "label_values(vllm_generation_tokens_total,model_name)", + "hide": 0, + "includeAll": true, + "label": "Deployment_ID", + "multi": true, + "name": "Deployment_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vllm_generation_tokens_total,model_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "gpu-prod", + "value": "gpu-prod" + }, + "definition": "label_values(vllm_cache_config_info,namespace)", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vllm_cache_config_info,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "vllm - Performance Statistics", + "uid": "vllm-perfomance", + "version": 243425255, + "weekStart": "" +} diff --git a/.helm/dashboards/query-stat.json b/.helm/dashboards/query-stat.json new file mode 100644 index 0000000..a3f860a --- /dev/null +++ b/.helm/dashboards/query-stat.json @@ -0,0 +1,1322 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "High-level overview of VLLM model deployment behavior and key performance indicators. Designed for Data Scientists and Product Managers to monitor request volume, token throughput, and latency", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 47, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [], + "title": "Request Over Time", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (model_name) (\n rate(vllm_request_success_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])\n)", + "interval": "1", + "legendFormat": "{{model_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Successful Requests Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(vllm_request_success_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests Avg Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calcultaions": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le, model_name) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "p50 Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calculation": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 4 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le, model_name) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "p90 Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calculation": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 4 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, model_name) (rate(vllm_e2e_request_latency_seconds_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "p99 Latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 19, + "panels": [], + "title": "Size Distribution", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (le, model_name) (rate(vllm_request_prompt_tokens_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "{{model_name}} le={{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Input Token Size Distribution", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "calculation ": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 8 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le, model_name) (rate(vllm_request_prompt_tokens_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Input Token Size p90", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calcultion": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 8 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by(le, model_name) (rate(vllm_request_prompt_tokens_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Input Token Size p50", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calcultaion": { + "index": 0, + "text": "mean" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 11 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(vllm_prompt_tokens_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))\n/\nsum(rate(vllm_request_success_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Input Token Size Avg", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calculation": { + "index": 0, + "text": "Last (not null)" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 11 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, model_name) (rate(vllm_request_prompt_tokens_bucket{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Input Token Size p99", + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 18, + "panels": [], + "title": "Input Token Over Time", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 15 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (model_name) (rate(vllm_prompt_tokens_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "{{model_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Input Tokens Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calculation": { + "index": 0, + "text": "mean" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 15 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(vllm_prompt_tokens_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Input Tokens/Sec Avg", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 17, + "panels": [], + "title": "Output Token Over Time", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 22 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (model_name) (rate(vllm_generation_tokens_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "{{model_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Output Tokens Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Calculation": { + "index": 0, + "text": "mean" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 10, + "y": 22 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "v10.4.19+security-01", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(vllm_generation_tokens_total{namespace=\"$namespace\",model_name=~\"$Deployment_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Output Tokens/Sec Avg", + "type": "stat" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default-metrics", + "value": "default-metrics" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "default-metrics" + }, + "definition": "label_values(vllm_request_success_total,model_name)", + "hide": 0, + "includeAll": true, + "label": "Deployment_ID", + "multi": true, + "name": "Deployment_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vllm_request_success_total,model_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "false", + "value": "All hours" + }, + "hide": 2, + "includeAll": false, + "label": "Rush Hours Only", + "multi": false, + "name": "rush_hours", + "options": [ + { + "selected": true, + "text": "false", + "value": "All hours" + }, + { + "selected": false, + "text": "true", + "value": "Rush hours" + } + ], + "query": "false : All hours, true : Rush hours", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "^All__.*$", + "value": "All" + }, + "hide": 2, + "includeAll": false, + "label": "Rush Hours Type", + "multi": false, + "name": "rush_hours_type", + "options": [ + { + "selected": true, + "text": "^All__.*$", + "value": "All" + }, + { + "selected": false, + "text": "^Static__.*$", + "value": "Static" + }, + { + "selected": false, + "text": "^Dynamic__.*$", + "value": "Dynamic" + } + ], + "query": "^All__.*$ : All, ^Static__.*$ : Static, ^Dynamic__.*$ : Dynamic", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "text": "gpu-prod", + "value": "gpu-prod" + }, + "datasource": { + "type": "prometheus", + "uid": "default-metrics" + }, + "definition": "label_values(vllm_cache_config_info,namespace)", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vllm_cache_config_info,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "vllm - Query Statistics", + "uid": "vllm-qeru-stat", + "version": 243438166, + "weekStart": "" +} diff --git a/.helm/secret/dev/tls.crt b/.helm/secret/dev/tls.crt new file mode 100644 index 0000000..508411f --- /dev/null +++ b/.helm/secret/dev/tls.crt @@ -0,0 +1 @@ +1000a87fcc753af125a76627ec9a8b7fd04fd69b2ed69d3c8c88f7d53c61bded472d9d020e6d8bf88426ca9afd8fc764030ce08a9521cafab37337b905a79c8a2c769062e83c3f287440fb533c3ac20b18e9cb63d4bdd01faa76539a9ef23b5b15fe9968dbd45cc1bc8ceaa603eae86a00bb1c514519e9bef2132cc85ebb862e4524ea1280776cfc5539c22b150922a056b6124c659f172d075960f5a659983f9104bb0d939491fa0477a652eea3ecdbf251792792190cf56fdb262954a0109cc3d37e217098a876e9433352e2053a69ac41bf7fd465d40c580ad5704d6361dc274b2a65e854bd4fe7f8fb1925617f68dbb96df1bd76e6115cab1326d05254ce9ff9d45fa08c6d39bf7cb23a9a7901da6534e1b28d7f2b16f38358ccc24224c9427601d8bf2b7df0bdda42571a474f40b50df059480ed9ddb1b49adbac3956c0b8761db814107d1ec3580501fad52866f8b94734e406cc32b2fd1337979c0ed5c460c5bb2039cae318db66151d9261aba3d20e736d30366860e01f8e8c575ac2510ecc0bc68e379f2808407d66839404bdc0d3c26d6f9c33783ee519f72153ffa28791b2e5036f8307b749c537088488a4b46bf9c0b92403de4eaa79b2dca70b549c86b8947187cd922f6a0b95decfa498459786ed11a4e5b440697df8f0e058437e14a25031219909918deb1d8fbf96a10cdcdaf4484872c3296db7885bccfcf42d0b3ef7c6c32869b83a8a1cdbcc87fe852da4e79525c313c23fa00d140e49626eff0cd23d31e4fb5a4f2c1dccc94766e20a39765515fda33d092acd86e84f3b76c266315ba76c7a354f7e9a7e502fc02f72ba59e7df42e4f844ea761622c57851f590b994766885d951da93357dc816d644fb2eb962a630ec4b35330396e2468a5c999060cc6ca6d180ef2ac1ee3ebdc58ee32d29e90675916b0da4e2f6d91ce538fcb4925d0b88b0df7254d4496a1efdd9250c7c58ee49cd4b579e88307d0a0f771fd553e71970afb8062d31e6cac1e7c9f3fad1fd7997721742ff77b59fb67a41677ec72087c74f9f189eb1ede6dfd020b95fe4fc683ffe488dc54f0a7312a4bc2b4df23ddbb7bd55a4b4fa9d04246ce5217f5a86924c0dc9b334739b1498e79b82cd5f6a6f403f3588cd50f1ed423a603a77589fef81623b936b39128e42b98c5e7631b3ee62f04c80b7649cb827ee05f4c4c45de36d6641110f8ce906969c11209535379f0e51082dcec5692b48bae0e96d3e1d081e7ec9aea4902102832a2059a376b851d98761ec6d463e77fe9095e6126f909f525cc1e1ae81d96954ccdf70aa8471dd3e0d66e4a9a630d1a93dbea424622e91641ee113fbe632ebfd643352df0b4818346774dd526441129a1f12be5289c8b6264241d35063d2b00a47e6e55bfa5d9d5175b76343f1b218f2b89664d95a575d26183b84c38ebb1353ff0d93ece8ee3a07512f14d1014082566629c38ae7ea79566fc26ac0c3d7c12a1a5092885897d422e695f5f02f8e5716c58ca655712ada3e91004ccad7ab6cc29f3a305f4536bca4a044f50ee5f4518b9988d9972635ec2bc48a8aed92a2a11ef5cb950c8ba519110a28bbc8f307c6fe5e45b5621836d5c2971523e2396c88747c08db57ae0006c9cf7b7455c10d2729d8b22dc2f20318a296d58dea93112defe3c1be84747af4b367f7a2ecaea0d05214993f3727b7bd9da9bb0d692ef5908422134e37559a7f3dd77d25a4a0e89689ed90fdd9cc102922cfc900d5b05c5c9c98 diff --git a/.helm/secret/dev/tls.key b/.helm/secret/dev/tls.key new file mode 100644 index 0000000..6c8635c --- /dev/null +++ b/.helm/secret/dev/tls.key @@ -0,0 +1 @@ +100090ee0002669c9b739e82d64d14823a3fe5cb96ad17e5890fa1dc9f568c6a319b57b80c5f596bb693a6cf99d029b553a0e2d1d74b451393e8c8504f8905e2536636218a4531eca56be9c0273e5d3e6c60aafcff3a068db0f69913af419108e9a6714616bf9530f25de705efe63c2a9293f4101d441c1e0c1e17f1a3ab57e0bb2c1d2f940761a793eeb4d5f6d3f2387e4617c49bf01dd2ea58433c7e98355fb0bc1b96dd2f2a4a266bcae3715899d138d07ea6cec3c829e7162227d4c1dc9a4b35c62f6ff3a253fd31e9c2bc1daf85aa32f8828115dec8a14990f32689598a2abde8ef51663a1ba584cb3b538548a9d1a174f49c26d6e331709dbadaa75d7a3ce92300708dcbf9c42bb9c9a778ca7ffcee37749596302624d166973a32b909defe1a73329da8ad8dff5114edd18715448e304467311cbf6ed49eba81016402521dc872cad93807b38493e8bc9bd8d6cb5c567246cbd77b91dd92171e0e7abd4ae818912ae5b4fbf15d7fca8d0458cf1efca1c4777b31eb19fe8538a6e631ebb02cc7371b337e6949fcbbcf2eb741d2c2e5d30968c391aee3e0df1c9eccab8504c0344b5adc6809a4d5d7540fae278a145016bb6c67ea40e2b9103dfa87f379937e49273316b9f2510210889ac59f8a4c339df6877d0c8128483edaed673d0c9a2a98c1b4392bd14edddf518dd458da6822adc3515f16dc9a81e41d05c41683df491cbb016e40d04310eabae21de60856069509e0411ecbc6e88dce6e48649c54941eb7b004fda94f9a4df19a935b5f90b4f9b927a4cfd2885e6488df39f2cbd270cd1d1efccae59b689800cdf35571d3f341f558376c4b17b9ae21b678c05ce8f47bb322db3dc19b5d27c3be9b3332ce2c0e918dcf71d4c04a8da7bb045ae1381ee981e7b7e47b9977fefde48977d00d207214a09bdec24be6678770adc42ab94cfc49cc7575e6a68f4b2a5becf25e326cc3beb30bafafee137847d17e49406bee4c84d7d2a31d53260d34efe47d54fd2dde366e732d9d426d68abffcea0f1c645742186cee20966b902cb9702b00d611b79a6ac766407ac5845272fb51b3a925a52d8183574078825e8ba857eba5eef2771e86b723164fdf90bf9029e8bbaa5f4e094afb126660b96bbcefbf4a2e08fd21351a95a1c7d0fd6d6a232a3ff4fbdb54f5c5f0192ccf5a486c2b306d5aeb1b7f7488b9fe46c6b13c8cf0e1c59cecf97087bfa9d6bd6b8035def576eadf1445fd8d12ec3750161fe9b0539a79705a81555b7cc2c65788c78152dd28ac37ff6950a19d696ff5021226d75cd7e1514e503b2a6526ee1075995b7cb7ccdd32df5c7114105b91b7b4e723f8cee7178759c69ad34f5beb9997c65881e3cc06857dc4ba82c237b9b22b3e1e8b1a2e03288c89dcb50ba6769dbc2555f3faf4b77e874e7393abf62f931c93e7c0627b0dce3b8a489410f3d8e83c3d023df610b547631da6031fba093f463a3dadce6f685d681a3035b42d6584660d8d49776780d8834ce63b4e7fae8bf6dd6f666bf72ff9513d7b49fcca6db021f0e069cf6913dfaca4a67125d8743aee511fb7c6422f202ae7073bda0439971f791bcaa2c220c0000bfbd834b7cddfc6b549b0cc93859193dbd31b2b3e98f10316fee0d9bf237ebf4485b3762bcb55170d5fb208ed6f08704e5fa90dd6ad291b4d1adc9fec9fc1d1e76c9c924e1b6784c131119b1ce2ce76413be632b9041bdeddffa4de0bad3abcdf5a37b443a21d813ddba52cd7b69670169cebe0884d1f583312f0b2b158796b88dc3dc7fd014cacfbd9d5a3d3e969021c002048882c3ecdbe3a7c8cf1696d387c0e9cf526c128f38603aefc1156f3a723e5029d8ca8d3f788e39f103b1283cf95216375aa75c9f3910006923665564bad033da332fb944ea89129c5b00685fc1387c91a8b576b4250b99d58265da239916a9031d70c8026ced8df521b268790a2ee4df4d0257d7bf46a641e674adddc15bbb7021cbdcc2a860d1c508dc0c0703bee566c844a152402732414e8c6f42f5ab54742d98c0ecd5c64e61937d3cb531f502db76e419f87ebececef59d6c1b667362bfa236a485a6aa1d6593fe08ba4226aaa7fb37256f916e293ae711814b4ac3f849c5fcf72388f92512be4c0f46c2b7d5facda57642dc68f8423474ac51de2a2def3dfd9ce4ee20a38bae734eb283d438b7b3f87c626094c1227f2c4e48997a2f40b2a8f1ae8545f2246dabcb16deac1d29b2fc3920405fcb5373a2f3e8421cc0b28c6cd916b71dda5fbeacbde380ffece90cec8eaca0ec075411a9b7c4c99d7d6e9d2e219ff9ad2c802d4fd13d336cdf87bf1152a19a6651be9d6852860d764ecc779ceea817d17a3ca7f4be381e5021ec75a48d57241e0ec865ff2f2d17578ee46c42b1d758b1fccdc72d2c805d451283a13246d1b4d4de916fecbaf9465c diff --git a/.helm/secret/preprod/tls.crt b/.helm/secret/preprod/tls.crt new file mode 100644 index 0000000..65c333a --- /dev/null +++ b/.helm/secret/preprod/tls.crt @@ -0,0 +1 @@ +10003e4514bf6986a63de271219426291d677e606094b1f55b00820a4e8c64764f4f517de21d53f0c90cffe0fd690bffdd8dc0427160cdfff5d0d92a855929e89015f1c5b49677e395bc4722ad15b7f9a574343577102f8e432794b0fb70737d9d7473031a5ada4cf062f74bece01bc868c6da42a73b7d9527e129e1824da90962dbe2383eee83176a7817a0cbc8400f223d97b029a8cc1a0b03d3b8d757625854c9ce3c5faf15030d5a9243c625dab5fd75f1ef1ae738ccbd673e697d49acf27189a8eec4b491f199d92a4aa0fe5d961a260675ce3e324f50aadfcea31fa96581af4f098d588d01bdcd2d8d4cd04b42f904e5f6c6422da0dc71d46d6bf98442661e09c0fae772602e84390aa199e77da3b12a2b7f1fea005dd0f95d3bbaf4a6d59b36d6abe241bbc76b2fb6b123d6f4ff1ea0009d1ddc873a30b4cb6c4d0a5bf08019686e9f51cdf5836efa8172dcee3e48fe6fd2d94dc2367b4152e20a9874a7da32555a9228c7bd8a07adc7a6bf4b52307965b61269c7c12a90d806f42adb5c18a99ec213d98f796484e2e54574cff17ab371e5b6cf87cac54de078e044c2f84546a752e1456512acf4314c678af325672293104c85b86442e77ba741e9bf6d5a76a813f479268a120381984a99068c54d3edc144e49e55b40f0a39a2148ff274fccf8959c4fb81b39fce3153c051f391395ce7102f55c08a13c41847d6e53e543fdb070a46c061e60add46f305ffc8356f388d75cfe2edf877dba0a40aa76dc003954639f11f3886532c3dcb4c3ea7cef795367ff1a21419e53a6e911960bead9df1bd80ba9fb49b53dabcdcaf91ba88210629003cfa79a884c8c1cff847e53035c391c03f8c10dc9fd435bd91f8cc9e85919ad2478a26a71fc3a58aecf35d4fd062279871b0b6e2624b5ed99046783fc39fd662b58b0543c2b985224e9a4b0a128e9ffe529d5907f2b197e52a90b8f4cd396917cadfaf0b003adef992a49fac1565fbf6089014fe0581676271d988c45595ea43ff7eb0fe8d41625a0cf50ac53d32e6d03566a03388145bd69bde8bac3604b31fa6063dee0af14ee43b9f150592376d62fa07e61cbf7e1adb1e7e67f6370ed47f683dba928c96532b8c4b25e803674f4baf41bc0bf9cc58e8a661e304b0dc2e1b4b801537502c84b4984c5d95ba46c22491df987fe02d0108a3dcf6e0dcceaebb756e51962c2d4b3a71d409c57ec9d7ca9983d91a665e2e89b3763ab51385b5676b015b3f188dc62d6b11e0a290c6d89354d611f2b93ddf34fcdb32da565f58dfa2c1741aa997849c9b6d6754e5a3907a533ba43c7909438575288cbff0f23ae1a83252b86709b60ca8aa8a0bf9c059b834777297ef7697dc421a06af40086e2e7ce37e064af16142d17fe16bf5b298a2686804cec6ce2949e2285265ffe5b3a0d4bad4c71ecd1f6e2e3c35ff259fe02fa40c7a03d332826acdb0d77131906d284140fb97db73a1f30fffe856e068c6a932fa6a6bd1509bc6cb1472a5823377c4c326ea8eebd3e0b6abd8055d958e392c166600cf672d674d8dd5c1279e891fa519ea84fbb304103de6dfff5081dcddb8c4067d5a2d30ad2a0a4bb614150cde8dba17d015ef6fcd0a8597e9eb6f1866456b2158fb4f705392fc63aeed7a04e95cbf0b108dd7a11562f87ae724bdf60a0962c5f81ca523f0b0c3f059de32b287b05b9444737e135dcd6465bb9ab7d7d781582725da3822502e1da820bd814f7d03f6b4c6c718c9d868e48290fb diff --git a/.helm/secret/preprod/tls.key b/.helm/secret/preprod/tls.key new file mode 100644 index 0000000..6e3965f --- /dev/null +++ b/.helm/secret/preprod/tls.key @@ -0,0 +1 @@ +1000674ba6111c1e8b0fdb20848116dd7d90a24129a422252df47eed1809fb216340162f00894bd7182aca0bd6e7836c2b2e8808923a94f95212c31a1ea9136a06ee5279cddb9273aeb251d2b2ebefca8b2ed245a174038768d051ecf849cc72be32a054ad0c259b05fa7f84f7ecd56b9c240df23ee46612758cf7468ce24e2e49b8edf89e7c764b417116f8ad5ab5c506bda777633b82c566f7b96b26bc92a7c0d3f8084fb43c8d576780998315f6b3ccaf5d62b25dc09f0b342f31d03fd39f849b3c94d06fe0435f24df01fc14a1ebb2874d24f7a4b8b5224a2aa335adf1f0069b13a43fc8fd64f7509f8b530e88baef730c3b799a3913ae4d7d167afa62485f8ca608aaf5e2b40f592ec3f51cf7dae4e4bff932c9080187b345d8b5541ed11a7cb84420fd633499d62acd466154a6aad48057154dd69ef2b4f4dab3127a14ff4c8b6203aae8c3a55722569c9d227b3c79057cef7025024defacd5d86ea44a836cb56a62beb1c7333249dbd851376e3dda9dfb09c83da5f1615c4d788e0efad19b5207e9890c8b13a0176419aa16e884f18d0c62a8b155f865ab5789aa08fbcf010e59b8dbfc7057c86083aea9bd7f31e3077a33af9a6e7b152c799f7b68f78e69a49999b9e63c4dc816e440a8dd11d6fbc1010cfbbcc92f454ca9342dd9abfe732f51bb1ee1af9ec44f848bd414d1554957ec70b120519213fe706af9b745b78b96eb9c259512c60ff955b2ef2e5712ac5cb27ba2ebe8586735a3ce6c0b613ac567be498068569fd5b9795a41b5a06aeea19d821bd7ea19856b6bbe18899ef1d7ca9221e2899b2377dd20c55f9fe8d998482137498dd4845812c344a20718180e5e0811aa8119f2fedf0f4fbfc55d7078ac2620acf7dd1bfbf482f03e33955e3c2d7e29a90fa5f42b3905be1ce5f64c32a52e5dacd7699328ee71cb3c59662317dadfa07b3ab0e4e2df6701e54ba9369e13940cf44402e1c203e06e53c541606718aeb8ad3c019167b2665c6de7dd06eb19753a2de69f2db113e605694915ce1a72120a830a639b5281368bf4a90ca4751a39ba8b104332f8a9ed1610fb31b00cd9bfe07f3eaeaeee179a7b73fbc324768a45edc13ea23211db73ad19adda544cadc29f7513ff6450ab535e9f12e9c18f21d3d3a4bdcfa0b6e1d689daf86da75fefc6f8f919aca91b3d0a453085f1836761986201fc0c51d6834595e7f60c2a8006e91dab1693788bb918e30d45568f2f04b15fa96da15f4ff4f0722c1348007347125bdef39461b254ecb2b1304e143dfcb5578957d2b006119d95149eef55a9ea1f1103ce09aeda156eabd30b666e5d9100a89a2355ead4b988f909ef412acc9eafba1fbf1f603c464e7857a35f9fcf045b925069cb1ff873dfc09b827edd4881683feb1be9b40b4d4e3203053d59ad2715cbc4930df2b2330c89cf597bdf2f3c5505c48a0f26e25342ff7038b974a7f95447c62154a9019047037fb33f0142194f92caef76844d07ef58ff3d94388f2b2a12f73f94d9d805b4800cfa57cc3165c637396ee992f578977e11651ee928d5d3b7f3a4d3b3ba8ff0a68a09ffca9ab9d4dbd564d04439411cd7216e5bbd0f4e5e3df6432187adde3005f9ed8c007dced11dc4f4a2269302cd935bb33eea420be908af21c43a9eb8620143df1a92ad8cab292f724c9d5464eb5f9b0e4fe87d5ef4a1822240c628ea9ad86fd6c9dafc23a4841a33a3deda57a568cb0ea17453c2cb03f817cc61fcf81f85acd636a97c0516a00c888a7f4e1ae25d926df7852fc9ecd1e6b30f13aa91e31f28cc84243546dc50c6f417f4f3b1b4c4c10917942a7c22ff28bc53dbfe2e5cae16a6132c81ba6f9e6da612ba41c3d0d4c309edf6a88b963db4f4c8a64a65d12dce9353208c64d63d9a660de29a9b708349f9fd08c0889867d8774de903530adaaf2b4c72480220f2e48626747f32dabab77dda1fc0cb509a12dc3738f0444ee9e77f7f1e77a17cf1405288419ce01b5c09ff77035e993829b1928e9658ca724e4f1d37d566d8290974342f9c8e9b1a1ea8067202979efa9048d8b14e141bdae1777b09564e1efd6c53333eecae035471ce66376f1f494e8efe3a69e86fc730d338e3efc5302e244afa386d60bccfb3bd711f1f27f9e9612bb9a7628d169547a6671f15f4a05daf6e0d4301a47615bfa5d909e2379ea91da0b95a8a4c2b0bb82b4c4ad11de78c317fa45df6b10f9b2c44aa08b89bfd400fe9f3dbb39e8602edd418e9ac955f758f13ab23f1d2f557d6c39b0f48cee45a1940b88d7f47646280982015dbe3ded6a65bd6c8ed93c24968c47fb93130fa63dffa6233beb75923caa85678ae5fad0348755f1c2fcc08aed56c0fb638024020d2499354707b7263bb9593e990be66d5844bd8829589fe96a7cef259931cbbaea9fffac2fb69d5458da0041d807000 diff --git a/.helm/secret/prod/tls.crt b/.helm/secret/prod/tls.crt new file mode 100644 index 0000000..10982ed --- /dev/null +++ b/.helm/secret/prod/tls.crt @@ -0,0 +1 @@ +100015d0b169a0268951df843788c7bac06c942ce9bda003fd3abea57c6854756666b597ee4db4bb8268818cd39a2310af03e6e7e5373e67a944aa3b189aaa3e68ab14b447926c8c1545f8006ef53b212e7d133b9422f6b2eb31b1b2a0182e830244542fbc780da3f7c1c03bd4db9f2ef95cf2fdb4f6ddcb3f277f26ac807898dc29677e6713a460a8dc6743ccfe1cf303b49bd151949e2d5280e926cfdc3aa6d47f94cd099a325bb6feac45397cb1ca7fcb6ccd2efb7c7c22590e38b4ef096b2064989b0ddfa3480454e9dda52e05bac281d56c09c6d6adc2ef1176464ce62e49023146032d136c4cc71647ace6cb4aa79d1834198035c087d01174b7ae6c52874f01be23a17ca354c436fb9829f248488e1f25003125861d2dbe46eeac3ab0380f09d45bb1ee28182c17d61e8abac8a7493a864c4e1eb9cde25664f87a27c2a5831e07b73ade95f6eed89b2661b203040cbc429e8b9fd808de801e38ee1c66b4f544abcfd578a9d130bba43d320dd3e09ca115f01375d5335fe88ffaf8dda9f8ccd68b5fea7aefa41abd72c0ed15925d8dd483232c5b37f2d8cc52c0cdf5525f7166b5b74f4fc682c90eb721cea324ba7284c23047ab95ab1a60a58210b3853a7bc720115ef6431a8ecc4317cd981c6ed27c21fa0ac4d3bdf9fd6a86968adb1746661fdce84f3b977039d7fc857c394fc0be7309cca257ca52759260d5d7a2971ec3d22cf9770fca0c0050beb8c72484142916c4cc666cc439e268740229433ea5202644f78177fe8171814afeb2f6be614546fc2964d977e71c7e456a7bb168746e0ff966a27e5793ea4c92513934eb8db43f98d2573b7003b1bfdcab73deff7bcf29e0702d930c0ef49541de8ce8ef3a4a58b5fb61d8a35e993458c359fb396ba4e6d0ad2716772d39e9a1d413e44078a18bd4ad4a3283a830c4258ba1a556f73b81c13902f5fb3da2bf3e12fa12be3993384035a031b4341ccc66ce74741418adb4c8701aea3d42d076b412e52dbc2c4963891d49cb1c3cd2f56f7bc3c433575a0ff041ebe1a095f6074944a9f58ff04b1c4cd573c9cbef75aa9cce5a13ee0a57871841f89396cc3aa56cc7120d6793488310ee5f5cd61252643ee0384ba4f27767698e01109f4b39460c8601421ccf260383650e595b1b8bd7aa4eea39165c722720af4587c871de40dda7e0aae3f5af26d2d3702c8980ea655d4b9a85dbde963081be7cc1184015882029880a49d6782edae4dfb45352c6bdc23d9f46683c29c94f1827b00a552c4f2c99073f7b1dee5987f58c0bfa70a1d4450c2aba15ec5dca764b19a41040f34f2229da4e7e8669aec5f48313b28a9a6ada846b955441446773c3ddb680127c50a0dfbf54b0c985ad0d4897164a2e5209d43c4e045c493c9226c18f2ffea28c56bcf74240a642a5141f292115f0348175b4ee28cc6fc5ad7ca09d8163d192eb34b994ae64e93a0bfbf00f8125645556f9a52c74e79c5a98c249ecf7e98bdf4b93e78e0801a982d54570f2cd082fb00b2b201db9ee3b108e8ed2915cdd5d67cf7a1340f075f0b09687b3ac0f4aec10ea83e71c07c6580a892fbbdfcf74333ac4bb32109ad7ba852cdba3f411acda807a127586ba38536f7e499b65c75d866b149de1c115959e90f197d9074849031e9b7e4528a4599d30581223c36b501e5d65358a1a6abe7c7c32eb4edbf82cbbe40182488826d890da88c77d1b473b56ba847947cd3648e3921fd8b08da92645d445332c94d07f6dc2 diff --git a/.helm/secret/prod/tls.key b/.helm/secret/prod/tls.key new file mode 100644 index 0000000..8ddd273 --- /dev/null +++ b/.helm/secret/prod/tls.key @@ -0,0 +1 @@ +1000b27f4a8441c1f70f20fc5b84c97a10b05979049b958d7761cf391ee3e4928a14751308ed9d92a819d59e2f0375bbf4bf71a2fc123507e29fa4004079c3748c156abba064bc0c508a9efb092fde86c1986601b05250aaa4b03fb344976217c0124c1f194e39ce18ae5e7048ce0470917e8cf6b4914597d824aa7077aac46f888d1f2e55189cafc347dd4cd0b201d1d3cc5c82e1471111b3906913ec60c2d5bcde7a7c2ccf9de0556b1be243ce2407f3d6ceb91c97887e0ac9e3e4d726ef386528b5c32f596e594f09b0225a61583b407f88aaf183000b34fdd0c559427aaccef413a92d2b1cc87835802d0eac23ada23ed4fe3036847b59853cb320c37d6d628f04075c1a54ad276609e34eb10e121dd6932e81abf3cb97508a1b0903fa2f447cbe57ce17c090db1baecfb86a85b1acf1a2f861393fbf5cdae3ea8b31975968939e3e10047b9090179cb62fe249a2a968eecd4471423cc59e720fc3e4a148884394023e264c24204ca04adfad70de9499338da9ca064db3e655d653fad7fcb85a3eb7cf6bb4b249569eca8c836afd57c34ad643a3be9b519631a40d8d8662320da872a6192a7ee309b6e2d690b04a563f1a910e688f38495d4a85aad1ab8bc9ec7d1a39f4b721d800c6d3855c750af41c966b88c47f47d102316a7bafd884deb530e3e4722dbc60b659a6fb4dc6e3978b68bcc935fea4c8ab7ea454cab0fed06d7dde601993c44ba05e5359e4a78fb6d9084df4d4308fcde565b6872b37f17af9f0c8e6204637b033d43d7bd85e3e1bc549434a5d15445b6a0bbcfe843ae7afdb17903f76e13f5f2b1615e5ab16e9e4a94348780040fa4f9bff3a5f7ce8b8f976507e317893c5d4e1165d21dce280efa3610cfea8b98b68c71e9e216fc42aa6e67d80e5d003f0219737668b9d0b814af7d8d17d4ce346893c9ad3e11c9a5974de5c8526519b8857495ae59c85520bf2b52216cdc438a3d76b780b9dd923ba7a50ba7771d9bb131a80073097129f4fd6990c3af98a4b5773ed6c0c0e401d21685d3ffeadf10f24f5d4cca95890f065b1ccf54f9a2679c3d7453d1daec34e590353f571f91f43f8f6c8761772e6bab7fb5b166eda680306723c91dfad236c35cf5042df7dc5a3cc319223dcd8c608e9cd53bdb103a427e3018a366e24c2bc1d05cf7a0aa2e01c54b3778b8856816fec893b4e35d65409eb588aa0f9137c6bec5b69f9ef7083e3dd1a655875695290dd4b9c0533d828d6f042dd0516e58c63f4e0ddf21023ecba4f329fad80bb78bff696b4886ec3056955e8c3fe75a4045674b3c80d3a56b95660d016de348c5acff3ead9a768846b9db937bbbeff9a7f3d28afd8b537b7f8be52a477b745df503fbd3f9b9ef2b7c57b8284083168932699c269c713771c5dd19eacb6030976a66b0a0e0039efb9ce8fae8c45dacc974cb8edbc4b4e926ceb0142a5f75dd612369e8ea681fcbac7df4e332d1cc5193aa7834839fb871d44682146109a85b2ed6f94b98fbf89fdd3ce8584ee534f61e31079f0099d39a22228c76a6cf0d1ad8b034ed4d1e0dd264826f403091279b1a39164034e807735ed46050e3159e5e15c53f1b2218953625243dbf69b604309f143fe1f934c4a98981846e95df5853e6a07398d476c5ddd28ef05f05a0082c51173abb617cd1cab5a65502b3ec1935e721316e6dbb4ded886b0a26aa236dafefa884dd36a9829fea720afea9b9d91ffd61edcaf956b24a8b21e2d04a2b969a86c32d5c07a0080c51c69bbb54cf1145ebd21d8421de8d55f136710e4e2b440d4f5469770963c9a9d3203e59c7b3224b2697036087fe77593220632c3ec1a6cd4b584169ded59176012ad04d6a935115bde6ed134ce3f5f0ed108a5ea99cf75a8c028a4e17ec6d92b5a93116f34fe88c00b31ae1bccc63610f343cb43a8aaf53a22c6d36443909f2e97177324cf498cc8c3f9d9bd982975aacdc0e18806db88dcb6a5f595e00f33f366a846de0c21aacbc8639d3bbb29ddfa9364b382de0b05b539cea03151b4c49d09ffdfc6e6bf886b641b7a39888f5c0c15bb0ca51c393179d3e0561f90b88e3c5530c9d22c9313b691512b7ee08f022275db6d86a00ad3b44188b377fb0a76dec254d582314a1059c8732dc43319f427c3e9522bc369797077318aff92200c529896ae0428672edce80103f15646f698f4ca63bfd8c99008cb3f76f13c7d9523396100205360af6a4a009f23fc78a8fbbaf474abb0f78247de21b4abfe713628240604cb34baff8a4c1c40151c0826e43ab5408b25edb653811f1a652641a18e5a8969ba071353591a5b6f8d755a3c31b379b7eb779e688a3d558d5e26b89593d8a4fc51e17b858996a83ce6bdb42f5c559e5b651b9dfc98fe293f40670bdf38067aa17053736357aef320ac717b640b6f9c53375d8adbaa2fcb16d0101e diff --git a/.helm/templates/dashboard.yaml b/.helm/templates/dashboard.yaml new file mode 100644 index 0000000..e2ef7ac --- /dev/null +++ b/.helm/templates/dashboard.yaml @@ -0,0 +1,28 @@ +{{- if eq .Values.werf.env "prod" }} +{{- range $path, $_ := .Files.Glob "dashboards/*.json" }} + {{- $raw := $.Files.Get $path }} + {{- $json := $raw | fromJson }} + {{- $filename := base $path }} + {{- $name := trimSuffix ".json" $filename }} +--- +apiVersion: observability.deckhouse.io/v1alpha1 +kind: ClusterObservabilityDashboard +metadata: + name: {{ $.Chart.Name }}-{{ $name }} + annotations: + metadata.deckhouse.io/title: {{ $json.title | quote }} + metadata.deckhouse.io/category: Applications +spec: + definition: | + {{- $raw | nindent 4 }} +--- +apiVersion: deckhouse.io/v1alpha1 +kind: GrafanaDashboardDefinition +metadata: + name: {{ $.Chart.Name }}-{{ $name }} +spec: + folder: Applications + definition: | + {{- $raw | nindent 4 }} +{{- end }} +{{- end }} diff --git a/.helm/templates/ingress.yaml b/.helm/templates/ingress.yaml new file mode 100644 index 0000000..ec6731f --- /dev/null +++ b/.helm/templates/ingress.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Chart.Name }} + labels: + app: {{ .Chart.Name }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-body-size: 1024m + nginx.ingress.kubernetes.io/proxy-connect-timeout: "10000" + nginx.ingress.kubernetes.io/proxy-send-timeout: "10000" + nginx.ingress.kubernetes.io/proxy-read-timeout: "10000" + nginx.ingress.kubernetes.io/upstream-keepalive-timeout: "10000" +spec: + ingressClassName: {{ dig .Values.werf.env .Values.ingress.className._default .Values.ingress.className | quote }} + tls: + - hosts: + - {{ dig .Values.werf.env .Values.ingress.host._default .Values.ingress.host | quote }} + secretName: {{ dig .Values.werf.env .Values.ingress.secretName._default .Values.ingress.secretName | quote }} + rules: + - host: {{ dig .Values.werf.env .Values.ingress.host._default .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Chart.Name }} + port: + name: http +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ dig .Values.werf.env .Values.ingress.secretName._default .Values.ingress.secretName }} + annotations: + werf.io/sensitive: "true" +type: kubernetes.io/tls +stringData: + tls.crt: | + {{- werf_secret_file (printf "%s/%s" .Values.werf.env "tls.crt") | nindent 4 }} + tls.key: | + {{- werf_secret_file (printf "%s/%s" .Values.werf.env "tls.key") | nindent 4 }} diff --git a/.helm/templates/sts.yaml b/.helm/templates/sts.yaml new file mode 100644 index 0000000..a108dfa --- /dev/null +++ b/.helm/templates/sts.yaml @@ -0,0 +1,129 @@ +{{- $gpuRole := .Values.gpu.nodeRoleKey }} +{{- $tolKey := .Values.gpu.tolerationKey }} +{{- $tolVal := .Values.gpu.tolerationValue }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Chart.Name }} + labels: + app: {{ .Chart.Name }} +spec: + serviceName: {{ .Chart.Name }} + minReadySeconds: 10 + updateStrategy: + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: {{ .Chart.Name }} + topologyKey: kubernetes.io/hostname + weight: 100 + {{- if $gpuRole }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ $gpuRole | quote }} + operator: Exists + {{- end }} + {{- if and $tolKey $tolVal }} + tolerations: + - key: {{ $tolKey | quote }} + operator: Equal + value: {{ $tolVal | quote }} + effect: NoExecute + {{- end }} + imagePullSecrets: + - name: registrysecret + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.werf.image.vllm }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /root/.cache/huggingface + ports: + - name: http + containerPort: {{ .Values.port }} + protocol: TCP + args: + - {{ .Values.env.VLLM_MODEL }} + - --async-scheduling + - --host + - 0.0.0.0 + - --port + - "{{ .Values.port }}" + - --gpu-memory-utilization + - "0.2" + - --max-model-len + - "{{ .Values.env.VLLM_MAX_MODEL_LEN }}" + - --enforce-eager + - --tensor-parallel-size + - "1" + - --trust-remote-code + - --chat-template-content-format + - "string" + env: + {{- range $n, $v := .Values.env }} + - name: {{ $n }} + value: {{ $v | quote }} + {{- end }} + livenessProbe: + httpGet: + path: /health + port: {{ .Values.port }} + initialDelaySeconds: 120 + periodSeconds: 9 + timeoutSeconds: 6 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v1/models + port: {{ .Values.port }} + initialDelaySeconds: 120 + periodSeconds: 5 + timeoutSeconds: 6 + failureThreshold: 3 + resources: {{ dig .Values.werf.env .Values.resources._default .Values.resources | toYaml | nindent 12 }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ dig .Values.werf.env .Values.storage.class._default .Values.storage.class | quote }} + resources: + requests: + storage: {{ dig .Values.werf.env .Values.storage.size._default .Values.storage.size | quote }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + labels: + app: {{ .Chart.Name }} + prometheus.deckhouse.io/custom-target: {{ .Chart.Name }} + annotations: + prometheus.deckhouse.io/port: "{{ .Values.port }}" + prometheus.deckhouse.io/path: "/metrics/" +spec: + type: ClusterIP + ports: + - port: {{ .Values.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Chart.Name }} diff --git a/.helm/values.yaml b/.helm/values.yaml new file mode 100644 index 0000000..139db74 --- /dev/null +++ b/.helm/values.yaml @@ -0,0 +1,47 @@ +port: 8004 +env: + VLLM_MODEL: kristaller486/dots.ocr-1.5 + VLLM_MAX_TOKENS: 24000 + VLLM_TEMPERATURE: 0.2 + VLLM_TOP_P: 0.9 + VLLM_REQUEST_TIMEOUT: 3000.0 + VLLM_MAX_MODEL_LEN: 32768 + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_LOGGING_LEVEL: INFO + HF_HUB_ENABLE_HF_TRANSFER: 1 + HF_HOME: /root/.cache/huggingface + VLLM_DOWNLOAD_DIR: /root/.cache/huggingface + CUDA_LAUNCH_BLOCKING: 0 + LOG_LEVEL: INFO + VLLM_PORT: 8005 + TRANSFORMERS_OFFLINE: 1 + +gpu: + nodeRoleKey: node-role.deckhouse.io/worker + tolerationKey: dedicated + tolerationValue: "" + +resources: + _default: + requests: + cpu: "1" + memory: "10Gi" + limits: + memory: "10Gi" + nvidia.com/gpu: 1 + +storage: + size: + _default: 10Gi + class: + _default: local-storage-class-worker + +ingress: + className: + _default: nginx + host: + _default: vllm.pro.lukoil.com + preprod: vllm.preprod.pro.lukoil.com + prod: vllm.prod.pro.lukoil.com + secretName: + _default: vllm-tls diff --git a/werf.yaml b/werf.yaml new file mode 100644 index 0000000..c14eab9 --- /dev/null +++ b/werf.yaml @@ -0,0 +1,9 @@ +--- +configVersion: 1 +project: vllm +deploy: + namespace: '[[ env ]]' + helmRelease: '[[ project ]]-[[ env ]]' +--- +image: vllm +from: registry.lukoil.ru:8123/external-pub/vllm/vllm-openai:v0.14.0