LMBenchmark/run_benchmarks.sh at main · LMCache/LMBenchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/bin/bash

# Get the directory where this script is located
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [[ $# -lt 3 ]]; then
    echo "Usage: $0 <model> <base url> <save file key> [scenarios...] [qps_values...]"
    echo ""
    echo "Scenarios:"
    echo "  sharegpt        - ShareGPT benchmark"
    echo "  short-input     - Short input, short output benchmark"
    echo "  long-input      - Long input, short output benchmark"
    echo "  long-long       - Long input, long output benchmark"
    echo "  all             - Run all benchmarks"
    echo ""
    echo "Examples:"
    echo "  # Run all benchmarks with default QPS"
    echo "  $0 meta-llama/Llama-3.1-8B-Instruct http://localhost:8000 /mnt/requests/benchmark all"
    echo ""
    echo "  # Run specific benchmarks with custom QPS"
    echo "  $0 meta-llama/Llama-3.1-8B-Instruct http://localhost:8000 /mnt/requests/benchmark sharegpt short-input 1.34 2.0 3.0"
    exit 1
fi

MODEL=$1
BASE_URL=$2
KEY=$3

# Parse scenarios and QPS values
SCENARIOS=()
QPS_VALUES=()
found_scenarios=true

for arg in "${@:4}"; do
    if [[ "$arg" == "sharegpt" || "$arg" == "short-input" || "$arg" == "long-input" || "$arg" == "long-long" || "$arg" == "all" ]]; then
        SCENARIOS+=("$arg")
    else
        found_scenarios=false
        QPS_VALUES+=("$arg")
    fi
done

# If no scenarios specified, default to all
if [ ${#SCENARIOS[@]} -eq 0 ]; then
    SCENARIOS=("all")
fi

# If no QPS values specified, use defaults for each scenario
if [ ${#QPS_VALUES[@]} -eq 0 ]; then
    QPS_VALUES=()
fi

# Function to run ShareGPT benchmark
run_sharegpt() {
    echo "Running ShareGPT benchmark..."
    if [ ${#QPS_VALUES[@]} -eq 0 ]; then
        "${SCRIPT_DIR}/sharegpt/run.sh" "$MODEL" "$BASE_URL" "${KEY}_sharegpt"
    else
        "${SCRIPT_DIR}/sharegpt/run.sh" "$MODEL" "$BASE_URL" "${KEY}_sharegpt" "${QPS_VALUES[@]}"
    fi
}

# Function to run short input benchmark
run_short_input() {
    echo "Running short input benchmark..."
    if [ ${#QPS_VALUES[@]} -eq 0 ]; then
        "${SCRIPT_DIR}/synthetic-multi-round-qa/short_input_short_output.sh" "$MODEL" "$BASE_URL" "${KEY}_short_input"
    else
        "${SCRIPT_DIR}/synthetic-multi-round-qa/short_input_short_output.sh" "$MODEL" "$BASE_URL" "${KEY}_short_input" "${QPS_VALUES[@]}"
    fi
}

# Function to run long input benchmark
run_long_input() {
    echo "Running long input benchmark..."

    # Then run the actual benchmark
    if [ ${#QPS_VALUES[@]} -eq 0 ]; then
        "${SCRIPT_DIR}/synthetic-multi-round-qa/long_input_short_output_run.sh" "$MODEL" "$BASE_URL" "${KEY}_long_input"
    else
        "${SCRIPT_DIR}/synthetic-multi-round-qa/long_input_short_output_run.sh" "$MODEL" "$BASE_URL" "${KEY}_long_input" "${QPS_VALUES[@]}"
    fi
}

# Function to run long-long benchmark
run_long_long() {
    echo "Running long-long benchmark..."
    if [ ${#QPS_VALUES[@]} -eq 0 ]; then
        "${SCRIPT_DIR}/synthetic-multi-round-qa/long_input_long_output.sh" "$MODEL" "$BASE_URL" "${KEY}_long_long"
    else
        "${SCRIPT_DIR}/synthetic-multi-round-qa/long_input_long_output.sh" "$MODEL" "$BASE_URL" "${KEY}_long_long" "${QPS_VALUES[@]}"
    fi
}

# Run selected scenarios
for scenario in "${SCENARIOS[@]}"; do
    case "$scenario" in
        "sharegpt")
            run_sharegpt
            ;;
        "short-input")
            run_short_input
            ;;
        "long-input")
            run_long_input
            ;;
        "long-long")
            run_long_long
            ;;
        "all")
            run_sharegpt
            run_short_input
            run_long_input
            ;;
    esac
done