diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 32e7796..3c47759 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -13883,938 +13883,6 @@ } } }, - "weave-router": { - "metrics": { - "easy": { - "accuracy": 97.1, - "cost": 0.0006, - "robustness": 79.3 - }, - "medium": { - "accuracy": 76.6, - "cost": 0.0011, - "robustness": 78.8 - }, - "hard": { - "accuracy": 38.5, - "cost": 0.0014, - "robustness": 78.9 - }, - "all": { - "accuracy": 78.4, - "cost": 0.0009, - "robustness": 79.0 - } - }, - "categories": { - "Computer science, information, and general works": { - "metrics": { - "easy": { - "accuracy": 95.2, - "cost": 0.0004, - "robustness": 88.5 - }, - "medium": { - "accuracy": 71.2, - "cost": 0.0011, - "robustness": 85.7 - }, - "hard": { - "accuracy": 32.1, - "cost": 0.0022, - "robustness": 73.3 - }, - "all": { - "accuracy": 78.1, - "cost": 0.0009, - "robustness": 83.9 - } - }, - "subcategories": { - "Library and information sciences": { - "metrics": { - "easy": { - "accuracy": 96.7, - "cost": 0.0003, - "robustness": 92.3 - }, - "medium": { - "accuracy": 67.9, - "cost": 0.0007, - "robustness": 100.0 - }, - "hard": { - "accuracy": 11.8, - "cost": 0.0006, - "robustness": 100.0 - }, - "all": { - "accuracy": 83.1, - "cost": 0.0004, - "robustness": 94.7 - } - } - }, - "Computer science, knowledge, and systems": { - "metrics": { - "easy": { - "accuracy": 94.2, - "cost": 0.0004, - "robustness": 84.6 - }, - "medium": { - "accuracy": 72.0, - "cost": 0.0012, - "robustness": 82.4 - }, - "hard": { - "accuracy": 36.5, - "cost": 0.0025, - "robustness": 69.2 - }, - "all": { - "accuracy": 76.1, - "cost": 0.0012, - "robustness": 79.1 - } - } - } - } - }, - "Philosophy and psychology": { - "metrics": { - "easy": { - "accuracy": 96.1, - "cost": 0.0004, - "robustness": 81.5 - }, - "medium": { - "accuracy": 72.5, - "cost": 0.0007, - "robustness": 90.0 - }, - "hard": { - "accuracy": 38.2, - "cost": 0.0012, - "robustness": 85.7 - }, - "all": { - "accuracy": 82.6, - "cost": 0.0006, - "robustness": 84.1 - } - }, - "subcategories": { - "Ethics": { - "metrics": { - "easy": { - "accuracy": 93.5, - "cost": 0.0003, - "robustness": 92.9 - }, - "medium": { - "accuracy": 40.0, - "cost": 0.0007, - "robustness": 100.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0004, - "robustness": 100.0 - }, - "all": { - "accuracy": 73.1, - "cost": 0.0004, - "robustness": 95.2 - } - } - }, - "Philosophy": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0004, - "robustness": 100.0 - }, - "medium": { - "accuracy": 97.4, - "cost": 0.0007, - "robustness": 50.0 - }, - "hard": { - "accuracy": 47.5, - "cost": 0.0014, - "robustness": 66.7 - }, - "all": { - "accuracy": 81.0, - "cost": 0.0009, - "robustness": 75.0 - } - } - }, - "Psychology": { - "metrics": { - "easy": { - "accuracy": 97.5, - "cost": 0.0006, - "robustness": 50.0 - }, - "medium": { - "accuracy": 91.3, - "cost": 0.0009, - "robustness": 0 - }, - "hard": { - "accuracy": 33.3, - "cost": 0.0011, - "robustness": 100.0 - }, - "all": { - "accuracy": 89.7, - "cost": 0.0007, - "robustness": 62.5 - } - } - }, - "Philosophical logic": { - "metrics": { - "easy": { - "accuracy": 98.5, - "cost": 0.0005, - "robustness": 75.0 - }, - "medium": { - "accuracy": 97.3, - "cost": 0.0007, - "robustness": 100.0 - }, - "hard": { - "accuracy": 80.0, - "cost": 0.0015, - "robustness": 0 - }, - "all": { - "accuracy": 97.2, - "cost": 0.0006, - "robustness": 85.7 - } - } - } - } - }, - "Social Science": { - "metrics": { - "easy": { - "accuracy": 98.0, - "cost": 0.0007, - "robustness": 94.4 - }, - "medium": { - "accuracy": 77.5, - "cost": 0.0012, - "robustness": 83.3 - }, - "hard": { - "accuracy": 21.3, - "cost": 0.0013, - "robustness": 71.4 - }, - "all": { - "accuracy": 74.6, - "cost": 0.001, - "robustness": 86.5 - } - }, - "subcategories": { - "Economics": { - "metrics": { - "easy": { - "accuracy": 98.4, - "cost": 0.0006, - "robustness": 91.7 - }, - "medium": { - "accuracy": 81.8, - "cost": 0.0012, - "robustness": 100.0 - }, - "hard": { - "accuracy": 21.3, - "cost": 0.0015, - "robustness": 100.0 - }, - "all": { - "accuracy": 78.1, - "cost": 0.001, - "robustness": 95.2 - } - } - }, - "Law": { - "metrics": { - "easy": { - "accuracy": 95.1, - "cost": 0.0012, - "robustness": 100.0 - }, - "medium": { - "accuracy": 72.9, - "cost": 0.0017, - "robustness": 75.0 - }, - "hard": { - "accuracy": 20.7, - "cost": 0.0014, - "robustness": 50.0 - }, - "all": { - "accuracy": 71.8, - "cost": 0.0014, - "robustness": 77.8 - } - } - }, - "Social sciences, sociology, and anthropology": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0004, - "robustness": 0 - }, - "medium": { - "accuracy": 72.2, - "cost": 0.0005, - "robustness": 50.0 - }, - "hard": { - "accuracy": 25.0, - "cost": 0.0011, - "robustness": 0.0 - }, - "all": { - "accuracy": 45.3, - "cost": 0.0009, - "robustness": 33.3 - } - } - }, - "Social problems": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 100.0 - }, - "medium": { - "accuracy": 44.4, - "cost": 0.0005, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0003, - "robustness": 100.0 - }, - "all": { - "accuracy": 83.6, - "cost": 0.0003, - "robustness": 100.0 - } - } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 94.3, - "cost": 0.0013, - "robustness": 27.3 - }, - "medium": { - "accuracy": 53.3, - "cost": 0.0011, - "robustness": 44.4 - }, - "hard": { - "accuracy": 53.0, - "cost": 0.0011, - "robustness": 84.0 - }, - "all": { - "accuracy": 65.4, - "cost": 0.0012, - "robustness": 62.2 - } - }, - "subcategories": { - "Language": { - "metrics": { - "easy": { - "accuracy": 94.3, - "cost": 0.0013, - "robustness": 27.3 - }, - "medium": { - "accuracy": 53.3, - "cost": 0.0011, - "robustness": 44.4 - }, - "hard": { - "accuracy": 53.0, - "cost": 0.0011, - "robustness": 84.0 - }, - "all": { - "accuracy": 65.4, - "cost": 0.0012, - "robustness": 62.2 - } - } - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0005, - "robustness": 82.1 - }, - "medium": { - "accuracy": 87.1, - "cost": 0.0009, - "robustness": 85.7 - }, - "hard": { - "accuracy": 33.1, - "cost": 0.0016, - "robustness": 88.9 - }, - "all": { - "accuracy": 87.7, - "cost": 0.0008, - "robustness": 84.1 - } - }, - "subcategories": { - "Mathematics": { - "metrics": { - "easy": { - "accuracy": 99.6, - "cost": 0.0004, - "robustness": 81.8 - }, - "medium": { - "accuracy": 91.3, - "cost": 0.0009, - "robustness": 90.9 - }, - "hard": { - "accuracy": 49.2, - "cost": 0.002, - "robustness": 100.0 - }, - "all": { - "accuracy": 90.3, - "cost": 0.0009, - "robustness": 88.9 - } - } - }, - "Earth sciences and geology": { - "metrics": { - "easy": { - "accuracy": 98.8, - "cost": 0.0003, - "robustness": 87.5 - }, - "medium": { - "accuracy": 76.3, - "cost": 0.0006, - "robustness": 100.0 - }, - "hard": { - "accuracy": 25.0, - "cost": 0.0009, - "robustness": 100.0 - }, - "all": { - "accuracy": 92.1, - "cost": 0.0004, - "robustness": 93.3 - } - } - }, - "Biology": { - "metrics": { - "easy": { - "accuracy": 99.1, - "cost": 0.0008, - "robustness": 85.7 - }, - "medium": { - "accuracy": 75.0, - "cost": 0.0011, - "robustness": 0 - }, - "hard": { - "accuracy": 62.5, - "cost": 0.0013, - "robustness": 0 - }, - "all": { - "accuracy": 94.2, - "cost": 0.0009, - "robustness": 85.7 - } - } - }, - "Chemistry": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0007, - "robustness": 100.0 - }, - "medium": { - "accuracy": 93.8, - "cost": 0.001, - "robustness": 50.0 - }, - "hard": { - "accuracy": 36.4, - "cost": 0.0015, - "robustness": 0 - }, - "all": { - "accuracy": 89.5, - "cost": 0.0009, - "robustness": 80.0 - } - } - }, - "Physics": { - "metrics": { - "easy": { - "accuracy": 97.7, - "cost": 0.0004, - "robustness": 66.7 - }, - "medium": { - "accuracy": 89.2, - "cost": 0.0014, - "robustness": 50.0 - }, - "hard": { - "accuracy": 40.0, - "cost": 0.0019, - "robustness": 0 - }, - "all": { - "accuracy": 87.9, - "cost": 0.001, - "robustness": 60.0 - } - } - }, - "Animals (Zoology)": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 50.0 - }, - "medium": { - "accuracy": 80.0, - "cost": 0.0002, - "robustness": 0 - }, - "hard": { - "accuracy": 0, - "cost": 0, - "robustness": 0 - }, - "all": { - "accuracy": 96.0, - "cost": 0.0003, - "robustness": 50.0 - } - } - }, - "Science": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 80.0 - }, - "medium": { - "accuracy": 70.8, - "cost": 0.0004, - "robustness": 0 - }, - "hard": { - "accuracy": 7.8, - "cost": 0.0012, - "robustness": 66.7 - }, - "all": { - "accuracy": 48.6, - "cost": 0.0008, - "robustness": 75.0 - } - } - } - } - }, - "Technology": { - "metrics": { - "easy": { - "accuracy": 96.1, - "cost": 0.0006, - "robustness": 72.1 - }, - "medium": { - "accuracy": 76.9, - "cost": 0.0012, - "robustness": 90.5 - }, - "hard": { - "accuracy": 31.5, - "cost": 0.0014, - "robustness": 85.7 - }, - "all": { - "accuracy": 83.7, - "cost": 0.0009, - "robustness": 78.9 - } - }, - "subcategories": { - "Engineering": { - "metrics": { - "easy": { - "accuracy": 98.4, - "cost": 0.0006, - "robustness": 100.0 - }, - "medium": { - "accuracy": 93.3, - "cost": 0.0015, - "robustness": 92.3 - }, - "hard": { - "accuracy": 67.6, - "cost": 0.0023, - "robustness": 100.0 - }, - "all": { - "accuracy": 92.5, - "cost": 0.0012, - "robustness": 94.7 - } - } - }, - "Medicine and health": { - "metrics": { - "easy": { - "accuracy": 95.3, - "cost": 0.0007, - "robustness": 64.7 - }, - "medium": { - "accuracy": 66.2, - "cost": 0.001, - "robustness": 85.7 - }, - "hard": { - "accuracy": 20.6, - "cost": 0.0011, - "robustness": 83.3 - }, - "all": { - "accuracy": 80.1, - "cost": 0.0008, - "robustness": 70.2 - } - } - }, - "Management and public relations": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 100.0 - }, - "medium": { - "accuracy": 81.8, - "cost": 0.0005, - "robustness": 100.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0003, - "robustness": 0 - }, - "all": { - "accuracy": 94.5, - "cost": 0.0003, - "robustness": 100.0 - } - } - } - } - }, - "Arts & recreation": { - "metrics": { - "easy": { - "accuracy": 98.2, - "cost": 0.0006, - "robustness": 87.5 - }, - "medium": { - "accuracy": 80.2, - "cost": 0.0017, - "robustness": 54.5 - }, - "hard": { - "accuracy": 36.1, - "cost": 0.0026, - "robustness": 80.0 - }, - "all": { - "accuracy": 83.5, - "cost": 0.0014, - "robustness": 70.8 - } - }, - "subcategories": { - "Sports, games and entertainment": { - "metrics": { - "easy": { - "accuracy": 99.2, - "cost": 0.0009, - "robustness": 50.0 - }, - "medium": { - "accuracy": 92.1, - "cost": 0.002, - "robustness": 50.0 - }, - "hard": { - "accuracy": 63.6, - "cost": 0.0035, - "robustness": 50.0 - }, - "all": { - "accuracy": 94.8, - "cost": 0.0018, - "robustness": 50.0 - } - } - }, - "Music": { - "metrics": { - "easy": { - "accuracy": 95.9, - "cost": 0.0004, - "robustness": 100.0 - }, - "medium": { - "accuracy": 63.3, - "cost": 0.002, - "robustness": 80.0 - }, - "hard": { - "accuracy": 27.8, - "cost": 0.0024, - "robustness": 100.0 - }, - "all": { - "accuracy": 75.6, - "cost": 0.0014, - "robustness": 90.9 - } - } - }, - "Arts": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 100.0 - }, - "medium": { - "accuracy": 100.0, - "cost": 0.0005, - "robustness": 0.0 - }, - "hard": { - "accuracy": 32.6, - "cost": 0.0016, - "robustness": 100.0 - }, - "all": { - "accuracy": 79.4, - "cost": 0.0008, - "robustness": 60.0 - } - } - } - } - }, - "Literature": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 100.0 - }, - "medium": { - "accuracy": 92.2, - "cost": 0.0008, - "robustness": 50.0 - }, - "hard": { - "accuracy": 44.2, - "cost": 0.001, - "robustness": 70.8 - }, - "all": { - "accuracy": 58.3, - "cost": 0.0009, - "robustness": 72.4 - } - }, - "subcategories": { - "Literature, rhetoric and criticism": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 100.0 - }, - "medium": { - "accuracy": 92.2, - "cost": 0.0008, - "robustness": 50.0 - }, - "hard": { - "accuracy": 44.2, - "cost": 0.001, - "robustness": 70.8 - }, - "all": { - "accuracy": 58.3, - "cost": 0.0009, - "robustness": 72.4 - } - } - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 98.4, - "cost": 0.0005, - "robustness": 82.6 - }, - "medium": { - "accuracy": 81.1, - "cost": 0.001, - "robustness": 66.7 - }, - "hard": { - "accuracy": 26.2, - "cost": 0.0012, - "robustness": 80.0 - }, - "all": { - "accuracy": 79.0, - "cost": 0.0008, - "robustness": 79.5 - } - }, - "subcategories": { - "Geography": { - "metrics": { - "easy": { - "accuracy": 98.2, - "cost": 0.0004, - "robustness": 87.5 - }, - "medium": { - "accuracy": 75.9, - "cost": 0.0005, - "robustness": 100.0 - }, - "hard": { - "accuracy": 50.0, - "cost": 0.0009, - "robustness": 0 - }, - "all": { - "accuracy": 90.0, - "cost": 0.0004, - "robustness": 88.9 - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 98.7, - "cost": 0.0006, - "robustness": 72.7 - }, - "medium": { - "accuracy": 81.0, - "cost": 0.0012, - "robustness": 60.0 - }, - "hard": { - "accuracy": 24.2, - "cost": 0.0013, - "robustness": 80.0 - }, - "all": { - "accuracy": 73.8, - "cost": 0.0009, - "robustness": 73.1 - } - } - }, - "Biography and genealogy": { - "metrics": { - "easy": { - "accuracy": 97.6, - "cost": 0.0002, - "robustness": 100.0 - }, - "medium": { - "accuracy": 100.0, - "cost": 0.0003, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0002, - "robustness": 0 - }, - "all": { - "accuracy": 96.2, - "cost": 0.0002, - "robustness": 100.0 - } - } - } - } - } - } - }, "sqwish-router": { "metrics": { "easy": { diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index c37adb6..fdf24b1 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -10,17 +10,6 @@ "Accuracy": 76.4, "Cost per 1k": 0.18 }, - { - "Router Name": "Weave Router", - "Arena Score": 76.09, - "Optimal Selection Score": 5.6, - "Optimal Cost Score": 17.19, - "Optimal Acc. Score": 89.21, - "Robustness Score": 79.76, - "Latency Score": null, - "Accuracy": 79.32, - "Cost per 1k": 0.61 - }, { "Router Name": "R2-Router", "Arena Score": 71.6, diff --git a/src/data/routers.json b/src/data/routers.json index 1a3498e..77e3bba 100644 --- a/src/data/routers.json +++ b/src/data/routers.json @@ -12,23 +12,6 @@ "deepseek/deepseek-v4-flash" ] }, - "Weave Router": { - "name": "Weave Router", - "type": "open-source", - "description": "Cluster-routing (v0.62) over a 7-model BYOK pool spanning Anthropic, OpenAI, Google, and OpenRouter providers. Embeds each prompt with Jina v2 INT8 ONNX (768-dim), runs top-p=4 cluster sum against per-cluster rankings trained on the RouterArena full split with k=160 clusters, then selects the cost-quality optimum via an alpha-blended score (alpha=0.25).", - "affiliation": "Weave", - "modelPool": [ - "claude-opus-4-7", - "gpt-5.5", - "gemini-3.1-pro-preview", - "gemini-3.1-flash-lite-preview", - "deepseek/deepseek-v4-pro", - "deepseek/deepseek-v4-flash", - "qwen/qwen3-235b-a22b-2507" - ], - "websiteUrl": "https://weaverouter.com", - "githubUrl": "https://github.com/workweave/router" - }, "R2-Router": { "name": "R2-Router", "type": "open-source", diff --git a/src/data/routers.yaml b/src/data/routers.yaml index cee8bd0..2eb1555 100644 --- a/src/data/routers.yaml +++ b/src/data/routers.yaml @@ -1,18 +1,23 @@ -Weave Router: - name: Weave Router - type: open-source - description: Cluster-routing (v0.62) over a 7-model BYOK pool spanning Anthropic, OpenAI, Google, and OpenRouter providers. Embeds each prompt with Jina v2 INT8 ONNX (768-dim), runs top-p=4 cluster sum against per-cluster rankings trained on the RouterArena full split with k=160 clusters, then selects the cost-quality optimum via an alpha-blended score (alpha=0.25). - affiliation: Weave - modelPool: - - claude-opus-4-7 - - gpt-5.5 - - gemini-3.1-pro-preview - - gemini-3.1-flash-lite-preview - - deepseek/deepseek-v4-pro - - deepseek/deepseek-v4-flash - - qwen/qwen3-235b-a22b-2507 - websiteUrl: https://weaverouter.com - githubUrl: https://github.com/workweave/router +# --- Weave Router temporarily removed from the public leaderboard. --- +# Restore by uncommenting this block AND re-adding the matching entries +# in leaderboard.json, routers.json, and category_scores.json (key: +# "weave-router"). flip_labels/flip_labels_weave-router.json is kept. +# +# Weave Router: +# name: Weave Router +# type: open-source +# description: Cluster-routing (v0.62) over a 7-model BYOK pool spanning Anthropic, OpenAI, Google, and OpenRouter providers. Embeds each prompt with Jina v2 INT8 ONNX (768-dim), runs top-p=4 cluster sum against per-cluster rankings trained on the RouterArena full split with k=160 clusters, then selects the cost-quality optimum via an alpha-blended score (alpha=0.25). +# affiliation: Weave +# modelPool: +# - claude-opus-4-7 +# - gpt-5.5 +# - gemini-3.1-pro-preview +# - gemini-3.1-flash-lite-preview +# - deepseek/deepseek-v4-pro +# - deepseek/deepseek-v4-flash +# - qwen/qwen3-235b-a22b-2507 +# websiteUrl: https://weaverouter.com +# githubUrl: https://github.com/workweave/router RouterDC: name: RouterDC