diff --git a/models/ref/sdk-coding-cheat-sheet.mdx b/models/ref/sdk-coding-cheat-sheet.mdx index a22764cd84..5e7ddfa7cd 100644 --- a/models/ref/sdk-coding-cheat-sheet.mdx +++ b/models/ref/sdk-coding-cheat-sheet.mdx @@ -12,9 +12,6 @@ Select a card to view code examples in that category. Code examples to initialize, manage, and fork W&B runs - - Code examples to configure, create, and run hyperparameter sweeps with W&B - Code examples to log metrics, hyperparameters, tables, and custom data to W&B diff --git a/models/ref/sdk-coding-cheat-sheet/artifacts.mdx b/models/ref/sdk-coding-cheat-sheet/artifacts.mdx index df17e04b3b..1090e498d7 100644 --- a/models/ref/sdk-coding-cheat-sheet/artifacts.mdx +++ b/models/ref/sdk-coding-cheat-sheet/artifacts.mdx @@ -75,3 +75,4 @@ Create, update, download, and manage W&B Artifacts for data versioning. ## Given an existing artifact, update its description, metadata, and aliases without creating a new run + diff --git a/models/ref/sdk-coding-cheat-sheet/logging.mdx b/models/ref/sdk-coding-cheat-sheet/logging.mdx index 6a34821596..85fd10183c 100644 --- a/models/ref/sdk-coding-cheat-sheet/logging.mdx +++ b/models/ref/sdk-coding-cheat-sheet/logging.mdx @@ -60,3 +60,4 @@ Log metrics, hyperparameters, tables, and custom data to W&B. ## Log a table + diff --git a/models/ref/sdk-coding-cheat-sheet/registry.mdx b/models/ref/sdk-coding-cheat-sheet/registry.mdx index 5d46a5be97..3eb367bd5b 100644 --- a/models/ref/sdk-coding-cheat-sheet/registry.mdx +++ b/models/ref/sdk-coding-cheat-sheet/registry.mdx @@ -45,3 +45,4 @@ Work with W&B Model Registry to organize and manage model versions. ## Remove a tag from a collection in a registry + diff --git a/models/ref/sdk-coding-cheat-sheet/runs.mdx b/models/ref/sdk-coding-cheat-sheet/runs.mdx index 010b1aa5ae..455341d243 100644 --- a/models/ref/sdk-coding-cheat-sheet/runs.mdx +++ b/models/ref/sdk-coding-cheat-sheet/runs.mdx @@ -55,3 +55,4 @@ Initialize and manage W&B runs to organize your experiments and track your work. ## Add one or more tags to previously saved runs + diff --git a/snippets/CodeSnippet.jsx b/snippets/CodeSnippet.jsx index 19f9e836b5..adcf2c4b03 100644 --- a/snippets/CodeSnippet.jsx +++ b/snippets/CodeSnippet.jsx @@ -12,7 +12,6 @@ * AUTO-GENERATED: Do not edit manually. Run sync_code_examples.sh to regenerate. */ - // Import all MDX-wrapped code examples import ArtifactAddAlias from '/snippets/_includes/code-examples/artifact_add_alias.mdx'; import ArtifactAddAliasExisting from '/snippets/_includes/code-examples/artifact_add_alias_existing.mdx'; @@ -61,6 +60,13 @@ import SweepConfig from '/snippets/_includes/code-examples/sweep_config.mdx'; import SweepCreate from '/snippets/_includes/code-examples/sweep_create.mdx'; import SweepInitialize from '/snippets/_includes/code-examples/sweep_initialize.mdx'; import SweepStart from '/snippets/_includes/code-examples/sweep_start.mdx'; +import WeaveEvalBasic from '/snippets/_includes/code-examples/weave_eval_basic.mdx'; +import WeaveExportMetrics from '/snippets/_includes/code-examples/weave_export_metrics.mdx'; +import WeavePublishDataset from '/snippets/_includes/code-examples/weave_publish_dataset.mdx'; +import WeaveScoringFunction from '/snippets/_includes/code-examples/weave_scoring_function.mdx'; +import WeaveTraceCall from '/snippets/_includes/code-examples/weave_trace_call.mdx'; +import WeaveTraceImages from '/snippets/_includes/code-examples/weave_trace_images.mdx'; +import WeaveTraceOp from '/snippets/_includes/code-examples/weave_trace_op.mdx'; // Map filenames to imported content const snippets = { @@ -111,11 +117,18 @@ const snippets = { 'sweep_create.py': SweepCreate, 'sweep_initialize.py': SweepInitialize, 'sweep_start.py': SweepStart, + 'weave_eval_basic.py': WeaveEvalBasic, + 'weave_export_metrics.py': WeaveExportMetrics, + 'weave_publish_dataset.py': WeavePublishDataset, + 'weave_scoring_function.py': WeaveScoringFunction, + 'weave_trace_call.py': WeaveTraceCall, + 'weave_trace_images.py': WeaveTraceImages, + 'weave_trace_op.py': WeaveTraceOp, }; export const CodeSnippet = ({ file }) => { const Component = snippets[file]; - + if (!Component) { return (
@@ -123,7 +136,7 @@ export const CodeSnippet = ({ file }) => {
); } - + return ; }; diff --git a/snippets/_includes/code-examples/weave_eval_basic.mdx b/snippets/_includes/code-examples/weave_eval_basic.mdx new file mode 100644 index 0000000000..60bf9cb7d7 --- /dev/null +++ b/snippets/_includes/code-examples/weave_eval_basic.mdx @@ -0,0 +1,71 @@ +```python +''' +Create a basic Weave evaluation pipeline for scoring responses from a model. +''' +import json +import asyncio +import openai +import weave +from weave.scorers import MultiTaskBinaryClassificationF1 + +# Initialize Weave once +weave.init('your-team-name/your-project-name') + +# Define Model +class ExtractFruitsModel(weave.Model): + model_name: str + prompt_template: str + + @weave.op() + async def predict(self, sentence: str) -> dict: + client = openai.AsyncClient() + response = await client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": self.prompt_template.format(sentence=sentence)}], + ) + result = response.choices[0].message.content + if result is None: + raise ValueError("No response from model") + return json.loads(result) + +# Instantiate model +model = ExtractFruitsModel( + model_name='gpt-3.5-turbo-1106', + prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' +) + +# Create and publish dataset +sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", +"Pounits are a bright green color and are more savory than sweet.", +"Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] +labels = [ + {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, + {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, + {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} +] +examples = [ + {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, + {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, + {'id': '2', 'sentence': sentences[2], 'target': labels[2]} +] + +dataset = weave.Dataset(name='fruits', rows=examples) +weave.publish(dataset) + +# Define a scoring function +@weave.op() +def fruit_name_score(target: dict, output: dict) -> dict: + return {'correct': target['fruit'] == output['fruit']} + +# Run the evaluation +evaluation = weave.Evaluation( + name='fruit_eval', + dataset=dataset, + scorers=[ + MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), + fruit_name_score + ], +) + +print(asyncio.run(evaluation.evaluate(model))) +``` diff --git a/snippets/_includes/code-examples/weave_eval_basic.py b/snippets/_includes/code-examples/weave_eval_basic.py new file mode 100644 index 0000000000..01ccad8ce4 --- /dev/null +++ b/snippets/_includes/code-examples/weave_eval_basic.py @@ -0,0 +1,69 @@ +''' +Create a basic Weave evaluation pipeline for scoring responses from a model. +''' +import json +import asyncio +import openai +import weave +from weave.scorers import MultiTaskBinaryClassificationF1 + +# Initialize Weave once +weave.init('your-team-name/your-project-name') + +# Define Model +class ExtractFruitsModel(weave.Model): + model_name: str + prompt_template: str + + @weave.op() + async def predict(self, sentence: str) -> dict: + client = openai.AsyncClient() + response = await client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": self.prompt_template.format(sentence=sentence)}], + ) + result = response.choices[0].message.content + if result is None: + raise ValueError("No response from model") + return json.loads(result) + +# Instantiate model +model = ExtractFruitsModel( + model_name='gpt-3.5-turbo-1106', + prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' +) + +# Create and publish dataset +sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", +"Pounits are a bright green color and are more savory than sweet.", +"Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] +labels = [ + {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, + {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, + {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} +] +examples = [ + {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, + {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, + {'id': '2', 'sentence': sentences[2], 'target': labels[2]} +] + +dataset = weave.Dataset(name='fruits', rows=examples) +weave.publish(dataset) + +# Define a scoring function +@weave.op() +def fruit_name_score(target: dict, output: dict) -> dict: + return {'correct': target['fruit'] == output['fruit']} + +# Run the evaluation +evaluation = weave.Evaluation( + name='fruit_eval', + dataset=dataset, + scorers=[ + MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), + fruit_name_score + ], +) + +print(asyncio.run(evaluation.evaluate(model))) \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_export_metrics.mdx b/snippets/_includes/code-examples/weave_export_metrics.mdx new file mode 100644 index 0000000000..58f2e10a02 --- /dev/null +++ b/snippets/_includes/code-examples/weave_export_metrics.mdx @@ -0,0 +1,42 @@ +```python +''' +Retrieve metrics about your calls +''' + +import requests +import json +import os + +# Weave API URL +url = "https://trace.wandb.ai/calls/stats" + +# Configure the types of metrics to retrieve for a specified time range +payload = { + "project_id": "", +# Specify time range + "start": "2026-03-01T00:00:00Z", + "end": "2026-03-10T00:00:00Z", +# Specify the size of the buckets, in seconds. + "granularity": 86400, + "filter": { + "trace_roots_only": True, + "op_names": ["web_app"] + }, +# Specify metrics and their aggregate function + "usage_metrics": [ + {"metric": "total_tokens", "aggregations": ["sum"]}, + {"metric": "total_cost", "aggregations": ["sum"]} + ], + "call_metrics": [ + {"metric": "call_count", "aggregations": ["sum"]}, + {"metric": "error_count", "aggregations": ["sum"]}, + {"metric": "latency_ms", "aggregations": ["avg", "min", "max"], "percentiles": [50, 95, 99]} + ] +} + +API_KEY = os.getenv("WANDB_API_KEY") + +response = requests.post(url, json=payload, auth=("api", API_KEY)) + +print(json.dumps(response.json(), indent=2)) +``` diff --git a/snippets/_includes/code-examples/weave_export_metrics.py b/snippets/_includes/code-examples/weave_export_metrics.py new file mode 100644 index 0000000000..884be5d444 --- /dev/null +++ b/snippets/_includes/code-examples/weave_export_metrics.py @@ -0,0 +1,40 @@ +''' +Retrieve metrics about your calls +''' + +import requests +import json +import os + +# Weave API URL +url = "https://trace.wandb.ai/calls/stats" + +# Configure the types of metrics to retrieve for a specified time range +payload = { + "project_id": "", +# Specify time range + "start": "2026-03-01T00:00:00Z", + "end": "2026-03-10T00:00:00Z", +# Specify the size of the buckets, in seconds. + "granularity": 86400, + "filter": { + "trace_roots_only": True, + "op_names": ["web_app"] + }, +# Specify metrics and their aggregate function + "usage_metrics": [ + {"metric": "total_tokens", "aggregations": ["sum"]}, + {"metric": "total_cost", "aggregations": ["sum"]} + ], + "call_metrics": [ + {"metric": "call_count", "aggregations": ["sum"]}, + {"metric": "error_count", "aggregations": ["sum"]}, + {"metric": "latency_ms", "aggregations": ["avg", "min", "max"], "percentiles": [50, 95, 99]} + ] +} + +API_KEY = os.getenv("WANDB_API_KEY") + +response = requests.post(url, json=payload, auth=("api", API_KEY)) + +print(json.dumps(response.json(), indent=2)) \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_publish_dataset.mdx b/snippets/_includes/code-examples/weave_publish_dataset.mdx new file mode 100644 index 0000000000..e5a21ea13c --- /dev/null +++ b/snippets/_includes/code-examples/weave_publish_dataset.mdx @@ -0,0 +1,29 @@ +```python +''' +Publish a dataset to Weave +''' + +import weave +from weave import Dataset +# Initialize Weave +weave.init('/') + +# Create a dataset +dataset = Dataset( + name='grammar', + rows=[ + {'id': '0', 'sentence': "He no likes ice cream.", 'correction': "He doesn't like ice cream."}, + {'id': '1', 'sentence': "She goed to the store.", 'correction': "She went to the store."}, + {'id': '2', 'sentence': "They plays video games all day.", 'correction': "They play video games all day."} + ] +) + +# Publish the dataset +weave.publish(dataset) + +# Retrieve the dataset +dataset_ref = weave.ref('grammar').get() + +# Access a specific example in the dataset +example_label = dataset_ref.rows[2]['sentence'] +``` diff --git a/snippets/_includes/code-examples/weave_publish_dataset.py b/snippets/_includes/code-examples/weave_publish_dataset.py new file mode 100644 index 0000000000..d6509767f6 --- /dev/null +++ b/snippets/_includes/code-examples/weave_publish_dataset.py @@ -0,0 +1,27 @@ +''' +Publish a dataset to Weave +''' + +import weave +from weave import Dataset +# Initialize Weave +weave.init('/') + +# Create a dataset +dataset = Dataset( + name='grammar', + rows=[ + {'id': '0', 'sentence': "He no likes ice cream.", 'correction': "He doesn't like ice cream."}, + {'id': '1', 'sentence': "She goed to the store.", 'correction': "She went to the store."}, + {'id': '2', 'sentence': "They plays video games all day.", 'correction': "They play video games all day."} + ] +) + +# Publish the dataset +weave.publish(dataset) + +# Retrieve the dataset +dataset_ref = weave.ref('grammar').get() + +# Access a specific example in the dataset +example_label = dataset_ref.rows[2]['sentence'] \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_scoring_function.mdx b/snippets/_includes/code-examples/weave_scoring_function.mdx new file mode 100644 index 0000000000..76ce0110a0 --- /dev/null +++ b/snippets/_includes/code-examples/weave_scoring_function.mdx @@ -0,0 +1,20 @@ +```python +''' +Define custom scoring function +''' + +import weave + +# Collect your examples +examples = [ + {"question": "What is the capital of France?", "expected": "Paris"}, + {"question": "Who wrote 'To Kill a Mockingbird'?", "expected": "Harper Lee"}, + {"question": "What is the square root of 64?", "expected": "8"}, +] + +# Define any custom scoring function +@weave.op() +def match_score1(expected: str, output: dict) -> dict: + # Define the logic to score the output + return {'match': expected == output['generated_text']} +``` diff --git a/snippets/_includes/code-examples/weave_scoring_function.py b/snippets/_includes/code-examples/weave_scoring_function.py new file mode 100644 index 0000000000..ad7b135b2c --- /dev/null +++ b/snippets/_includes/code-examples/weave_scoring_function.py @@ -0,0 +1,18 @@ +''' +Define custom scoring function +''' + +import weave + +# Collect your examples +examples = [ + {"question": "What is the capital of France?", "expected": "Paris"}, + {"question": "Who wrote 'To Kill a Mockingbird'?", "expected": "Harper Lee"}, + {"question": "What is the square root of 64?", "expected": "8"}, +] + +# Define any custom scoring function +@weave.op() +def match_score1(expected: str, output: dict) -> dict: + # Define the logic to score the output + return {'match': expected == output['generated_text']} \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_trace_call.mdx b/snippets/_includes/code-examples/weave_trace_call.mdx new file mode 100644 index 0000000000..7d58fabaa1 --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_call.mdx @@ -0,0 +1,39 @@ +```python +''' +Trace a basic request to an LLM using W&B Weave's autopatching. +''' + +# Import W&B +import weave +from openai import OpenAI + +client = OpenAI() + +# Define a function that sends a request to a Weave supported LLM +def llm_request(question: str) -> dict: + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """You are a helpful assistant.""" + }, + { + "role": "user", + "content": question + } + ], + response_format={ "type": "text" } + ) + return response.choices[0].message.content + +# Initialize Weave with your team and project names +weave.init('/') + +question = "What is the capitol of Paris?" + +# Invoke function after Weave has been initialized +result = llm_request(question) + +print(result) +``` diff --git a/snippets/_includes/code-examples/weave_trace_call.py b/snippets/_includes/code-examples/weave_trace_call.py new file mode 100644 index 0000000000..7acd73a7ba --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_call.py @@ -0,0 +1,37 @@ +''' +Trace a basic request to an LLM using W&B Weave's autopatching. +''' + +# Import W&B +import weave +from openai import OpenAI + +client = OpenAI() + +# Define a function that sends a request to a Weave supported LLM +def llm_request(question: str) -> dict: + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """You are a helpful assistant.""" + }, + { + "role": "user", + "content": question + } + ], + response_format={ "type": "text" } + ) + return response.choices[0].message.content + +# Initialize Weave with your team and project names +weave.init('/') + +question = "What is the capitol of Paris?" + +# Invoke function after Weave has been initialized +result = llm_request(question) + +print(result) \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_trace_images.mdx b/snippets/_includes/code-examples/weave_trace_images.mdx new file mode 100644 index 0000000000..c21eea9917 --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_images.mdx @@ -0,0 +1,32 @@ +```python +''' +Trace images using Weave +''' + +import weave +from weave import Content +from PIL import Image, ImageDraw +from typing import Annotated + +weave.init('/') + +# Create and save a sample image +img = Image.new('RGB', (200, 100), color='lightblue') +draw = ImageDraw.Draw(img) +draw.text((50, 40), "Hello Weave!", fill='black') +img.save("sample_image.png") + +# Method 1: Content annotation (recommended) +@weave.op +def load_image_content(path: Annotated[str, Content]) -> Annotated[bytes, Content]: + with open(path, 'rb') as f: + return f.read() + +# Method 2: PIL Image object +@weave.op +def load_image_pil(path: Annotated[str, Content]) -> Image.Image: + return Image.open(path) + +result1 = load_image_content("sample_image.png") +result2 = load_image_pil("sample_image.png") +``` diff --git a/snippets/_includes/code-examples/weave_trace_images.py b/snippets/_includes/code-examples/weave_trace_images.py new file mode 100644 index 0000000000..a737594ae6 --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_images.py @@ -0,0 +1,30 @@ +''' +Trace images using Weave +''' + +import weave +from weave import Content +from PIL import Image, ImageDraw +from typing import Annotated + +weave.init('/') + +# Create and save a sample image +img = Image.new('RGB', (200, 100), color='lightblue') +draw = ImageDraw.Draw(img) +draw.text((50, 40), "Hello Weave!", fill='black') +img.save("sample_image.png") + +# Method 1: Content annotation (recommended) +@weave.op +def load_image_content(path: Annotated[str, Content]) -> Annotated[bytes, Content]: + with open(path, 'rb') as f: + return f.read() + +# Method 2: PIL Image object +@weave.op +def load_image_pil(path: Annotated[str, Content]) -> Image.Image: + return Image.open(path) + +result1 = load_image_content("sample_image.png") +result2 = load_image_pil("sample_image.png") \ No newline at end of file diff --git a/snippets/_includes/code-examples/weave_trace_op.mdx b/snippets/_includes/code-examples/weave_trace_op.mdx new file mode 100644 index 0000000000..a1dfa6c591 --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_op.mdx @@ -0,0 +1,18 @@ +```python +''' +Trace any function using a W&B Weave decorator +''' + +# Import W&B Weave +import weave + +# Add a Weave decorator to the function you want to trace +@weave.op +def helloUser(name: str): + return "Hello, " + name + +# Initialize Weave with your team and project names +weave.init('your-team-name/your-project-name') + +print(helloUser('')) +``` diff --git a/snippets/_includes/code-examples/weave_trace_op.py b/snippets/_includes/code-examples/weave_trace_op.py new file mode 100644 index 0000000000..5c187218c7 --- /dev/null +++ b/snippets/_includes/code-examples/weave_trace_op.py @@ -0,0 +1,16 @@ +''' +Trace any function using a W&B Weave decorator +''' + +# Import W&B Weave +import weave + +# Add a Weave decorator to the function you want to trace +@weave.op +def helloUser(name: str): + return "Hello, " + name + +# Initialize Weave with your team and project names +weave.init('your-team-name/your-project-name') + +print(helloUser('')) \ No newline at end of file