test: run examples end to end against the live API

.github/workflows/examples-e2e.yml

@@ -0,0 +1,41 @@
+name: Examples E2E
+
+on:
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  examples-e2e:
+    name: live Ruby examples
+    if: >-
+      github.ref == 'refs/heads/main' &&
+      github.repository == 'openai/openai-ruby'
+    runs-on: ubuntu-latest
+    environment: ci
+    timeout-minutes: 90
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1
+        with:
+          ruby-version: "4.0"
+          bundler-cache: true
+      - name: Run examples against the live API
+        env:
+          EXAMPLES_E2E_REPORT_DIR: ${{ runner.temp }}/examples-e2e
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: bundle exec rake test:examples:e2e
+      - name: Upload example result reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: examples-e2e-${{ github.run_id }}
+          path: ${{ runner.temp }}/examples-e2e
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -9,3 +9,4 @@ bin/tapioca
 Brewfile.lock.json
 doc/
 sorbet/tapioca/*
+tmp/

CONTRIBUTING.md

@@ -79,6 +79,26 @@ $ ./scripts/mock
 $ bundle exec rake test
 ```
 
+### Running examples end-to-end
+
+The live example suite executes every example marked as `covered` in
+`examples/e2e.yml`. It requires `OPENAI_API_KEY`, makes real API requests, and
+writes JSON and Markdown execution reports under `tmp/examples-e2e/` by default.
+
+```bash
+$ bundle exec rake test:examples:e2e
+```
+
+To validate the example inventory without making API requests:
+
+```bash
+$ bundle exec rake test:examples:inventory
+```
+
+Every `examples/**/*.rb` file must be classified as covered or explicitly
+excluded with a reason. In GitHub Actions, live execution is available only
+through the manually dispatched `Examples E2E` workflow.
+
 ## Linting and formatting
 
 This repository uses [rubocop](https://github.com/rubocop/rubocop) for linting and formatting of `*.rb` files; And [syntax_tree](https://github.com/ruby-syntax-tree/syntax_tree) is used for formatting of both `*.rbi` and `*.rbs` files.

README.md

@@ -112,7 +112,7 @@ puts(transcription.text)
 image = OpenAI::FilePart.new(Pathname("dog.jpg"), content_type: "image/jpeg")
 edited = openai.images.edit(
   prompt: "make this image look like a painting",
-  model: "gpt-image-1",
+  model: "gpt-image-2",
   size: "1024x1024",
   image: image
 )

Rakefile

@@ -30,7 +30,7 @@ multitask(:"docs:preview") do
 end
 
 desc("Run test suites; use `TEST=path/to/test.rb` to run a specific test file")
-multitask(:test) do
+multitask(test: [:"test:examples:inventory"]) do
   rb =
     FileList[ENV.fetch("TEST", "./test/**/*_test.rb")]
     .map { "require_relative(#{_1.dump});" }
@@ -151,6 +151,16 @@ end
 desc("Typecheck and validate everything")
 multitask(typecheck: [:"typecheck:sorbet", :"validate:rbs"])
 
+desc("Validate the Ruby example E2E inventory without making live requests")
+task("test:examples:inventory") do
+  ruby(*%w[scripts/examples-e2e.rb --inventory-only])
+end
+
+desc("Run covered Ruby examples end-to-end against the live API")
+task("test:examples:e2e") do
+  ruby(*%w[scripts/examples-e2e.rb])
+end
+
 desc("Lint and typecheck")
 multitask(lint: [:"lint:rubocop", :"lint:rubocop_directives", :typecheck])
 

examples/advanced_streaming.rb

@@ -31,6 +31,16 @@
     end
 
   pp(all_choices)
+  abort("The eager stream completed without choices") if all_choices.empty?
+  eager_stream_finished = all_choices.any? do |choice|
+    case choice[:finish_reason]
+    when String, Symbol then true
+    else false
+    end
+  end
+  unless eager_stream_finished
+    abort("The eager stream ended before a terminal choice was received")
+  end
 
   # once the stream has been consumed, it will become "empty"
   pp("this will print an empty array")
@@ -67,9 +77,19 @@
 
   # method calls that do not return another `enumerable` will consume the intermediary stream
   #   and perform cleanup
+  lazy_choice_count = 0
+  lazy_terminal_choice_count = 0
   stream_of_choices.each do |choice|
+    lazy_choice_count += 1
+    case choice[:finish_reason]
+    when String, Symbol then lazy_terminal_choice_count += 1
+    end
     pp(choice)
   end
+  abort("The lazy stream completed without choices") if lazy_choice_count.zero?
+  if lazy_terminal_choice_count.zero?
+    abort("The lazy stream ended before a terminal choice was received")
+  end
 
   # at this point the stream has been consumed already, so it will return an empty array
   pp(stream_of_choices.to_a)

examples/chat/streaming_basic.rb

@@ -13,11 +13,19 @@
   ]
 )
 
+content_received = false
+completion_received = false
 stream.each do |event|
   case event
   when OpenAI::Streaming::ChatContentDeltaEvent
+    content_received ||= !event.delta.strip.empty?
     print(event.delta)
   when OpenAI::Streaming::ChatContentDoneEvent
+    completion_received = true
     puts
   end
 end
+
+abort("The stream completed without content") unless content_received
+abort("The stream ended before content completion") unless completion_received
+puts("Streamed content received.")

examples/chat/streaming_follow_up.rb

@@ -16,6 +16,7 @@
 
 puts "First streamed completion:"
 assistant_text = ""
+first_stream_completed = false
 
 stream1 = client.chat.completions.stream(
   model: "gpt-4o-mini",
@@ -28,10 +29,14 @@
     assistant_text += event.delta
     print(event.delta)
   when OpenAI::Streaming::ChatContentDoneEvent
+    first_stream_completed = true
     puts
   end
 end
 
+abort("The first stream completed without content") if assistant_text.strip.empty?
+abort("The first stream ended before its content completed") unless first_stream_completed
+
 # 2. Start a new streamed completion that includes the prior assistant turn
 #    and adds a follow-up user instruction.
 messages << {role: :assistant, content: assistant_text}
@@ -45,14 +50,21 @@
   messages: messages
 )
 
+follow_up_text = ""
+follow_up_stream_completed = false
 stream2.each do |event|
   case event
   when OpenAI::Streaming::ChatContentDeltaEvent
+    follow_up_text += event.delta
     print(event.delta)
   when OpenAI::Streaming::ChatContentDoneEvent
+    follow_up_stream_completed = true
     puts
   end
 end
 
+abort("The follow-up stream completed without content") if follow_up_text.strip.empty?
+abort("The follow-up stream ended before its content completed") unless follow_up_stream_completed
+
 puts
 puts "Done. The second stream is a new completion that used the prior turns as context."

examples/chat/streaming_logprobs.rb

@@ -15,6 +15,7 @@
   ]
 )
 
+logprobs_received = false
 stream.each do |event|
   case event
   when OpenAI::Streaming::ChatContentDeltaEvent
@@ -27,6 +28,10 @@
     alts = last.top_logprobs.map { |t| "#{t.token}=#{format('%.2f', t.logprob)}" }.join(", ")
     puts("\nlogprobs: [#{alts}]")
   when OpenAI::Streaming::ChatLogprobsContentDoneEvent
+    abort("The logprobs stream completed without tokens") if event.content.empty?
+
+    logprobs_received = true
     puts("\n--- logprobs collection finished (#{event.content.length} tokens) ---")
   end
 end
+abort("The logprobs stream ended before completion") unless logprobs_received

examples/chat/streaming_multi_choice.rb

@@ -37,7 +37,15 @@
   end
 end
 
+expected_choice_indices = [0, 1]
+completed_choices = expected_choice_indices.select do |index|
+  choice_finished[index] && !choice_contents.fetch(index, "").strip.empty?
+end
+unless completed_choices == expected_choice_indices
+  abort("Expected two completed choices with content; received #{completed_choices.length}")
+end
+
 puts("------ final choices ------")
-choice_contents.keys.sort.each do |i|
+expected_choice_indices.each do |i|
   puts("[#{i}] #{choice_contents[i]}")
 end

examples/chat/streaming_structured_outputs.rb

@@ -38,9 +38,17 @@ class MathResponse < OpenAI::BaseModel
 
 puts
 puts("----- parsed outputs from final response -----")
+completed_parsed_output_received = false
 response
   .choices
   .each do |choice|
     # parsed is an instance of `MathResponse`
-    pp(choice.message.parsed)
+    parsed = choice.message.parsed
+    next unless parsed.is_a?(MathResponse) && !choice.finish_reason.nil?
+
+    completed_parsed_output_received = true
+    pp(parsed)
   end
+unless completed_parsed_output_received
+  abort("The final completion did not contain a completed parsed MathResponse")
+end

examples/chat/streaming_text.rb

@@ -13,7 +13,15 @@
   ]
 )
 
+content_received = false
 stream.text.each do |text|
+  content_received ||= !text.strip.empty?
   print(text)
 end
 puts
+
+abort("The text stream completed without content") unless content_received
+completion = stream.get_final_completion
+finished_choice_received = completion.choices.any? { |choice| !choice.finish_reason.nil? }
+abort("The text stream ended before content completion") unless finished_choice_received
+puts("Streamed content received.")

examples/chat/streaming_tools.rb

@@ -13,20 +13,26 @@ class GetWeather < OpenAI::BaseModel
 stream = client.chat.completions.stream(
   model: "gpt-4o-mini",
   tools: [GetWeather],
+  tool_choice: {type: :function, function: {name: "GetWeather"}},
   messages: [
     {role: :user, content: "Call get_weather with location San Francisco in JSON."}
   ]
 )
 
+tool_call_received = false
 stream.each do |event|
   case event
   when OpenAI::Streaming::ChatFunctionToolCallArgumentsDeltaEvent
     puts("delta: #{event.arguments_delta}")
     pp(event.parsed)
   when OpenAI::Streaming::ChatFunctionToolCallArgumentsDoneEvent
+    abort("The finalized tool call did not contain parsed arguments") unless event.parsed.is_a?(GetWeather)
+
+    tool_call_received = true
     puts("--- Tool call finalized ---")
     puts("name: #{event.name}")
     puts("args: #{event.arguments}")
     pp(event.parsed)
   end
 end
+abort("The stream ended without a finalized tool call") unless tool_call_received

examples/demo.rb

@@ -21,7 +21,9 @@
     ]
   )
 
-  pp(completion.choices.first&.message&.content)
+  content = completion.choices.first&.message&.content
+  abort("The standard request completed without content") if content.to_s.strip.empty?
+  pp(content)
 end
 
 begin
@@ -38,9 +40,23 @@
     ]
   )
 
+  streamed_content = String.new
+  terminal_choice_count = 0
   stream.each do |chunk|
     next if chunk.choices.to_a.empty?
 
-    pp(chunk.choices.first&.delta&.content)
+    terminal_choice_count += chunk.choices.count do |choice|
+      case choice[:finish_reason]
+      when String, Symbol then true
+      else false
+      end
+    end
+    content = chunk.choices.first&.delta&.content
+    streamed_content << content.to_s
+    pp(content)
+  end
+  abort("The streaming request completed without content") if streamed_content.strip.empty?
+  if terminal_choice_count.zero?
+    abort("The streaming request ended before a terminal choice was received")
   end
 end

examples/demo_sorbet.rb

@@ -21,7 +21,9 @@
     model: "gpt-4"
   )
 
-  pp(completion.choices.first&.message&.content)
+  content = completion.choices.first&.message&.content
+  abort("The named-arguments request completed without content") if content.to_s.strip.empty?
+  pp(content)
 end
 
 begin
@@ -57,5 +59,7 @@
   #   into compatible methods that have named arguments
   completion = client.chat.completions.create(**params)
 
-  pp(completion.choices.first&.message&.content)
+  content = completion.choices.first&.message&.content
+  abort("The params-class request completed without content") if content.to_s.strip.empty?
+  pp(content)
 end