DeepgramAudioStreamer.cpp

#include <windows.h>
#include <mmdeviceapi.h>
#include <audioclient.h>
#include <avrt.h>
#include <iostream>
#include <vector>
#include <string>
#include <thread>
#include <chrono>
#include <cstdint>
#include <atomic>

// WebSocket libraries - using Beast (part of Boost)
#include <boost/beast/core.hpp>
#include <boost/beast/websocket.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/stream.hpp>

// For base64 encoding of API key
#include <boost/beast/core/detail/base64.hpp>

namespace beast = boost::beast;
namespace http = beast::http;
namespace websocket = beast::websocket;
namespace net = boost::asio;
namespace ssl = boost::asio::ssl;
using tcp = boost::asio::ip::tcp;

// Configuration parameters
struct DeepgramConfig {
    std::string apiKey;
    std::string model = "nova-3";
    std::string language = "en-US";
    bool interimResults = true;
    bool punctuate = true;
    int sampleRate = 16000;
    int encoding = 16;    // 16-bit PCM
    int channels = 1;     // Mono
};

// Audio resampler for converting to mono 16-bit PCM at 16kHz
class AudioResampler {
public:
    // Convert audio format to match Deepgram requirements
    static std::vector<int16_t> ConvertAudio(const BYTE* data, UINT32 frames, WAVEFORMATEX* format) {
        std::vector<int16_t> result;
        
        // Determine if this is a float format
        bool isFloat = (format->wFormatTag == 3); // WAVE_FORMAT_IEEE_FLOAT
        
        // For extensible format, check the SubFormat
        if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && format->cbSize >= 22) {
            WAVEFORMATEXTENSIBLE* extFormat = reinterpret_cast<WAVEFORMATEXTENSIBLE*>(format);
            isFloat = (extFormat->SubFormat.Data1 == 3); // Check first DWORD of GUID
        }
        
        // Calculate total samples
        UINT32 totalSamples = frames * format->nChannels;
        result.reserve(frames); // Reserve space for mono output
        
        // Process based on format
        if (isFloat && format->wBitsPerSample == 32) {
            // Convert 32-bit float to 16-bit int and downsample to mono
            const float* floatData = reinterpret_cast<const float*>(data);
            
            // Process all frames
            for (UINT32 frame = 0; frame < frames; frame++) {
                float sampleSum = 0.0f;
                
                // Mix all channels to mono
                for (UINT16 ch = 0; ch < format->nChannels; ch++) {
                    sampleSum += floatData[frame * format->nChannels + ch];
                }
                
                // Average the channels
                float monoSample = sampleSum / format->nChannels;
                
                // Clamp the value
                if (monoSample > 1.0f) monoSample = 1.0f;
                if (monoSample < -1.0f) monoSample = -1.0f;
                
                // Convert to int16
                int16_t sample = static_cast<int16_t>(monoSample * 32767.0f);
                result.push_back(sample);
            }
        }
        else if (format->wBitsPerSample == 16) {
            // Already 16-bit PCM, just need to downsample to mono
            const int16_t* pcmData = reinterpret_cast<const int16_t*>(data);
            
            // Process all frames
            for (UINT32 frame = 0; frame < frames; frame++) {
                int32_t sampleSum = 0;
                
                // Mix all channels to mono
                for (UINT16 ch = 0; ch < format->nChannels; ch++) {
                    sampleSum += pcmData[frame * format->nChannels + ch];
                }
                
                // Average the channels
                int16_t monoSample = static_cast<int16_t>(sampleSum / format->nChannels);
                result.push_back(monoSample);
            }
        }
        else if (format->wBitsPerSample == 32 && !isFloat) {
            // 32-bit integer to 16-bit
            const int32_t* intData = reinterpret_cast<const int32_t*>(data);
            
            // Process all frames
            for (UINT32 frame = 0; frame < frames; frame++) {
                int64_t sampleSum = 0;
                
                // Mix all channels to mono
                for (UINT16 ch = 0; ch < format->nChannels; ch++) {
                    sampleSum += intData[frame * format->nChannels + ch];
                }
                
                // Average and convert to 16-bit
                int16_t monoSample = static_cast<int16_t>(sampleSum / format->nChannels >> 16);
                result.push_back(monoSample);
            }
        }
        else if (format->wBitsPerSample == 24) {
            // 24-bit to 16-bit conversion with mono downmixing
            
            // Process all frames
            for (UINT32 frame = 0; frame < frames; frame++) {
                int32_t sampleSum = 0;
                
                // Mix all channels to mono
                for (UINT16 ch = 0; ch < format->nChannels; ch++) {
                    // Extract 24-bit sample (3 bytes) and convert to 32-bit
                    size_t sampleOffset = (frame * format->nChannels + ch) * 3; // 3 bytes per sample
                    int32_t sample = (data[sampleOffset] << 8) | (data[sampleOffset + 1] << 16) | (data[sampleOffset + 2] << 24);
                    sampleSum += sample;
                }
                
                // Average and convert to 16-bit
                int16_t monoSample = static_cast<int16_t>(sampleSum / format->nChannels >> 8);
                result.push_back(monoSample);
            }
        }
        else if (format->wBitsPerSample == 8) {
            // 8-bit to 16-bit conversion with mono downmixing
            
            // Process all frames
            for (UINT32 frame = 0; frame < frames; frame++) {
                int32_t sampleSum = 0;
                
                // Mix all channels to mono
                for (UINT16 ch = 0; ch < format->nChannels; ch++) {
                    // Convert 8-bit unsigned (0-255) to signed 16-bit (-32768 to 32767)
                    int16_t sample = static_cast<int16_t>((static_cast<int>(data[frame * format->nChannels + ch]) - 128) << 8);
                    sampleSum += sample;
                }
                
                // Average channels
                int16_t monoSample = static_cast<int16_t>(sampleSum / format->nChannels);
                result.push_back(monoSample);
            }
        }
        
        return result;
    }
    
    // Simple resampling if needed (for future implementation)
    static std::vector<int16_t> Resample(const std::vector<int16_t>& input, int originalRate, int targetRate) {
        // If rates match, return the original data
        if (originalRate == targetRate) {
            return input;
        }
        
        // Simple linear interpolation resampling
        // This is a basic implementation - for production, consider a better resampling algorithm
        double ratio = static_cast<double>(originalRate) / targetRate;
        size_t outputSize = static_cast<size_t>(input.size() / ratio);
        std::vector<int16_t> output(outputSize);
        
        for (size_t i = 0; i < outputSize; i++) {
            double pos = i * ratio;
            size_t idx = static_cast<size_t>(pos);
            double frac = pos - idx;
            
            if (idx + 1 < input.size()) {
                output[i] = static_cast<int16_t>((1.0 - frac) * input[idx] + frac * input[idx + 1]);
            } else {
                output[i] = input[idx];
            }
        }
        
        return output;
    }
};

// Deepgram WebSocket client
class DeepgramClient {
private:
    net::io_context ioc;
    ssl::context ctx{ssl::context::tlsv12_client};
    websocket::stream<ssl::stream<tcp::socket>> ws;
    DeepgramConfig config;
    std::thread receiveThread;
    std::atomic<bool> isConnected{false};
    std::atomic<bool> shouldStop{false};

public:
    DeepgramClient(const DeepgramConfig& config) 
        : config(config), ws(ioc, ctx) {
        
        // Set up SSL
        ctx.set_default_verify_paths();
        ctx.set_verify_mode(ssl::verify_peer);
    }
    
    ~DeepgramClient() {
        disconnect();
    }
    
    bool connect() {
        try {
            // Look up the domain name
            auto const results = resolver.resolve("api.deepgram.com", "443");
            
            // Make the connection on the IP address we get
            net::connect(ws.next_layer().next_layer(), results);
            
            // Perform the SSL handshake
            ws.next_layer().handshake(ssl::stream_base::client);
            
            // Set up WebSocket connection
            ws.set_option(websocket::stream_base::decorator(
                [this](websocket::request_type& req) {
                    req.set(http::field::authorization, "Token " + config.apiKey);
                }));
            
            // Build the WebSocket URL with query parameters
            std::string target = "/v1/listen?";
            target += "model=" + config.model;
            target += "&language=" + config.language;
            target += "&interim_results=" + std::string(config.interimResults ? "true" : "false");
            target += "&punctuate=" + std::string(config.punctuate ? "true" : "false");
            target += "&encoding=linear16";
            target += "&sample_rate=" + std::to_string(config.sampleRate);
            target += "&channels=" + std::to_string(config.channels);
            
            // Perform the WebSocket handshake
            ws.handshake("api.deepgram.com", target);
            
            isConnected = true;
            
            // Start the message receiving thread
            receiveThread = std::thread([this]() {
                receiveMessages();
            });
            
            return true;
        }
        catch(std::exception const& e) {
            std::cerr << "Error connecting to Deepgram: " << e.what() << std::endl;
            return false;
        }
    }
    
    void disconnect() {
        if (!isConnected) return;
        
        shouldStop = true;
        
        try {
            // Close the WebSocket connection
            ws.close(websocket::close_code::normal);
            
            // Wait for the receive thread to finish
            if (receiveThread.joinable()) {
                receiveThread.join();
            }
            
            isConnected = false;
        }
        catch(std::exception const& e) {
            std::cerr << "Error disconnecting from Deepgram: " << e.what() << std::endl;
        }
    }
    
    bool sendAudio(const std::vector<int16_t>& audioData) {
        if (!isConnected) return false;
        
        try {
            // Send the audio data as binary
            ws.binary(true);
            ws.write(net::buffer(audioData.data(), audioData.size() * sizeof(int16_t)));
            return true;
        }
        catch(std::exception const& e) {
            std::cerr << "Error sending audio to Deepgram: " << e.what() << std::endl;
            return false;
        }
    }
    
private:
    tcp::resolver resolver{ioc};
    
    void receiveMessages() {
        beast::flat_buffer buffer;
        
        while (!shouldStop) {
            try {
                // Read a message into the buffer
                ws.read(buffer);
                
                // Parse the JSON response
                std::string message(beast::buffers_to_string(buffer.data()));
                buffer.consume(buffer.size());
                
                // Print the transcription result
                std::cout << "Deepgram transcription: " << message << std::endl;
            }
            catch(websocket::close_reason const& reason) {
                std::cerr << "WebSocket closed: " << reason.reason << std::endl;
                break;
            }
            catch(std::exception const& e) {
                if (!shouldStop) {
                    std::cerr << "Error receiving message from Deepgram: " << e.what() << std::endl;
                }
                break;
            }
        }
    }
};

// Main audio capture and streaming class
class AudioStreamer {
private:
    IMMDeviceEnumerator* pEnumerator = nullptr;
    IMMDevice* pDevice = nullptr;
    IAudioClient* pAudioClient = nullptr;
    IAudioCaptureClient* pCaptureClient = nullptr;
    WAVEFORMATEX* pwfx = nullptr;
    
    DeepgramClient* deepgramClient = nullptr;
    DeepgramConfig config;
    
    std::atomic<bool> isRunning{false};
    std::thread captureThread;

public:
    AudioStreamer(const DeepgramConfig& config) : config(config) {}
    
    ~AudioStreamer() {
        stop();
        cleanup();
    }
    
    bool initialize() {
        HRESULT hr;
        
        // Initialize COM
        hr = CoInitialize(nullptr);
        if (FAILED(hr)) {
            std::cerr << "CoInitialize failed: " << std::hex << hr << std::endl;
            return false;
        }
        
        // Create device enumerator
        hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, 
                          __uuidof(IMMDeviceEnumerator), (void**)&pEnumerator);
        if (FAILED(hr)) {
            std::cerr << "Failed to create device enumerator: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Get default audio device
        hr = pEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, &pDevice);
        if (FAILED(hr)) {
            std::cerr << "Failed to get default audio device: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Activate audio client
        hr = pDevice->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, (void**)&pAudioClient);
        if (FAILED(hr)) {
            std::cerr << "Failed to activate audio client: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Get audio format
        hr = pAudioClient->GetMixFormat(&pwfx);
        if (FAILED(hr)) {
            std::cerr << "Failed to get mix format: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Print audio format for debugging
        std::cout << "Audio Format: " << std::endl;
        std::cout << "  Format Tag: " << pwfx->wFormatTag << std::endl;
        std::cout << "  Channels: " << pwfx->nChannels << std::endl;
        std::cout << "  Sample Rate: " << pwfx->nSamplesPerSec << std::endl;
        std::cout << "  Bits Per Sample: " << pwfx->wBitsPerSample << std::endl;
        
        // Initialize audio client
        hr = pAudioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_LOOPBACK,
                                  0, 0, pwfx, nullptr);
        if (FAILED(hr)) {
            std::cerr << "Failed to initialize audio client: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Get capture client
        hr = pAudioClient->GetService(__uuidof(IAudioCaptureClient), (void**)&pCaptureClient);
        if (FAILED(hr)) {
            std::cerr << "Failed to get capture client: " << std::hex << hr << std::endl;
            cleanup();
            return false;
        }
        
        // Create and connect to Deepgram
        deepgramClient = new DeepgramClient(config);
        if (!deepgramClient->connect()) {
            std::cerr << "Failed to connect to Deepgram" << std::endl;
            cleanup();
            return false;
        }
        
        return true;
    }
    
    bool start() {
        if (isRunning) return true;
        
        // Start audio capture
        HRESULT hr = pAudioClient->Start();
        if (FAILED(hr)) {
            std::cerr << "Failed to start audio capture: " << std::hex << hr << std::endl;
            return false;
        }
        
        isRunning = true;
        
        // Start capture thread
        captureThread = std::thread([this]() {
            captureAndStream();
        });
        
        return true;
    }
    
    void stop() {
        if (!isRunning) return;
        
        isRunning = false;
        
        // Wait for capture thread to complete
        if (captureThread.joinable()) {
            captureThread.join();
        }
        
        // Stop audio capture
        if (pAudioClient) {
            pAudioClient->Stop();
        }
        
        // Disconnect from Deepgram
        if (deepgramClient) {
            deepgramClient->disconnect();
        }
    }

private:
    void captureAndStream() {
        HRESULT hr;
        UINT32 packetLength = 0;
        BYTE* pData;
        DWORD flags;
        UINT32 numFrames;
        
        std::cout << "Audio capture started. Streaming to Deepgram..." << std::endl;
        
        while (isRunning) {
            // Get next packet size
            hr = pCaptureClient->GetNextPacketSize(&packetLength);
            if (FAILED(hr)) {
                std::cerr << "Failed to get next packet size: " << std::hex << hr << std::endl;
                break;
            }
            
            if (packetLength == 0) {
                // No data yet, wait a bit
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
                continue;
            }
            
            // Get the captured data
            hr = pCaptureClient->GetBuffer(&pData, &numFrames, &flags, nullptr, nullptr);
            if (FAILED(hr)) {
                std::cerr << "Failed to get buffer: " << std::hex << hr << std::endl;
                break;
            }
            
            // Skip silent frames
            if (!(flags & AUDCLNT_BUFFERFLAGS_SILENT)) {
                // Convert audio to format expected by Deepgram (16-bit PCM, mono, 16kHz)
                std::vector<int16_t> monoAudio = AudioResampler::ConvertAudio(pData, numFrames, pwfx);
                
                // Resample if needed
                if (pwfx->nSamplesPerSec != config.sampleRate) {
                    monoAudio = AudioResampler::Resample(monoAudio, pwfx->nSamplesPerSec, config.sampleRate);
                }
                
                // Send to Deepgram
                if (!deepgramClient->sendAudio(monoAudio)) {
                    std::cerr << "Failed to send audio to Deepgram" << std::endl;
                    break;
                }
            }
            
            // Release the buffer
            hr = pCaptureClient->ReleaseBuffer(numFrames);
            if (FAILED(hr)) {
                std::cerr << "Failed to release buffer: " << std::hex << hr << std::endl;
                break;
            }
        }
        
        std::cout << "Audio capture stopped." << std::endl;
    }
    
    void cleanup() {
        if (deepgramClient) {
            delete deepgramClient;
            deepgramClient = nullptr;
        }
        
        if (pCaptureClient) {
            pCaptureClient->Release();
            pCaptureClient = nullptr;
        }
        
        if (pAudioClient) {
            pAudioClient->Release();
            pAudioClient = nullptr;
        }
        
        if (pwfx) {
            CoTaskMemFree(pwfx);
            pwfx = nullptr;
        }
        
        if (pDevice) {
            pDevice->Release();
            pDevice = nullptr;
        }
        
        if (pEnumerator) {
            pEnumerator->Release();
            pEnumerator = nullptr;
        }
        
        CoUninitialize();
    }
};

int main(int argc, char* argv[]) {
    // Configuration
    DeepgramConfig config;
    
    if (argc < 2) {
        std::cerr << "Usage: " << argv[0] << " <Deepgram API Key>" << std::endl;
        std::cerr << "Example: " << argv[0] << " YOUR_DEEPGRAM_API_KEY" << std::endl;
        return 1;
    }
    
    // Get API key from command line
    config.apiKey = argv[1];
    
    // Create audio streamer
    AudioStreamer streamer(config);
    
    // Initialize
    if (!streamer.initialize()) {
        std::cerr << "Failed to initialize audio streamer" << std::endl;
        return 1;
    }
    
    // Start streaming
    if (!streamer.start()) {
        std::cerr << "Failed to start audio streaming" << std::endl;
        return 1;
    }
    
    std::cout << "Streaming desktop audio to Deepgram for transcription..." << std::endl;
    std::cout << "Press Enter to stop." << std::endl;
    
    // Wait for user to press Enter
    std::cin.get();
    
    // Stop streaming
    streamer.stop();
    
    std::cout << "Audio streaming stopped." << std::endl;
    
    return 0;
}