Azure-AI_Foundry

GPT-4o Vision — Image + Text

GPT-4o processes images and text in a single request, enabling applications that understand visual content.

Analyzing an Image

from openai import AzureOpenAI
import base64

client = AzureOpenAI(
    azure_endpoint="https://my-openai.openai.azure.com/",
    api_version="2024-08-01-preview",
)

# From a URL
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "Describe what you see in this image."},
                {
                    "type": "image_url",
                    "image_url": {"url": "https://example.com/photo.jpg"},
                },
            ],
        }
    ],
    max_tokens=300,
)
print(response.choices[0].message.content)

# From a local file (base64-encoded)
def encode_image(path: str) -> str:
    with open(path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

b64_image = encode_image("receipt.png")
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "Extract the total amount and date from this receipt."},
                {
                    "type": "image_url",
                    "image_url": {"url": f"data:image/png;base64,{b64_image}"},
                },
            ],
        }
    ],
)

Practical Vision Use Cases

Use Case	Input	Output
Receipt extraction	Photo of receipt	Structured JSON with items, totals
Diagram analysis	Architecture diagram	Text description of components
Quality inspection	Product photo	Defect classification
Accessibility	Any image	Alt-text description
Chart reading	Chart/graph image	Data interpretation

Whisper — Audio Transcription

# Transcribe audio to text
with open("meeting_recording.mp3", "rb") as audio_file:
    transcript = client.audio.transcriptions.create(
        model="whisper",
        file=audio_file,
        language="en",
        response_format="verbose_json",
        timestamp_granularities=["segment"],
    )

print(transcript.text)

# Access word-level timestamps
for segment in transcript.segments:
    print(f"[{segment['start']:.1f}s] {segment['text']}")

Text-to-Speech (TTS)

from pathlib import Path

response = client.audio.speech.create(
    model="tts-1-hd",
    voice="nova",       # Options: alloy, echo, fable, onyx, nova, shimmer
    input="Welcome to Azure AI Foundry. Let me help you get started.",
    speed=1.0,
)

# Save to file
speech_file = Path("output.mp3")
response.stream_to_file(speech_file)

# Stream audio in real-time (for web apps)
with response as stream:
    for chunk in stream.iter_bytes(chunk_size=4096):
        audio_player.write(chunk)  # Send to audio output

Combining Modalities

Build applications that combine vision, audio, and text in a single workflow:

async def analyze_meeting(video_path: str) -> dict:
    """Analyze a recorded meeting: extract frames, transcribe audio, summarize."""

    # Step 1: Transcribe audio
    with open(video_path, "rb") as f:
        transcript = client.audio.transcriptions.create(model="whisper", file=f)

    # Step 2: Extract key frames (every 60 seconds)
    frames = extract_key_frames(video_path, interval_seconds=60)

    # Step 3: Analyze slides/whiteboard from frames
    visual_notes = []
    for frame_b64 in frames:
        resp = client.chat.completions.create(
            model="gpt-4o",
            messages=[{
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe any slides, diagrams, or whiteboard content."},
                    {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{frame_b64}"}},
                ],
            }],
        )
        visual_notes.append(resp.choices[0].message.content)

    # Step 4: Generate combined summary
    summary = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "Summarize this meeting with action items."},
            {"role": "user", "content": f"Transcript:\n{transcript.text}\n\nVisual notes:\n{chr(10).join(visual_notes)}"},
        ],
    )

    return {
        "transcript": transcript.text,
        "visual_notes": visual_notes,
        "summary": summary.choices[0].message.content,
    }

Video Analysis Pattern

GPT-4o doesn’t accept video directly. Extract frames and analyze them:

import cv2

def extract_key_frames(video_path: str, interval_seconds: int = 30) -> list[str]:
    """Extract one frame per interval and return as base64 strings."""
    cap = cv2.VideoCapture(video_path)
    fps = cap.get(cv2.CAP_PROP_FPS)
    frames = []

    frame_interval = int(fps * interval_seconds)
    frame_count = 0

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        if frame_count % frame_interval == 0:
            _, buffer = cv2.imencode(".jpg", frame)
            b64 = base64.b64encode(buffer).decode("utf-8")
            frames.append(b64)
        frame_count += 1

    cap.release()
    return frames

Tip: For vision tasks, use the detail parameter: "low" for faster/cheaper analysis, "high" for detailed inspection. Each image at "high" detail uses more tokens but provides better accuracy for small text or fine details.