{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a1b2c3d4",
   "metadata": {},
   "source": [
    "# Crowd SDK MCP — Dataset Builder Demo\n",
    "\n",
    "End-to-end walkthrough: create a dataset spec → generate cases → generate samples → inspect results.\n",
    "\n",
    "The dataset_builder module runs locally — no Tagme API required.\n",
    "It needs `litellm` and an LLM API key for generation steps.\n",
    "\n",
    "## Enabling the module\n",
    "\n",
    "Add to your MCP config:\n",
    "```json\n",
    "{\n",
    "  \"mcpServers\": {\n",
    "    \"crowd-sdk\": {\n",
    "      \"command\": \"tagme\",\n",
    "      \"args\": [\"mcp\", \"dataset\"]\n",
    "    }\n",
    "  }\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2c3d4e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from pathlib import Path\n",
    "from typing import Any, Dict\n",
    "\n",
    "import ssl\n",
    "\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "\n",
    "from mcp import ClientSession, StdioServerParameters\n",
    "from mcp.client.stdio import stdio_client\n",
    "\n",
    "ENV_FILE = Path('/path/to/.env')\n",
    "MCP_COMMAND = 'tagme'\n",
    "MCP_ARGS = ['mcp', 'dataset']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3d4e5f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_env(path: Path) -> None:\n",
    "    if not path.exists():\n",
    "        return\n",
    "\n",
    "    for raw_line in path.read_text(encoding='utf-8').splitlines():\n",
    "        line = raw_line.strip()\n",
    "        if not line or line.startswith('#') or '=' not in line:\n",
    "            continue\n",
    "\n",
    "        key, value = line.split('=', 1)\n",
    "        os.environ.setdefault(key.strip(), value.strip().strip('\"').strip(\"'\"))\n",
    "\n",
    "\n",
    "def server_params() -> StdioServerParameters:\n",
    "    load_env(ENV_FILE)\n",
    "    return StdioServerParameters(\n",
    "        command=MCP_COMMAND,\n",
    "        args=MCP_ARGS,\n",
    "        env=os.environ.copy(),\n",
    "        cwd=str(Path.cwd()),\n",
    "    )\n",
    "\n",
    "\n",
    "def payload_from_mcp_result(tool_result: Any) -> Any:\n",
    "    structured_content = getattr(tool_result, 'structuredContent', None) or {}\n",
    "    if 'result' in structured_content:\n",
    "        return structured_content['result']\n",
    "    return [content.model_dump(mode='json') for content in tool_result.content]\n",
    "\n",
    "\n",
    "async def call_tool(session: ClientSession, tool_name: str, arguments: Dict[str, Any] | None = None) -> Any:\n",
    "    result = await session.call_tool(tool_name, arguments or {})\n",
    "    return payload_from_mcp_result(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e5f6a7",
   "metadata": {},
   "source": [
    "## Step 1: Create dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5f6a7b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_NAME = \"dialog_quality_demo\"\n",
    "\n",
    "async with stdio_client(server_params()) as (read, write):\n",
    "    async with ClientSession(read, write) as session:\n",
    "        await session.initialize()\n",
    "        result = await call_tool(\n",
    "            session,\n",
    "            \"create_dataset\",\n",
    "            {\n",
    "                \"name\": DATASET_NAME,\n",
    "                \"dataset_type\": \"chat\",\n",
    "            },\n",
    "        )\n",
    "        payload = result\n",
    "        print(json.dumps(payload, ensure_ascii=False, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6a7b8c9",
   "metadata": {},
   "source": [
    "> Edit `datasets/dialog_quality_demo/spec.md` to describe your dataset — define `## Case Dimensions` for diverse coverage, and write generation instructions. Then edit `config.yaml` to set `n_samples`, `model`, etc."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7b8c9d0",
   "metadata": {},
   "source": [
    "## Step 2: Generate cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c9d0e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "async with stdio_client(server_params()) as (read, write):\n",
    "    async with ClientSession(read, write) as session:\n",
    "        await session.initialize()\n",
    "        result = await call_tool(\n",
    "            session,\n",
    "            \"generate_cases\",\n",
    "            {\n",
    "                \"name\": DATASET_NAME,\n",
    "                \"n_cases\": 10,\n",
    "            },\n",
    "        )\n",
    "        payload = result\n",
    "        print(f\"Generated {payload.get('count')} cases\")\n",
    "        print(\"\\nSample cases:\")\n",
    "        for c in payload.get(\"sample_cases\", []):\n",
    "            print(\" \", c)\n",
    "        print(\"\\nDimension coverage:\")\n",
    "        print(json.dumps(payload.get(\"dimension_coverage\", {}), ensure_ascii=False, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9d0e1f2",
   "metadata": {},
   "source": [
    "## Step 3: Generate dataset (dry run first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0e1f2a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "async with stdio_client(server_params()) as (read, write):\n",
    "    async with ClientSession(read, write) as session:\n",
    "        await session.initialize()\n",
    "        result = await call_tool(\n",
    "            session,\n",
    "            \"generate_dataset\",\n",
    "            {\n",
    "                \"name\": DATASET_NAME,\n",
    "                \"n_samples\": 5,\n",
    "                \"dry_run\": True,\n",
    "            },\n",
    "        )\n",
    "        payload = result\n",
    "        print(json.dumps(payload, ensure_ascii=False, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1f2a3b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Requires: ANTHROPIC_API_KEY or OPENAI_API_KEY in environment\n",
    "async with stdio_client(server_params()) as (read, write):\n",
    "    async with ClientSession(read, write) as session:\n",
    "        await session.initialize()\n",
    "        result = await call_tool(\n",
    "            session,\n",
    "            \"generate_dataset\",\n",
    "            {\n",
    "                \"name\": DATASET_NAME,\n",
    "                \"n_samples\": 5,\n",
    "            },\n",
    "        )\n",
    "        payload = result\n",
    "        print(f\"Generated: {payload.get('generated')}, Errors: {payload.get('errors')}\")\n",
    "        print(f\"Tokens used: {payload.get('tokens')}\")\n",
    "        print(\"\\nLast sample:\")\n",
    "        print(json.dumps(payload.get(\"last_sample\"), ensure_ascii=False, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a3b4c5",
   "metadata": {},
   "source": [
    "## Step 4: Inspect results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3b4c5d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "async with stdio_client(server_params()) as (read, write):\n",
    "    async with ClientSession(read, write) as session:\n",
    "        await session.initialize()\n",
    "        result = await call_tool(\n",
    "            session,\n",
    "            \"inspect_dataset\",\n",
    "            {\n",
    "                \"name\": DATASET_NAME,\n",
    "                \"n_examples\": 2,\n",
    "            },\n",
    "        )\n",
    "        payload = result\n",
    "        print(f\"Total samples: {payload.get('total_samples')}\")\n",
    "        print(f\"Token stats: {payload.get('token_stats')}\")\n",
    "        print(f\"Fields: {payload.get('fields')}\")\n",
    "        print(f\"Models used: {payload.get('models_used')}\")\n",
    "        print(\"\\nExamples:\")\n",
    "        for ex in payload.get(\"examples\", []):\n",
    "            print(json.dumps(ex, ensure_ascii=False, indent=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}